From 07dd9c160daf39974c3cf515f1aa5f64f1d209cd Mon Sep 17 00:00:00 2001 From: Cedric Cordenier Date: Wed, 5 Aug 2026 14:43:29 +0100 Subject: [PATCH] cleanup consensus cap --- pkg/capabilities/consensus/ocr3/batching.go | 348 --- .../consensus/ocr3/batching_test.go | 1860 ----------------- .../consensus/ocr3/benchmark_test.go | 494 ----- pkg/capabilities/consensus/ocr3/capability.go | 303 --- .../consensus/ocr3/capability_test.go | 541 ----- pkg/capabilities/consensus/ocr3/factory.go | 104 - pkg/capabilities/consensus/ocr3/models.go | 22 - pkg/capabilities/consensus/ocr3/ocr3.go | 112 - pkg/capabilities/consensus/ocr3/ocr3_test.go | 71 - .../consensus/ocr3/reporting_plugin.go | 611 ------ .../consensus/ocr3/reporting_plugin_test.go | 1302 ------------ .../testdata/fixtures/capability/schema.json | 140 -- .../testdata/fixtures/capability/test.yaml | 21 - .../consensus/ocr3/transmitter.go | 159 -- .../consensus/ocr3/transmitter_test.go | 194 -- .../consensus/ocr3/validation_service.go | 38 - .../consensus/ocr3/value_map_encoder.go | 19 - .../consensus/ocr3/value_map_encoder_test.go | 65 - .../fixtures_test.go} | 35 +- .../consensus/requests/handler_test.go | 35 +- .../consensus/requests/store_stats_test.go | 5 +- .../consensus/requests/store_test.go | 31 +- 22 files changed, 52 insertions(+), 6458 deletions(-) delete mode 100644 pkg/capabilities/consensus/ocr3/batching.go delete mode 100644 pkg/capabilities/consensus/ocr3/batching_test.go delete mode 100644 pkg/capabilities/consensus/ocr3/benchmark_test.go delete mode 100644 pkg/capabilities/consensus/ocr3/capability.go delete mode 100644 pkg/capabilities/consensus/ocr3/capability_test.go delete mode 100644 pkg/capabilities/consensus/ocr3/factory.go delete mode 100644 pkg/capabilities/consensus/ocr3/models.go delete mode 100644 pkg/capabilities/consensus/ocr3/ocr3.go delete mode 100644 pkg/capabilities/consensus/ocr3/ocr3_test.go delete mode 100644 pkg/capabilities/consensus/ocr3/reporting_plugin.go delete mode 100644 pkg/capabilities/consensus/ocr3/reporting_plugin_test.go delete mode 100644 pkg/capabilities/consensus/ocr3/testdata/fixtures/capability/schema.json delete mode 100644 pkg/capabilities/consensus/ocr3/testdata/fixtures/capability/test.yaml delete mode 100644 pkg/capabilities/consensus/ocr3/transmitter.go delete mode 100644 pkg/capabilities/consensus/ocr3/transmitter_test.go delete mode 100644 pkg/capabilities/consensus/ocr3/validation_service.go delete mode 100644 pkg/capabilities/consensus/ocr3/value_map_encoder.go delete mode 100644 pkg/capabilities/consensus/ocr3/value_map_encoder_test.go rename pkg/capabilities/consensus/{ocr3/report_request.go => requests/fixtures_test.go} (67%) diff --git a/pkg/capabilities/consensus/ocr3/batching.go b/pkg/capabilities/consensus/ocr3/batching.go deleted file mode 100644 index 7d4a50b744..0000000000 --- a/pkg/capabilities/consensus/ocr3/batching.go +++ /dev/null @@ -1,348 +0,0 @@ -package ocr3 - -import ( - "google.golang.org/protobuf/proto" - - "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/ocr3/types" - "github.com/smartcontractkit/chainlink-protos/cre/go/values/pb" -) - -type idKey struct { - WorkflowExecutionId string - WorkflowId string - WorkflowOwner string - WorkflowName string - WorkflowDonId uint32 - WorkflowDonConfigVersion uint32 - ReportId string - KeyId string -} - -func GetIDKey(rq *ReportRequest) idKey { - return idKey{ - WorkflowExecutionId: rq.WorkflowExecutionID, - WorkflowId: rq.WorkflowID, - WorkflowOwner: rq.WorkflowOwner, - WorkflowName: rq.WorkflowName, - WorkflowDonId: rq.WorkflowDonID, - WorkflowDonConfigVersion: rq.WorkflowDonConfigVersion, - ReportId: rq.ReportID, - KeyId: rq.KeyID, - } -} - -// varintSize calculates the size of a varint encoding for the given value -func varintSize(x uint64) int { - if x == 0 { - return 1 - } - size := 0 - for x > 0 { - size++ - x >>= 7 - } - return size -} - -// stringFieldSize calculates the protobuf wire format size for a string field -func stringFieldSize(fieldNumber int, s string) int { - if len(s) == 0 { - return 0 // empty strings are omitted in proto3 - } - tagSize := varintSize(uint64(fieldNumber<<3 | 2)) // wire type 2 for length-delimited - lengthSize := varintSize(uint64(len(s))) - return tagSize + lengthSize + len(s) -} - -// uint32FieldSize calculates the protobuf wire format size for a uint32 field -func uint32FieldSize(fieldNumber int, value uint32) int { - if value == 0 { - return 0 // zero values are omitted in proto3 - } - tagSize := varintSize(uint64(fieldNumber << 3)) // wire type 0 for varint - valueSize := varintSize(uint64(value)) - return tagSize + valueSize -} - -// calculateIdSize calculates the marshalled size of a single types.Id -func calculateIdSize(id *types.Id) int { - size := 0 - - // Field 1: workflowExecutionId (string) - size += stringFieldSize(1, id.WorkflowExecutionId) - - // Field 2: workflowId (string) - size += stringFieldSize(2, id.WorkflowId) - - // Field 3: workflowOwner (string) - size += stringFieldSize(3, id.WorkflowOwner) - - // Field 4: workflowName (string) - size += stringFieldSize(4, id.WorkflowName) - - // Field 6: reportId (string) - size += stringFieldSize(6, id.ReportId) - - // Field 7: workflowDonId (uint32) - size += uint32FieldSize(7, id.WorkflowDonId) - - // Field 8: workflowDonConfigVersion (uint32) - size += uint32FieldSize(8, id.WorkflowDonConfigVersion) - - // Field 9: keyId (string) - size += stringFieldSize(9, id.KeyId) - - return size -} - -// calculateQuerySize calculates the precise marshalled size of a types.Query -func calculateQuerySize(ids []*types.Id) int { - if len(ids) == 0 { - return 0 - } - - totalSize := 0 - - for _, id := range ids { - idSize := calculateIdSize(id) - // Each repeated field element includes: - // - tag for field 1 (ids field in Query message) - // - length of the Id message - // - the Id message content - // Note: Even empty messages (idSize=0) contribute tag+length overhead - tagSize := varintSize(uint64(1<<3 | 2)) // field 1, wire type 2 - lengthSize := varintSize(uint64(idSize)) - totalSize += tagSize + lengthSize + idSize - } - - return totalSize -} - -func QueryBatchHasCapacity(cachedSize int, newId *types.Id, sizeLimit int) (bool, int) { - // Calculate size if we add one more id - newIdSize := calculateIdSize(newId) - // Always add tag and length overhead, even for empty messages - totalSizeWithNewId := cachedSize + varintSize(uint64(1<<3|2)) + varintSize(uint64(newIdSize)) + newIdSize - - // Check against limits - if totalSizeWithNewId > sizeLimit { - // Stop adding more ids - return false, cachedSize - } - - return true, totalSizeWithNewId -} - -// messageFieldSize calculates the protobuf wire format size for a message field -func messageFieldSize(fieldNumber int, msg proto.Message) int { - if msg == nil { - return 0 // nil messages are omitted in proto3 - } - msgSize := proto.Size(msg) - if msgSize == 0 { - return 0 // empty messages are omitted in proto3 - } - tagSize := varintSize(uint64(fieldNumber<<3 | 2)) // wire type 2 for length-delimited - lengthSize := varintSize(uint64(msgSize)) - return tagSize + lengthSize + msgSize -} - -// listFieldSize calculates the protobuf wire format size for a List field -// This handles the special case where empty List{Fields: []} contributes tag+length overhead -func listFieldSize(fieldNumber int, list *pb.List) int { - if list == nil { - return 0 // nil lists are omitted in proto3 - } - msgSize := proto.Size(list) - // Even empty lists contribute tag + length overhead when explicitly set - tagSize := varintSize(uint64(fieldNumber<<3 | 2)) // wire type 2 for length-delimited - lengthSize := varintSize(uint64(msgSize)) - return tagSize + lengthSize + msgSize -} - -// mapFieldSize calculates the protobuf wire format size for a Map field -// This handles the special case where empty Map{Fields: map[string]*Value{}} contributes tag+length overhead -func mapFieldSize(fieldNumber int, mapField *pb.Map) int { - if mapField == nil { - return 0 // nil maps are omitted in proto3 - } - msgSize := proto.Size(mapField) - // Even empty maps contribute tag + length overhead when explicitly set - tagSize := varintSize(uint64(fieldNumber<<3 | 2)) // wire type 2 for length-delimited - lengthSize := varintSize(uint64(msgSize)) - return tagSize + lengthSize + msgSize -} - -// calculateObservationSize calculates the marshalled size of a single types.Observation -func calculateObservationSize(obs *types.Observation) int { - size := 0 - - // Field 1: id (Id message) - size += messageFieldSize(1, obs.Id) - - // Field 4: observations (values.v1.List message) - size += listFieldSize(4, obs.Observations) - - // Field 5: overriddenEncoderName (string) - size += stringFieldSize(5, obs.OverriddenEncoderName) - - // Field 6: overriddenEncoderConfig (values.v1.Map message) - size += mapFieldSize(6, obs.OverriddenEncoderConfig) - - return size -} - -// calculateObservationsSize calculates the precise marshalled size of a types.Observations -func calculateObservationsSize(observations []*types.Observation) int { - if len(observations) == 0 { - return 0 - } - - totalSize := 0 - - for _, obs := range observations { - obsSize := calculateObservationSize(obs) - // Each repeated field element includes: - // - tag for field 1 (observations field in Observations message) - // - length of the Observation message - // - the Observation message content - // Note: Even empty messages (obsSize=0) contribute tag+length overhead - tagSize := varintSize(uint64(1<<3 | 2)) // field 1, wire type 2 - lengthSize := varintSize(uint64(obsSize)) - totalSize += tagSize + lengthSize + obsSize - } - - return totalSize -} - -// checkObservationSizeLimit checks if adding a new observation would exceed the size limit -func checkObservationSizeLimit(cachedSize int, newObs *types.Observation, sizeLimit int) (bool, int) { - // Calculate size if we add one more observation - newObsSize := calculateObservationSize(newObs) - // Always add tag and length overhead, even for empty messages - totalSizeWithNewObs := cachedSize + varintSize(uint64(1<<3|2)) + varintSize(uint64(newObsSize)) + newObsSize - - // Check against limits - if totalSizeWithNewObs > sizeLimit { - // Stop adding more observations - return false, cachedSize - } - - return true, totalSizeWithNewObs -} - -// repeatedStringFieldSize calculates the protobuf wire format size for repeated string fields -func repeatedStringFieldSize(fieldNumber int, strings []string) int { - totalSize := 0 - for _, s := range strings { - if len(s) > 0 { - // Each string in repeated field has its own tag and length - tagSize := varintSize(uint64(fieldNumber<<3 | 2)) // wire type 2 for length-delimited - lengthSize := varintSize(uint64(len(s))) - totalSize += tagSize + lengthSize + len(s) - } - } - return totalSize -} - -// CalculateObservationsMessageSize calculates the marshalled size of a types.Observations message -func CalculateObservationsMessageSize(observations *types.Observations) int { - if observations == nil { - return 0 - } - - size := 0 - - // Field 1: observations (repeated Observation) - for _, obs := range observations.Observations { - obsSize := calculateObservationSize(obs) - // Always include tag and length overhead, even for empty messages - tagSize := varintSize(uint64(1<<3 | 2)) // field 1, wire type 2 - lengthSize := varintSize(uint64(obsSize)) - size += tagSize + lengthSize + obsSize - } - - // Field 2: registeredWorkflowIds (repeated string) - size += repeatedStringFieldSize(2, observations.RegisteredWorkflowIds) - - // Field 3: timestamp (google.protobuf.Timestamp message) - size += messageFieldSize(3, observations.Timestamp) - - return size -} - -// ObservationsBatchHasCapacity checks if adding a new observation to a types.Observations would exceed the size limit -func ObservationsBatchHasCapacity(cachedSize int, newObs *types.Observation, sizeLimit int) (bool, int) { - // Calculate size if we add one more observation to the observations field - newObsSize := calculateObservationSize(newObs) - // Always add tag and length overhead, even for empty messages - totalSizeWithNewObs := cachedSize + varintSize(uint64(1<<3|2)) + varintSize(uint64(newObsSize)) + newObsSize - - // Check against limits - if totalSizeWithNewObs > sizeLimit { - // Stop adding more observations - return false, cachedSize - } - - return true, totalSizeWithNewObs -} - -// calculateReportSize calculates the marshalled size of a single types.Report -func calculateReportSize(report *types.Report) int { - if report == nil { - return 0 - } - - size := 0 - - // Field 1: id (Id message) - size += messageFieldSize(1, report.Id) - - // Field 2: outcome (AggregationOutcome message) - size += messageFieldSize(2, report.Outcome) - - return size -} - -// calculateReportsSize calculates the precise marshalled size of current_reports from types.Outcome -func calculateReportsSize(reports []*types.Report) int { - if len(reports) == 0 { - return 0 - } - - totalSize := 0 - - for _, report := range reports { - reportSize := calculateReportSize(report) - // Each repeated field element includes: - // - tag for field 2 (current_reports field in Outcome message) - // - length of the Report message - // - the Report message content - // Note: Even empty messages (reportSize=0) contribute tag+length overhead - tagSize := varintSize(uint64(2<<3 | 2)) // field 2, wire type 2 - lengthSize := varintSize(uint64(reportSize)) - totalSize += tagSize + lengthSize + reportSize - } - - return totalSize -} - -// ReportBatchHasCapacity checks if adding a new report to the outcome would exceed size limits -func ReportBatchHasCapacity(cachedSize int, newReport *types.Report, sizeLimit int) (bool, int) { - if newReport == nil { - return true, cachedSize - } - - // Calculate size if we add one more report - newReportSize := calculateReportSize(newReport) - // Always add tag and length overhead, even for empty messages - totalSizeWithNewReport := cachedSize + varintSize(uint64(2<<3|2)) + varintSize(uint64(newReportSize)) + newReportSize - - // Check against limits - if totalSizeWithNewReport > sizeLimit { - // Stop adding more reports - return false, cachedSize - } - - return true, totalSizeWithNewReport -} diff --git a/pkg/capabilities/consensus/ocr3/batching_test.go b/pkg/capabilities/consensus/ocr3/batching_test.go deleted file mode 100644 index 3d8d83f695..0000000000 --- a/pkg/capabilities/consensus/ocr3/batching_test.go +++ /dev/null @@ -1,1860 +0,0 @@ -package ocr3 - -import ( - "testing" - "time" - - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/timestamppb" - - pbtypes "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/ocr3/types" - pbvalues "github.com/smartcontractkit/chainlink-protos/cre/go/values/pb" -) - -func TestQueryBatchHasCapacity(t *testing.T) { - // Helper function to create a simple ID with predictable size - createSimpleId := func(workflowExecutionId string) *pbtypes.Id { - return &pbtypes.Id{ - WorkflowExecutionId: workflowExecutionId, - WorkflowId: "workflow-1", - WorkflowOwner: "owner", - WorkflowName: "test", - ReportId: "report-1", - KeyId: "key-1", - } - } - - // Helper function to create an ID with all fields populated for larger size - createLargeId := func(suffix string) *pbtypes.Id { - return &pbtypes.Id{ - WorkflowExecutionId: "very-long-workflow-execution-id-" + suffix, - WorkflowId: "very-long-workflow-id-" + suffix, - WorkflowOwner: "very-long-workflow-owner-" + suffix, - WorkflowName: "very-long-workflow-name-" + suffix, - ReportId: "very-long-report-id-" + suffix, - WorkflowDonId: 12345, - WorkflowDonConfigVersion: 67890, - KeyId: "very-long-key-id-" + suffix, - } - } - - // Helper function to create an empty ID (zero values) - createEmptyId := func() *pbtypes.Id { - return &pbtypes.Id{} - } - - tests := []struct { - name string - existingIds []*pbtypes.Id - newId *pbtypes.Id - sizeLimit int - expected bool - description string - }{ - // Zero ID objects tests - { - name: "empty list, empty new ID, small limit", - existingIds: []*pbtypes.Id{}, - newId: createEmptyId(), - sizeLimit: 10, - expected: true, // Empty ID requires 2 bytes (tag + length), but 10-byte limit is sufficient - description: "Adding empty ID to empty list should be within reasonable limit", - }, - { - name: "empty list, empty new ID, zero limit", - existingIds: []*pbtypes.Id{}, - newId: createEmptyId(), - sizeLimit: 0, - expected: false, // Empty ID requires 2 bytes (tag + length) - description: "Empty ID should not fit in zero limit (requires tag + length overhead)", - }, - { - name: "empty list, simple ID, zero limit", - existingIds: []*pbtypes.Id{}, - newId: createSimpleId("exec-1"), - sizeLimit: 0, - expected: false, // Simple ID has size > 0, exceeds zero limit - description: "Non-empty ID should not fit in zero limit", - }, - - // Within limits tests - { - name: "empty list, simple ID, generous limit", - existingIds: []*pbtypes.Id{}, - newId: createSimpleId("exec-1"), - sizeLimit: 1000, - expected: true, - description: "Simple ID should fit in generous limit", - }, - { - name: "one existing ID, add another simple ID, generous limit", - existingIds: []*pbtypes.Id{createSimpleId("exec-1")}, - newId: createSimpleId("exec-2"), - sizeLimit: 1000, - expected: true, - description: "Two simple IDs should fit in generous limit", - }, - { - name: "three existing IDs, add fourth, generous limit", - existingIds: []*pbtypes.Id{ - createSimpleId("exec-1"), - createSimpleId("exec-2"), - createSimpleId("exec-3"), - }, - newId: createSimpleId("exec-4"), - sizeLimit: 1000, - expected: true, - description: "Four simple IDs should fit in generous limit", - }, - - // Above limits tests - { - name: "empty list, simple ID, very small limit", - existingIds: []*pbtypes.Id{}, - newId: createSimpleId("exec-1"), - sizeLimit: 1, - expected: false, - description: "Simple ID should exceed very small limit", - }, - { - name: "one existing ID, add large ID, small limit", - existingIds: []*pbtypes.Id{createSimpleId("exec-1")}, - newId: createLargeId("large"), - sizeLimit: 100, - expected: false, - description: "Large ID should exceed small limit when added to existing", - }, - { - name: "multiple existing IDs, add another, tight limit", - existingIds: []*pbtypes.Id{ - createSimpleId("exec-1"), - createSimpleId("exec-2"), - createSimpleId("exec-3"), - }, - newId: createSimpleId("exec-4"), - sizeLimit: 200, // Adjust based on actual size calculations - expected: false, - description: "Multiple IDs should exceed tight limit", - }, - - // Edge cases - { - name: "exactly at limit boundary", - existingIds: []*pbtypes.Id{}, - newId: createSimpleId("exec-1"), - sizeLimit: 0, // Will be set to exact size in the test - expected: true, - description: "ID exactly at limit should fit", - }, - { - name: "one byte over limit", - existingIds: []*pbtypes.Id{}, - newId: createSimpleId("exec-1"), - sizeLimit: 0, // Will be set to exact size - 1 in the test - expected: false, - description: "ID one byte over limit should not fit", - }, - { - name: "large ID alone", - existingIds: []*pbtypes.Id{}, - newId: createLargeId("huge"), - sizeLimit: 50, - expected: false, - description: "Large ID should exceed moderate limit", - }, - { - name: "mix of empty and non-empty existing IDs", - existingIds: []*pbtypes.Id{ - createEmptyId(), - createSimpleId("exec-1"), - createEmptyId(), - }, - newId: createSimpleId("exec-2"), - sizeLimit: 1000, - expected: true, - description: "Mix of empty and non-empty IDs should work correctly", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Handle special edge case tests that need dynamic size calculation - sizeLimit := tt.sizeLimit - switch tt.name { - case "exactly at limit boundary": - // Calculate exact size needed for the new ID - newIdSize := calculateIdSize(tt.newId) - if newIdSize > 0 { - tagSize := varintSize(uint64(1<<3 | 2)) - lengthSize := varintSize(uint64(newIdSize)) - sizeLimit = tagSize + lengthSize + newIdSize - } else { - sizeLimit = 0 - } - case "one byte over limit": - // Calculate exact size needed for the new ID minus 1 - newIdSize := calculateIdSize(tt.newId) - if newIdSize > 0 { - tagSize := varintSize(uint64(1<<3 | 2)) - lengthSize := varintSize(uint64(newIdSize)) - sizeLimit = tagSize + lengthSize + newIdSize - 1 - } else { - sizeLimit = -1 // This would be impossible, but for test completeness - } - } - - currentSize := calculateQuerySize(tt.existingIds) - result, _ := QueryBatchHasCapacity(currentSize, tt.newId, sizeLimit) - if result != tt.expected { - // Provide detailed debugging information - currentSize := calculateQuerySize(tt.existingIds) - newIdSize := calculateIdSize(tt.newId) - // Always add tag and length overhead, even for empty messages - totalSizeWithNewId := currentSize + varintSize(uint64(1<<3|2)) + varintSize(uint64(newIdSize)) + newIdSize - - t.Errorf("%s: enough() = %v, expected %v\n"+ - " Description: %s\n"+ - " Current size: %d\n"+ - " New ID size: %d\n"+ - " Total size with new ID: %d\n"+ - " Size limit: %d\n"+ - " Would exceed: %v", - tt.name, result, tt.expected, - tt.description, - currentSize, newIdSize, totalSizeWithNewId, sizeLimit, - totalSizeWithNewId > sizeLimit) - } - }) - } -} - -func TestQueryBatchHasCapacityWithRealSizes(t *testing.T) { - // Test with realistic size calculations to verify our understanding - simpleId := &pbtypes.Id{ - WorkflowExecutionId: "exec-123", - WorkflowId: "workflow-1", - WorkflowOwner: "owner", - WorkflowName: "test", - ReportId: "report-1", - KeyId: "key-1", - } - - t.Run("verify size calculations", func(t *testing.T) { - // Test empty list with simple ID - size := calculateQuerySize([]*pbtypes.Id{}) - if size != 0 { - t.Errorf("Empty list should have size 0, got %d", size) - } - - // Test single ID - singleIdSize := calculateQuerySize([]*pbtypes.Id{simpleId}) - if singleIdSize <= 0 { - t.Errorf("Single ID should have positive size, got %d", singleIdSize) - } - - t.Logf("Single ID size: %d bytes", singleIdSize) - - // Test that enough function works correctly with these sizes - result, _ := QueryBatchHasCapacity(0, simpleId, singleIdSize) - if !result { - t.Errorf("Should be able to add ID when limit equals exact size") - } - - result, _ = QueryBatchHasCapacity(0, simpleId, singleIdSize-1) - if result { - t.Errorf("Should not be able to add ID when limit is one byte less than size") - } - }) - - t.Run("verify behavior with empty IDs", func(t *testing.T) { - emptyId := &pbtypes.Id{} - ids := []*pbtypes.Id{simpleId} - currentSize := calculateQuerySize(ids) - result, _ := QueryBatchHasCapacity(currentSize, emptyId, 10000) - if !result { - t.Errorf("Should be able to add empty ID - it doesn't increase size") - } - }) -} - -func TestQueryBatchHasCapacityCaching(t *testing.T) { - // Test that the caching mechanism works correctly - id1 := &pbtypes.Id{WorkflowExecutionId: "exec-1", WorkflowId: "wf-1"} - id2 := &pbtypes.Id{WorkflowExecutionId: "exec-2", WorkflowId: "wf-2"} - id3 := &pbtypes.Id{WorkflowExecutionId: "exec-3", WorkflowId: "wf-3"} - - t.Run("incremental size calculation matches full recalculation", func(t *testing.T) { - // Build up incrementally using caching - cachedSize := 0 - ids := []*pbtypes.Id{} - - // Add first ID - canAdd, newSize := QueryBatchHasCapacity(cachedSize, id1, 10000) - if !canAdd { - t.Fatal("Should be able to add first ID") - } - ids = append(ids, id1) - cachedSize = newSize - - // Verify cached size matches full calculation - fullSize := calculateQuerySize(ids) - if cachedSize != fullSize { - t.Errorf("After adding id1: cached size %d != full calculation %d", cachedSize, fullSize) - } - - // Add second ID - canAdd, newSize = QueryBatchHasCapacity(cachedSize, id2, 10000) - if !canAdd { - t.Fatal("Should be able to add second ID") - } - ids = append(ids, id2) - cachedSize = newSize - - // Verify cached size matches full calculation - fullSize = calculateQuerySize(ids) - if cachedSize != fullSize { - t.Errorf("After adding id2: cached size %d != full calculation %d", cachedSize, fullSize) - } - - // Add third ID - canAdd, newSize = QueryBatchHasCapacity(cachedSize, id3, 10000) - if !canAdd { - t.Fatal("Should be able to add third ID") - } - ids = append(ids, id3) - cachedSize = newSize - - // Verify final cached size matches full calculation - fullSize = calculateQuerySize(ids) - if cachedSize != fullSize { - t.Errorf("After adding id3: cached size %d != full calculation %d", cachedSize, fullSize) - } - }) - - t.Run("size limit enforcement with caching", func(t *testing.T) { - // Calculate size of first two IDs - twoIds := []*pbtypes.Id{id1, id2} - twoIdsSize := calculateQuerySize(twoIds) - - // Set limit to exactly fit two IDs - limit := twoIdsSize - - // Build incrementally - cachedSize := 0 - - // Add first ID - canAdd, newSize := QueryBatchHasCapacity(cachedSize, id1, limit) - if !canAdd { - t.Fatal("Should be able to add first ID within limit") - } - cachedSize = newSize - - // Add second ID - canAdd, newSize = QueryBatchHasCapacity(cachedSize, id2, limit) - if !canAdd { - t.Fatal("Should be able to add second ID within limit") - } - cachedSize = newSize - - // Try to add third ID - should fail - canAdd, unchangedSize := QueryBatchHasCapacity(cachedSize, id3, limit) - if canAdd { - t.Error("Should not be able to add third ID - would exceed limit") - } - if unchangedSize != cachedSize { - t.Errorf("Size should remain unchanged when limit exceeded: got %d, expected %d", unchangedSize, cachedSize) - } - }) - - t.Run("empty ID handling with caching", func(t *testing.T) { - emptyId := &pbtypes.Id{} - cachedSize := 0 - - // Add empty ID - should add 2 bytes (tag + length) - canAdd, newSize := QueryBatchHasCapacity(cachedSize, emptyId, 1000) - if !canAdd { - t.Error("Should be able to add empty ID") - } - expectedSize := cachedSize + 2 // tag + length overhead - if newSize != expectedSize { - t.Errorf("Empty ID should add 2 bytes: got %d, expected %d", newSize, expectedSize) - } - - // Add real ID first - canAdd, newSize = QueryBatchHasCapacity(cachedSize, id1, 1000) - if !canAdd { - t.Fatal("Should be able to add real ID") - } - cachedSize = newSize - - // Add empty ID after real ID - should add 2 bytes (tag + length) - canAdd, newSize = QueryBatchHasCapacity(cachedSize, emptyId, 1000) - if !canAdd { - t.Error("Should be able to add empty ID after real ID") - } - expectedSize = cachedSize + 2 // tag + length overhead - if newSize != expectedSize { - t.Errorf("Empty ID should add 2 bytes after real ID: got %d, expected %d", newSize, expectedSize) - } - }) -} - -func TestQueryBatchHasCapacityPerformance(t *testing.T) { - // Performance test with many IDs - ids := make([]*pbtypes.Id, 100) - for i := range 100 { - ids[i] = &pbtypes.Id{ - WorkflowExecutionId: "exec-" + string(rune('A'+i%26)), - WorkflowId: "workflow-1", - ReportId: "report-1", - } - } - - newId := &pbtypes.Id{ - WorkflowExecutionId: "new-exec", - WorkflowId: "workflow-1", - ReportId: "report-1", - } - - t.Run("performance with many IDs", func(t *testing.T) { - currentSize := calculateQuerySize(ids) - result, _ := QueryBatchHasCapacity(currentSize, newId, 10000) - // Just ensure it completes without error - _ = result - }) -} - -func TestCheckObservationSizeLimit(t *testing.T) { - // Helper function to create a simple observation - createSimpleObservation := func(workflowExecutionId string) *pbtypes.Observation { - return &pbtypes.Observation{ - Id: &pbtypes.Id{ - WorkflowExecutionId: workflowExecutionId, - WorkflowId: "workflow-1", - WorkflowOwner: "owner", - WorkflowName: "test", - ReportId: "report-1", - KeyId: "key-1", - }, - OverriddenEncoderName: "encoder-1", - } - } - - // Helper function to create a large observation - createLargeObservation := func(suffix string) *pbtypes.Observation { - return &pbtypes.Observation{ - Id: &pbtypes.Id{ - WorkflowExecutionId: "very-long-workflow-execution-id-" + suffix, - WorkflowId: "very-long-workflow-id-" + suffix, - WorkflowOwner: "very-long-workflow-owner-" + suffix, - WorkflowName: "very-long-workflow-name-" + suffix, - ReportId: "very-long-report-id-" + suffix, - WorkflowDonId: 12345, - WorkflowDonConfigVersion: 67890, - KeyId: "very-long-key-id-" + suffix, - }, - OverriddenEncoderName: "very-long-encoder-name-" + suffix, - Observations: &pbvalues.List{ - Fields: []*pbvalues.Value{ - {Value: &pbvalues.Value_StringValue{StringValue: "observation-data-" + suffix}}, - {Value: &pbvalues.Value_Int64Value{Int64Value: 12345}}, - }, - }, - OverriddenEncoderConfig: &pbvalues.Map{ - Fields: map[string]*pbvalues.Value{ - "config-key-" + suffix: {Value: &pbvalues.Value_StringValue{StringValue: "config-value-" + suffix}}, - }, - }, - } - } - - // Helper function to create an empty observation - createEmptyObservation := func() *pbtypes.Observation { - return &pbtypes.Observation{} - } - - tests := []struct { - name string - existingObservations []*pbtypes.Observation - newObservation *pbtypes.Observation - sizeLimit int - expected bool - description string - }{ - // Zero observation objects tests - { - name: "empty list, empty new observation, small limit", - existingObservations: []*pbtypes.Observation{}, - newObservation: createEmptyObservation(), - sizeLimit: 10, - expected: true, - description: "Adding empty observation to empty list should be within any reasonable limit", - }, - { - name: "empty list, empty new observation, zero limit", - existingObservations: []*pbtypes.Observation{}, - newObservation: createEmptyObservation(), - sizeLimit: 0, - expected: false, - description: "Empty observation should not fit in zero limit (requires tag + length overhead)", - }, - { - name: "empty list, simple observation, zero limit", - existingObservations: []*pbtypes.Observation{}, - newObservation: createSimpleObservation("exec-1"), - sizeLimit: 0, - expected: false, - description: "Non-empty observation should not fit in zero limit", - }, - - // Within limits tests - { - name: "empty list, simple observation, generous limit", - existingObservations: []*pbtypes.Observation{}, - newObservation: createSimpleObservation("exec-1"), - sizeLimit: 1000, - expected: true, - description: "Simple observation should fit in generous limit", - }, - { - name: "one existing observation, add another simple observation, generous limit", - existingObservations: []*pbtypes.Observation{createSimpleObservation("exec-1")}, - newObservation: createSimpleObservation("exec-2"), - sizeLimit: 1000, - expected: true, - description: "Two simple observations should fit in generous limit", - }, - - // Above limits tests - { - name: "empty list, simple observation, very small limit", - existingObservations: []*pbtypes.Observation{}, - newObservation: createSimpleObservation("exec-1"), - sizeLimit: 1, - expected: false, - description: "Simple observation should exceed very small limit", - }, - { - name: "one existing observation, add large observation, small limit", - existingObservations: []*pbtypes.Observation{createSimpleObservation("exec-1")}, - newObservation: createLargeObservation("large"), - sizeLimit: 100, - expected: false, - description: "Large observation should exceed small limit when added to existing", - }, - - // Edge cases with complex values - { - name: "large observation alone", - existingObservations: []*pbtypes.Observation{}, - newObservation: createLargeObservation("huge"), - sizeLimit: 50, - expected: false, - description: "Large observation with complex values should exceed moderate limit", - }, - { - name: "mix of empty and non-empty existing observations", - existingObservations: []*pbtypes.Observation{ - createEmptyObservation(), - createSimpleObservation("exec-1"), - createEmptyObservation(), - }, - newObservation: createSimpleObservation("exec-2"), - sizeLimit: 1000, - expected: true, - description: "Mix of empty and non-empty observations should work correctly", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - currentSize := calculateObservationsSize(tt.existingObservations) - result, _ := checkObservationSizeLimit(currentSize, tt.newObservation, tt.sizeLimit) - if result != tt.expected { - // Provide detailed debugging information - currentSize := calculateObservationsSize(tt.existingObservations) - newObsSize := calculateObservationSize(tt.newObservation) - // Always add tag and length overhead, even for empty messages - totalSizeWithNewObs := currentSize + varintSize(uint64(1<<3|2)) + varintSize(uint64(newObsSize)) + newObsSize - - t.Errorf("%s: enoughObservation() = %v, expected %v\n"+ - " Description: %s\n"+ - " Current size: %d\n"+ - " New observation size: %d\n"+ - " Total size with new observation: %d\n"+ - " Size limit: %d\n"+ - " Would exceed: %v", - tt.name, result, tt.expected, - tt.description, - currentSize, newObsSize, totalSizeWithNewObs, tt.sizeLimit, - totalSizeWithNewObs > tt.sizeLimit) - } - }) - } -} - -func TestCheckObservationSizeLimitWithRealSizes(t *testing.T) { - // Test with realistic size calculations - simpleObs := &pbtypes.Observation{ - Id: &pbtypes.Id{ - WorkflowExecutionId: "exec-123", - WorkflowId: "workflow-1", - ReportId: "report-1", - }, - OverriddenEncoderName: "encoder-1", - } - - t.Run("verify observation size calculations", func(t *testing.T) { - // Test empty list - size := calculateObservationsSize([]*pbtypes.Observation{}) - if size != 0 { - t.Errorf("Empty observation list should have size 0, got %d", size) - } - - // Test single observation - singleObsSize := calculateObservationsSize([]*pbtypes.Observation{simpleObs}) - if singleObsSize <= 0 { - t.Errorf("Single observation should have positive size, got %d", singleObsSize) - } - - t.Logf("Single observation size: %d bytes", singleObsSize) - - // Test that enoughObservation function works correctly with these sizes - result, _ := checkObservationSizeLimit(0, simpleObs, singleObsSize) - if !result { - t.Errorf("Should be able to add observation when limit equals exact size") - } - - result, _ = checkObservationSizeLimit(0, simpleObs, singleObsSize-1) - if result { - t.Errorf("Should not be able to add observation when limit is one byte less than size") - } - }) -} - -func TestCheckObservationSizeLimitCaching(t *testing.T) { - // Test that the caching mechanism works correctly for observations - obs1 := &pbtypes.Observation{ - Id: &pbtypes.Id{WorkflowExecutionId: "exec-1", WorkflowId: "wf-1"}, - OverriddenEncoderName: "encoder-1", - } - obs2 := &pbtypes.Observation{ - Id: &pbtypes.Id{WorkflowExecutionId: "exec-2", WorkflowId: "wf-2"}, - OverriddenEncoderName: "encoder-2", - } - obs3 := &pbtypes.Observation{ - Id: &pbtypes.Id{WorkflowExecutionId: "exec-3", WorkflowId: "wf-3"}, - OverriddenEncoderName: "encoder-3", - } - - t.Run("incremental size calculation matches full recalculation", func(t *testing.T) { - // Build up incrementally using caching - cachedSize := 0 - observations := []*pbtypes.Observation{} - - // Add first observation - canAdd, newSize := checkObservationSizeLimit(cachedSize, obs1, 10000) - if !canAdd { - t.Fatal("Should be able to add first observation") - } - observations = append(observations, obs1) - cachedSize = newSize - - // Verify cached size matches full calculation - fullSize := calculateObservationsSize(observations) - if cachedSize != fullSize { - t.Errorf("After adding obs1: cached size %d != full calculation %d", cachedSize, fullSize) - } - - // Add second observation - canAdd, newSize = checkObservationSizeLimit(cachedSize, obs2, 10000) - if !canAdd { - t.Fatal("Should be able to add second observation") - } - observations = append(observations, obs2) - cachedSize = newSize - - // Verify cached size matches full calculation - fullSize = calculateObservationsSize(observations) - if cachedSize != fullSize { - t.Errorf("After adding obs2: cached size %d != full calculation %d", cachedSize, fullSize) - } - - // Add third observation - canAdd, newSize = checkObservationSizeLimit(cachedSize, obs3, 10000) - if !canAdd { - t.Fatal("Should be able to add third observation") - } - observations = append(observations, obs3) - cachedSize = newSize - - // Verify final cached size matches full calculation - fullSize = calculateObservationsSize(observations) - if cachedSize != fullSize { - t.Errorf("After adding obs3: cached size %d != full calculation %d", cachedSize, fullSize) - } - }) - - t.Run("size limit enforcement with caching", func(t *testing.T) { - // Calculate size of first two observations - twoObs := []*pbtypes.Observation{obs1, obs2} - twoObsSize := calculateObservationsSize(twoObs) - - // Set limit to exactly fit two observations - limit := twoObsSize - - // Build incrementally - cachedSize := 0 - - // Add first observation - canAdd, newSize := checkObservationSizeLimit(cachedSize, obs1, limit) - if !canAdd { - t.Fatal("Should be able to add first observation within limit") - } - cachedSize = newSize - - // Add second observation - canAdd, newSize = checkObservationSizeLimit(cachedSize, obs2, limit) - if !canAdd { - t.Fatal("Should be able to add second observation within limit") - } - cachedSize = newSize - - // Try to add third observation - should fail - canAdd, unchangedSize := checkObservationSizeLimit(cachedSize, obs3, limit) - if canAdd { - t.Error("Should not be able to add third observation - would exceed limit") - } - if unchangedSize != cachedSize { - t.Errorf("Size should remain unchanged when limit exceeded: got %d, expected %d", unchangedSize, cachedSize) - } - }) - - t.Run("empty observation handling with caching", func(t *testing.T) { - emptyObs := &pbtypes.Observation{} - cachedSize := 0 - - // Add empty observation - should add 2 bytes (tag + length) - canAdd, newSize := checkObservationSizeLimit(cachedSize, emptyObs, 1000) - if !canAdd { - t.Error("Should be able to add empty observation") - } - expectedSize := cachedSize + 2 // tag + length overhead - if newSize != expectedSize { - t.Errorf("Empty observation should add 2 bytes: got %d, expected %d", newSize, expectedSize) - } - - // Add real observation first - canAdd, newSize = checkObservationSizeLimit(cachedSize, obs1, 1000) - if !canAdd { - t.Fatal("Should be able to add real observation") - } - cachedSize = newSize - - // Add empty observation after real observation - should add 2 bytes (tag + length) - canAdd, newSize = checkObservationSizeLimit(cachedSize, emptyObs, 1000) - if !canAdd { - t.Error("Should be able to add empty observation after real observation") - } - expectedSize = cachedSize + 2 // tag + length overhead - if newSize != expectedSize { - t.Errorf("Empty observation should add 2 bytes after real observation: got %d, expected %d", newSize, expectedSize) - } - }) -} - -func TestObservationsBatchHasCapacity(t *testing.T) { - // Helper function to create a simple observations message - createSimpleObservations := func(observationsList []*pbtypes.Observation, workflowIds []string) *pbtypes.Observations { - return &pbtypes.Observations{ - Observations: observationsList, - RegisteredWorkflowIds: workflowIds, - Timestamp: timestamppb.New(time.Unix(1640995200, 0)), // Fixed timestamp for consistent testing - } - } - - // Helper function to create a simple observation - createSimpleObservation := func(workflowExecutionId string) *pbtypes.Observation { - return &pbtypes.Observation{ - Id: &pbtypes.Id{ - WorkflowExecutionId: workflowExecutionId, - WorkflowId: "workflow-1", - WorkflowOwner: "owner", - WorkflowName: "test", - ReportId: "report-1", - KeyId: "key-1", - }, - OverriddenEncoderName: "encoder-1", - } - } - - // Helper function to create a large observation - createLargeObservation := func(suffix string) *pbtypes.Observation { - return &pbtypes.Observation{ - Id: &pbtypes.Id{ - WorkflowExecutionId: "very-long-workflow-execution-id-" + suffix, - WorkflowId: "very-long-workflow-id-" + suffix, - WorkflowOwner: "very-long-workflow-owner-" + suffix, - WorkflowName: "very-long-workflow-name-" + suffix, - ReportId: "very-long-report-id-" + suffix, - WorkflowDonId: 12345, - WorkflowDonConfigVersion: 67890, - KeyId: "very-long-key-id-" + suffix, - }, - OverriddenEncoderName: "very-long-encoder-name-" + suffix, - Observations: &pbvalues.List{ - Fields: []*pbvalues.Value{ - {Value: &pbvalues.Value_StringValue{StringValue: "observation-data-" + suffix}}, - {Value: &pbvalues.Value_Int64Value{Int64Value: 12345}}, - }, - }, - OverriddenEncoderConfig: &pbvalues.Map{ - Fields: map[string]*pbvalues.Value{ - "config-key-" + suffix: {Value: &pbvalues.Value_StringValue{StringValue: "config-value-" + suffix}}, - }, - }, - } - } - - // Helper function to create an empty observation - createEmptyObservation := func() *pbtypes.Observation { - return &pbtypes.Observation{} - } - - tests := []struct { - name string - existingObservations *pbtypes.Observations - newObservation *pbtypes.Observation - sizeLimit int - expected bool - description string - }{ - // Zero observation objects tests - { - name: "empty observations message, empty new observation, small limit", - existingObservations: createSimpleObservations([]*pbtypes.Observation{}, []string{}), - newObservation: createEmptyObservation(), - sizeLimit: 100, - expected: true, - description: "Adding empty observation to empty observations message should be within reasonable limit", - }, - { - name: "nil observations message, empty new observation, small limit", - existingObservations: nil, - newObservation: createEmptyObservation(), - sizeLimit: 10, - expected: true, - description: "Adding empty observation to nil observations should be within any limit", - }, - { - name: "empty observations message, simple observation, zero limit", - existingObservations: createSimpleObservations([]*pbtypes.Observation{}, []string{}), - newObservation: createSimpleObservation("exec-1"), - sizeLimit: 0, - expected: false, - description: "Non-empty observation should not fit in zero limit", - }, - - // Within limits tests - { - name: "empty observations message, simple observation, generous limit", - existingObservations: createSimpleObservations([]*pbtypes.Observation{}, []string{"workflow-1"}), - newObservation: createSimpleObservation("exec-1"), - sizeLimit: 1000, - expected: true, - description: "Simple observation should fit in generous limit", - }, - { - name: "observations with one existing observation, add another simple observation, generous limit", - existingObservations: createSimpleObservations([]*pbtypes.Observation{createSimpleObservation("exec-1")}, []string{"workflow-1", "workflow-2"}), - newObservation: createSimpleObservation("exec-2"), - sizeLimit: 1000, - expected: true, - description: "Two simple observations should fit in generous limit", - }, - - // Above limits tests - { - name: "empty observations message, simple observation, very small limit", - existingObservations: createSimpleObservations([]*pbtypes.Observation{}, []string{}), - newObservation: createSimpleObservation("exec-1"), - sizeLimit: 1, - expected: false, - description: "Simple observation should exceed very small limit", - }, - { - name: "observations with existing observation, add large observation, small limit", - existingObservations: createSimpleObservations([]*pbtypes.Observation{createSimpleObservation("exec-1")}, []string{"workflow-1"}), - newObservation: createLargeObservation("large"), - sizeLimit: 100, - expected: false, - description: "Large observation should exceed small limit when added to existing observations", - }, - - // Edge cases with complex observations messages - { - name: "large observation alone with many registered workflow IDs", - existingObservations: createSimpleObservations([]*pbtypes.Observation{}, []string{"workflow-1", "workflow-2", "workflow-3", "very-long-workflow-name-for-testing"}), - newObservation: createLargeObservation("huge"), - sizeLimit: 100, - expected: false, - description: "Large observation with many registered workflow IDs should exceed moderate limit", - }, - { - name: "mix of empty and non-empty existing observations in observations message", - existingObservations: createSimpleObservations([]*pbtypes.Observation{ - createEmptyObservation(), - createSimpleObservation("exec-1"), - createEmptyObservation(), - }, []string{"workflow-1"}), - newObservation: createSimpleObservation("exec-2"), - sizeLimit: 1000, - expected: true, - description: "Mix of empty and non-empty observations in observations message should work correctly", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - currentSize := CalculateObservationsMessageSize(tt.existingObservations) - result, _ := ObservationsBatchHasCapacity(currentSize, tt.newObservation, tt.sizeLimit) - if result != tt.expected { - // Provide detailed debugging information - currentSize := CalculateObservationsMessageSize(tt.existingObservations) - newObsSize := calculateObservationSize(tt.newObservation) - // Always add tag and length overhead, even for empty messages - totalSizeWithNewObs := currentSize + varintSize(uint64(1<<3|2)) + varintSize(uint64(newObsSize)) + newObsSize - - t.Errorf("%s: enoughObservations() = %v, expected %v\n"+ - " Description: %s\n"+ - " Current size: %d\n"+ - " New observation size: %d\n"+ - " Total size with new observation: %d\n"+ - " Size limit: %d\n"+ - " Would exceed: %v", - tt.name, result, tt.expected, - tt.description, - currentSize, newObsSize, totalSizeWithNewObs, tt.sizeLimit, - totalSizeWithNewObs > tt.sizeLimit) - } - }) - } -} - -func TestObservationsBatchHasCapacityWithRealSizes(t *testing.T) { - // Test with realistic size calculations - simpleObs := &pbtypes.Observation{ - Id: &pbtypes.Id{ - WorkflowExecutionId: "exec-123", - WorkflowId: "workflow-1", - ReportId: "report-1", - }, - OverriddenEncoderName: "encoder-1", - } - - observationsMsg := &pbtypes.Observations{ - Observations: []*pbtypes.Observation{}, - RegisteredWorkflowIds: []string{"workflow-1", "workflow-2"}, - Timestamp: timestamppb.New(time.Unix(1640995200, 0)), - } - - t.Run("verify observations message size calculations", func(t *testing.T) { - // Test empty observations message - size := CalculateObservationsMessageSize(observationsMsg) - if size <= 0 { - t.Errorf("Observations message with workflow IDs and timestamp should have positive size, got %d", size) - } - - t.Logf("Empty observations message size: %d bytes", size) - - // Test adding observation - currentSize := CalculateObservationsMessageSize(observationsMsg) - result, _ := ObservationsBatchHasCapacity(currentSize, simpleObs, size+100) - if !result { - t.Errorf("Should be able to add observation when limit has buffer") - } - - // Calculate actual size with observation - observationsWithObs := &pbtypes.Observations{ - Observations: []*pbtypes.Observation{simpleObs}, - RegisteredWorkflowIds: []string{"workflow-1", "workflow-2"}, - Timestamp: timestamppb.New(time.Unix(1640995200, 0)), - } - sizeWithObs := CalculateObservationsMessageSize(observationsWithObs) - - t.Logf("Observations message with one observation size: %d bytes", sizeWithObs) - - currentSize = CalculateObservationsMessageSize(observationsMsg) - result, _ = ObservationsBatchHasCapacity(currentSize, simpleObs, sizeWithObs-1) - if result { - t.Errorf("Should not be able to add observation when limit is one byte less than required") - } - }) -} - -func TestQuerySizeCalculationMatchesRealMarshalling(t *testing.T) { - // Helper function to create a simple ID (reused from existing tests) - createSimpleId := func(workflowExecutionId string) *pbtypes.Id { - return &pbtypes.Id{ - WorkflowExecutionId: workflowExecutionId, - WorkflowId: "workflow-1", - WorkflowOwner: "owner", - WorkflowName: "test", - ReportId: "report-1", - KeyId: "key-1", - } - } - - // Create test data using existing helper - ids := []*pbtypes.Id{ - createSimpleId("exec-1"), - createSimpleId("exec-2"), - createSimpleId("exec-3"), - } - - // Calculate size using our function - calculatedSize := calculateQuerySize(ids) - - // Create actual Query message and marshal it - query := &pbtypes.Query{Ids: ids} - marshalled, err := proto.MarshalOptions{Deterministic: true}.Marshal(query) - if err != nil { - t.Fatalf("Failed to marshal query: %v", err) - } - actualSize := len(marshalled) - - // Verify they match - if calculatedSize != actualSize { - t.Errorf("Query size calculation mismatch: calculated=%d, actual=%d", calculatedSize, actualSize) - } - - t.Logf("Query size calculation matches: %d bytes", actualSize) -} - -func TestObservationsSizeCalculationMatchesRealMarshalling(t *testing.T) { - // Helper function to create a simple observation (reused from existing tests) - createSimpleObservation := func(workflowExecutionId string) *pbtypes.Observation { - return &pbtypes.Observation{ - Id: &pbtypes.Id{ - WorkflowExecutionId: workflowExecutionId, - WorkflowId: "workflow-1", - WorkflowOwner: "owner", - WorkflowName: "test", - ReportId: "report-1", - KeyId: "key-1", - }, - OverriddenEncoderName: "encoder-1", - } - } - - // Create test data using existing helpers - observations := &pbtypes.Observations{ - Observations: []*pbtypes.Observation{ - createSimpleObservation("exec-1"), - createSimpleObservation("exec-2"), - }, - RegisteredWorkflowIds: []string{"workflow-1", "workflow-2"}, - Timestamp: timestamppb.New(time.Unix(1640995200, 0)), - } - - // Calculate size using our function - calculatedSize := CalculateObservationsMessageSize(observations) - - // Marshal the actual message - marshalled, err := proto.MarshalOptions{Deterministic: true}.Marshal(observations) - if err != nil { - t.Fatalf("Failed to marshal observations: %v", err) - } - actualSize := len(marshalled) - - // Verify they match - if calculatedSize != actualSize { - t.Errorf("Observations size calculation mismatch: calculated=%d, actual=%d", calculatedSize, actualSize) - } - - t.Logf("Observations size calculation matches: %d bytes", actualSize) -} - -func TestReportSizeCalculationMatchesRealMarshalling(t *testing.T) { - // Helper function to create a simple report with realistic data - createSimpleReport := func(workflowExecutionId string) *pbtypes.Report { - return &pbtypes.Report{ - Id: &pbtypes.Id{ - WorkflowExecutionId: workflowExecutionId, - WorkflowId: "workflow-1", - WorkflowOwner: "owner", - WorkflowName: "test", - ReportId: "report-1", - KeyId: "key-1", - }, - Outcome: &pbtypes.AggregationOutcome{ - EncodableOutcome: &pbvalues.Map{ - Fields: map[string]*pbvalues.Value{ - "result": { - Value: &pbvalues.Value_StringValue{StringValue: "success"}, - }, - }, - }, - Metadata: []byte("test-metadata"), - ShouldReport: true, - LastSeenAt: 12345, - Timestamp: timestamppb.New(time.Unix(1640995200, 0)), - EncoderName: "test-encoder", - }, - } - } - - // Create test report - report := createSimpleReport("exec-1") - - // Calculate size using our function - calculatedSize := calculateReportSize(report) - - // Marshal the actual message - marshalled, err := proto.MarshalOptions{Deterministic: true}.Marshal(report) - if err != nil { - t.Fatalf("Failed to marshal report: %v", err) - } - actualSize := len(marshalled) - - // Verify they match - if calculatedSize != actualSize { - t.Errorf("Report size calculation mismatch: calculated=%d, actual=%d", calculatedSize, actualSize) - } - - t.Logf("Report size calculation matches: %d bytes", actualSize) -} - -func TestReportBatchHasCapacity(t *testing.T) { - // Helper function to create a simple report with predictable size - createSimpleReport := func(workflowExecutionId string) *pbtypes.Report { - return &pbtypes.Report{ - Id: &pbtypes.Id{ - WorkflowExecutionId: workflowExecutionId, - WorkflowId: "workflow-1", - WorkflowOwner: "owner", - WorkflowName: "test", - ReportId: "report-1", - KeyId: "key-1", - }, - Outcome: &pbtypes.AggregationOutcome{ - EncodableOutcome: &pbvalues.Map{ - Fields: map[string]*pbvalues.Value{ - "result": { - Value: &pbvalues.Value_StringValue{StringValue: "success"}, - }, - }, - }, - Metadata: []byte("metadata"), - ShouldReport: true, - LastSeenAt: 12345, - EncoderName: "encoder", - }, - } - } - - // Helper function to create a report with all fields populated for larger size - createLargeReport := func(suffix string) *pbtypes.Report { - return &pbtypes.Report{ - Id: &pbtypes.Id{ - WorkflowExecutionId: "very-long-workflow-execution-id-" + suffix, - WorkflowId: "very-long-workflow-id-" + suffix, - WorkflowOwner: "very-long-workflow-owner-" + suffix, - WorkflowName: "very-long-workflow-name-" + suffix, - ReportId: "very-long-report-id-" + suffix, - WorkflowDonId: 12345, - WorkflowDonConfigVersion: 67890, - KeyId: "very-long-key-id-" + suffix, - }, - Outcome: &pbtypes.AggregationOutcome{ - EncodableOutcome: &pbvalues.Map{ - Fields: map[string]*pbvalues.Value{ - "very-long-result-key-" + suffix: { - Value: &pbvalues.Value_StringValue{StringValue: "very-long-result-value-" + suffix}, - }, - "another-long-key-" + suffix: { - Value: &pbvalues.Value_StringValue{StringValue: "another-long-value-" + suffix}, - }, - }, - }, - Metadata: []byte("very-long-metadata-content-for-testing-" + suffix), - ShouldReport: true, - LastSeenAt: 123456789, - Timestamp: timestamppb.New(time.Unix(1640995200, 0)), - EncoderName: "very-long-encoder-name-" + suffix, - EncoderConfig: &pbvalues.Map{ - Fields: map[string]*pbvalues.Value{ - "config-key-" + suffix: { - Value: &pbvalues.Value_StringValue{StringValue: "config-value-" + suffix}, - }, - }, - }, - }, - } - } - - // Helper function to create an empty report (zero values) - createEmptyReport := func() *pbtypes.Report { - return &pbtypes.Report{} - } - - tests := []struct { - name string - existingReports []*pbtypes.Report - newReport *pbtypes.Report - sizeLimit int - expected bool - description string - }{ - // Zero report objects tests - { - name: "empty list, empty new report, small limit", - existingReports: []*pbtypes.Report{}, - newReport: createEmptyReport(), - sizeLimit: 10, - expected: true, // Empty report requires 2 bytes (tag + length), but 10-byte limit is sufficient - description: "Adding empty report to empty list should be within reasonable limit", - }, - { - name: "empty list, empty new report, zero limit", - existingReports: []*pbtypes.Report{}, - newReport: createEmptyReport(), - sizeLimit: 0, - expected: false, // Empty report requires 2 bytes (tag + length) - description: "Empty report should not fit in zero limit (requires tag + length overhead)", - }, - { - name: "empty list, simple report, zero limit", - existingReports: []*pbtypes.Report{}, - newReport: createSimpleReport("exec-1"), - sizeLimit: 0, - expected: false, // Simple report has size > 0, exceeds zero limit - description: "Non-empty report should not fit in zero limit", - }, - - // Within limits tests - { - name: "empty list, simple report, generous limit", - existingReports: []*pbtypes.Report{}, - newReport: createSimpleReport("exec-1"), - sizeLimit: 2000, - expected: true, - description: "Simple report should fit in generous limit", - }, - { - name: "one existing report, add another simple report, generous limit", - existingReports: []*pbtypes.Report{createSimpleReport("exec-1")}, - newReport: createSimpleReport("exec-2"), - sizeLimit: 2000, - expected: true, - description: "Two simple reports should fit in generous limit", - }, - { - name: "three existing reports, add fourth, generous limit", - existingReports: []*pbtypes.Report{ - createSimpleReport("exec-1"), - createSimpleReport("exec-2"), - createSimpleReport("exec-3"), - }, - newReport: createSimpleReport("exec-4"), - sizeLimit: 2000, - expected: true, - description: "Four simple reports should fit in generous limit", - }, - - // Above limits tests - { - name: "empty list, simple report, very small limit", - existingReports: []*pbtypes.Report{}, - newReport: createSimpleReport("exec-1"), - sizeLimit: 1, - expected: false, - description: "Simple report should exceed very small limit", - }, - { - name: "one existing report, add large report, small limit", - existingReports: []*pbtypes.Report{createSimpleReport("exec-1")}, - newReport: createLargeReport("large"), - sizeLimit: 200, - expected: false, - description: "Large report should exceed small limit when added to existing", - }, - { - name: "multiple existing reports, add another, tight limit", - existingReports: []*pbtypes.Report{ - createSimpleReport("exec-1"), - createSimpleReport("exec-2"), - createSimpleReport("exec-3"), - }, - newReport: createSimpleReport("exec-4"), - sizeLimit: 400, // Adjust based on actual size calculations - expected: false, - description: "Multiple reports should exceed tight limit", - }, - - // Edge cases - { - name: "exactly at limit boundary", - existingReports: []*pbtypes.Report{}, - newReport: createSimpleReport("exec-1"), - sizeLimit: 0, // Will be set to exact size in the test - expected: true, - description: "Report exactly at limit should fit", - }, - { - name: "one byte over limit", - existingReports: []*pbtypes.Report{}, - newReport: createSimpleReport("exec-1"), - sizeLimit: 0, // Will be set to exact size - 1 in the test - expected: false, - description: "Report one byte over limit should not fit", - }, - { - name: "large report alone", - existingReports: []*pbtypes.Report{}, - newReport: createLargeReport("huge"), - sizeLimit: 100, - expected: false, - description: "Large report should exceed moderate limit", - }, - { - name: "mix of empty and non-empty existing reports", - existingReports: []*pbtypes.Report{ - createEmptyReport(), - createSimpleReport("exec-1"), - createEmptyReport(), - }, - newReport: createSimpleReport("exec-2"), - sizeLimit: 2000, - expected: true, - description: "Mix of empty and non-empty reports should work correctly", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Handle special edge case tests that need dynamic size calculation - sizeLimit := tt.sizeLimit - switch tt.name { - case "exactly at limit boundary": - // Calculate exact size needed for the new report - newReportSize := calculateReportSize(tt.newReport) - // Always include tag and length overhead, even for empty messages - tagSize := varintSize(uint64(2<<3 | 2)) - lengthSize := varintSize(uint64(newReportSize)) - sizeLimit = tagSize + lengthSize + newReportSize - case "one byte over limit": - // Calculate exact size needed for the new report minus 1 - newReportSize := calculateReportSize(tt.newReport) - // Always include tag and length overhead, even for empty messages - tagSize := varintSize(uint64(2<<3 | 2)) - lengthSize := varintSize(uint64(newReportSize)) - sizeLimit = tagSize + lengthSize + newReportSize - 1 - } - - currentSize := calculateReportsSize(tt.existingReports) - result, _ := ReportBatchHasCapacity(currentSize, tt.newReport, sizeLimit) - if result != tt.expected { - // Provide detailed debugging information - currentSize := calculateReportsSize(tt.existingReports) - newReportSize := calculateReportSize(tt.newReport) - // Always add tag and length overhead, even for empty messages - totalSizeWithNewReport := currentSize + varintSize(uint64(2<<3|2)) + varintSize(uint64(newReportSize)) + newReportSize - - t.Errorf("%s: CheckReportSizeLimit() = %v, expected %v\n"+ - " Description: %s\n"+ - " Current size: %d\n"+ - " New report size: %d\n"+ - " Total size with new report: %d\n"+ - " Size limit: %d\n"+ - " Would exceed: %v", - tt.name, result, tt.expected, - tt.description, - currentSize, newReportSize, totalSizeWithNewReport, sizeLimit, - totalSizeWithNewReport > sizeLimit) - } - }) - } -} - -func TestReportBatchHasCapacityWithRealSizes(t *testing.T) { - // Test with realistic size calculations to verify our understanding - simpleReport := &pbtypes.Report{ - Id: &pbtypes.Id{ - WorkflowExecutionId: "exec-123", - WorkflowId: "workflow-1", - WorkflowOwner: "owner", - WorkflowName: "test", - ReportId: "report-1", - KeyId: "key-1", - }, - Outcome: &pbtypes.AggregationOutcome{ - EncodableOutcome: &pbvalues.Map{ - Fields: map[string]*pbvalues.Value{ - "result": { - Value: &pbvalues.Value_StringValue{StringValue: "success"}, - }, - }, - }, - Metadata: []byte("metadata"), - ShouldReport: true, - LastSeenAt: 12345, - EncoderName: "encoder", - }, - } - - // Helper function to create an outcome with reports - createOutcomeWithReports := func(reports []*pbtypes.Report) *pbtypes.Outcome { - return &pbtypes.Outcome{ - Outcomes: map[string]*pbtypes.AggregationOutcome{}, - CurrentReports: reports, - } - } - - t.Run("verify size calculations", func(t *testing.T) { - // Test empty list - emptyOutcome := createOutcomeWithReports([]*pbtypes.Report{}) - size := calculateReportsSize(emptyOutcome.CurrentReports) - if size != 0 { - t.Errorf("Empty list should have size 0, got %d", size) - } - - // Test single report - singleReportOutcome := createOutcomeWithReports([]*pbtypes.Report{simpleReport}) - singleReportSize := calculateReportsSize(singleReportOutcome.CurrentReports) - if singleReportSize <= 0 { - t.Errorf("Single report should have positive size, got %d", singleReportSize) - } - - t.Logf("Single report size: %d bytes", singleReportSize) - - // Test that size limit function works correctly with these sizes - result, _ := ReportBatchHasCapacity(0, simpleReport, singleReportSize) - if !result { - t.Errorf("Should be able to add report when limit equals exact size") - } - - result, _ = ReportBatchHasCapacity(0, simpleReport, singleReportSize-1) - if result { - t.Errorf("Should not be able to add report when limit is one byte less than size") - } - }) -} - -func TestReportBatchHasCapacityCaching(t *testing.T) { - // Test that the caching mechanism works correctly for reports - report1 := &pbtypes.Report{ - Id: &pbtypes.Id{WorkflowExecutionId: "exec-1", WorkflowId: "wf-1"}, - Outcome: &pbtypes.AggregationOutcome{ - EncodableOutcome: &pbvalues.Map{ - Fields: map[string]*pbvalues.Value{ - "result": {Value: &pbvalues.Value_StringValue{StringValue: "result1"}}, - }, - }, - }, - } - report2 := &pbtypes.Report{ - Id: &pbtypes.Id{WorkflowExecutionId: "exec-2", WorkflowId: "wf-2"}, - Outcome: &pbtypes.AggregationOutcome{ - EncodableOutcome: &pbvalues.Map{ - Fields: map[string]*pbvalues.Value{ - "result": {Value: &pbvalues.Value_StringValue{StringValue: "result2"}}, - }, - }, - }, - } - report3 := &pbtypes.Report{ - Id: &pbtypes.Id{WorkflowExecutionId: "exec-3", WorkflowId: "wf-3"}, - Outcome: &pbtypes.AggregationOutcome{ - EncodableOutcome: &pbvalues.Map{ - Fields: map[string]*pbvalues.Value{ - "result": {Value: &pbvalues.Value_StringValue{StringValue: "result3"}}, - }, - }, - }, - } - - t.Run("incremental size calculation matches full recalculation", func(t *testing.T) { - // Build up incrementally using caching - cachedSize := 0 - reports := []*pbtypes.Report{} - - // Add first report - canAdd, newSize := ReportBatchHasCapacity(cachedSize, report1, 10000) - if !canAdd { - t.Fatal("Should be able to add first report") - } - reports = append(reports, report1) - cachedSize = newSize - - // Verify cached size matches full calculation - fullSize := calculateReportsSize(reports) - if cachedSize != fullSize { - t.Errorf("After adding report1: cached size %d != full calculation %d", cachedSize, fullSize) - } - - // Add second report - canAdd, newSize = ReportBatchHasCapacity(cachedSize, report2, 10000) - if !canAdd { - t.Fatal("Should be able to add second report") - } - reports = append(reports, report2) - cachedSize = newSize - - // Verify cached size matches full calculation - fullSize = calculateReportsSize(reports) - if cachedSize != fullSize { - t.Errorf("After adding report2: cached size %d != full calculation %d", cachedSize, fullSize) - } - - // Add third report - canAdd, newSize = ReportBatchHasCapacity(cachedSize, report3, 10000) - if !canAdd { - t.Fatal("Should be able to add third report") - } - reports = append(reports, report3) - cachedSize = newSize - - // Verify final cached size matches full calculation - fullSize = calculateReportsSize(reports) - if cachedSize != fullSize { - t.Errorf("After adding report3: cached size %d != full calculation %d", cachedSize, fullSize) - } - }) - - t.Run("size limit enforcement with caching", func(t *testing.T) { - // Calculate size of first two reports - twoReports := []*pbtypes.Report{report1, report2} - twoReportsSize := calculateReportsSize(twoReports) - - // Set limit to exactly fit two reports - limit := twoReportsSize - - // Build incrementally - cachedSize := 0 - - // Add first report - canAdd, newSize := ReportBatchHasCapacity(cachedSize, report1, limit) - if !canAdd { - t.Fatal("Should be able to add first report within limit") - } - cachedSize = newSize - - // Add second report - canAdd, newSize = ReportBatchHasCapacity(cachedSize, report2, limit) - if !canAdd { - t.Fatal("Should be able to add second report within limit") - } - cachedSize = newSize - - // Try to add third report - should fail - canAdd, unchangedSize := ReportBatchHasCapacity(cachedSize, report3, limit) - if canAdd { - t.Error("Should not be able to add third report - would exceed limit") - } - if unchangedSize != cachedSize { - t.Errorf("Size should remain unchanged when limit exceeded: got %d, expected %d", unchangedSize, cachedSize) - } - }) - - t.Run("nil report handling with caching", func(t *testing.T) { - cachedSize := 100 // Some initial size - - // Add nil report - should not change size and should return true - canAdd, newSize := ReportBatchHasCapacity(cachedSize, nil, 1000) - if !canAdd { - t.Error("Should be able to add nil report") - } - if newSize != cachedSize { - t.Errorf("Nil report should not change size: got %d, expected %d", newSize, cachedSize) - } - }) - - t.Run("empty report handling with caching", func(t *testing.T) { - emptyReport := &pbtypes.Report{} - cachedSize := 0 - - // Add empty report - should add 2 bytes (tag + length) - canAdd, newSize := ReportBatchHasCapacity(cachedSize, emptyReport, 1000) - if !canAdd { - t.Error("Should be able to add empty report") - } - expectedSize := cachedSize + 2 // tag + length overhead - if newSize != expectedSize { - t.Errorf("Empty report should add 2 bytes: got %d, expected %d", newSize, expectedSize) - } - - // Add real report first - canAdd, newSize = ReportBatchHasCapacity(cachedSize, report1, 1000) - if !canAdd { - t.Fatal("Should be able to add real report") - } - cachedSize = newSize - - // Add empty report after real report - should add 2 bytes (tag + length) - canAdd, newSize = ReportBatchHasCapacity(cachedSize, emptyReport, 1000) - if !canAdd { - t.Error("Should be able to add empty report after real report") - } - expectedSize = cachedSize + 2 // tag + length overhead - if newSize != expectedSize { - t.Errorf("Empty report should add 2 bytes after real report: got %d, expected %d", newSize, expectedSize) - } - }) -} - -func TestSizeCalculationAccuracy(t *testing.T) { - t.Run("verify size calculations against real marshaling", func(t *testing.T) { - // Test 1: Empty messages - t.Run("empty messages", func(t *testing.T) { - // Empty ID - emptyId := &pbtypes.Id{} - calculatedSize := calculateIdSize(emptyId) - actualSize := proto.Size(emptyId) - if calculatedSize != actualSize { - t.Errorf("Empty ID size mismatch: calculated=%d, actual=%d", calculatedSize, actualSize) - } - - // Empty Observation - emptyObs := &pbtypes.Observation{} - calculatedSize = calculateObservationSize(emptyObs) - actualSize = proto.Size(emptyObs) - if calculatedSize != actualSize { - t.Errorf("Empty Observation size mismatch: calculated=%d, actual=%d", calculatedSize, actualSize) - } - - // Empty Report - emptyReport := &pbtypes.Report{} - calculatedSize = calculateReportSize(emptyReport) - actualSize = proto.Size(emptyReport) - if calculatedSize != actualSize { - t.Errorf("Empty Report size mismatch: calculated=%d, actual=%d", calculatedSize, actualSize) - } - }) - - // Test 2: Simple messages with basic data - t.Run("simple messages", func(t *testing.T) { - // Simple ID - simpleId := &pbtypes.Id{ - WorkflowExecutionId: "exec-123", - WorkflowId: "workflow-456", - WorkflowOwner: "owner-789", - WorkflowName: "test-workflow", - WorkflowDonId: uint32(123), - WorkflowDonConfigVersion: uint32(456), - ReportId: "report-789", - KeyId: "key-012", - } - calculatedSize := calculateIdSize(simpleId) - actualSize := proto.Size(simpleId) - if calculatedSize != actualSize { - t.Errorf("Simple ID size mismatch: calculated=%d, actual=%d", calculatedSize, actualSize) - } - - // Simple Observation with List - simpleObs := &pbtypes.Observation{ - Id: simpleId, - Observations: &pbvalues.List{ - Fields: []*pbvalues.Value{ - {Value: &pbvalues.Value_StringValue{StringValue: "test-observation-1"}}, - {Value: &pbvalues.Value_Int64Value{Int64Value: 12345}}, - }, - }, - } - calculatedSize = calculateObservationSize(simpleObs) - actualSize = proto.Size(simpleObs) - if calculatedSize != actualSize { - t.Errorf("Simple Observation size mismatch: calculated=%d, actual=%d", calculatedSize, actualSize) - } - - // Simple Report - simpleReport := &pbtypes.Report{ - Id: simpleId, - Outcome: &pbtypes.AggregationOutcome{ - EncodableOutcome: &pbvalues.Map{ - Fields: map[string]*pbvalues.Value{ - "result": {Value: &pbvalues.Value_StringValue{StringValue: "test-result"}}, - }, - }, - }, - } - calculatedSize = calculateReportSize(simpleReport) - actualSize = proto.Size(simpleReport) - if calculatedSize != actualSize { - t.Errorf("Simple Report size mismatch: calculated=%d, actual=%d", calculatedSize, actualSize) - } - }) - - // Test 3: Complex messages with various field types - t.Run("complex messages", func(t *testing.T) { - // Complex ID with all fields populated - complexId := &pbtypes.Id{ - WorkflowExecutionId: "very-long-workflow-execution-id-for-testing-purposes", - WorkflowId: "complex-workflow-id-with-special-chars-@#$%", - WorkflowOwner: "owner-with-long-name-and-special-characters", - WorkflowName: "test-workflow-with-very-descriptive-name", - WorkflowDonId: uint32(999999), - WorkflowDonConfigVersion: uint32(888888), - ReportId: "report-id-with-uuid-like-structure-12345678", - KeyId: "key-id-with-cryptographic-hash-abcdef", - } - calculatedSize := calculateIdSize(complexId) - actualSize := proto.Size(complexId) - if calculatedSize != actualSize { - t.Errorf("Complex ID size mismatch: calculated=%d, actual=%d", calculatedSize, actualSize) - } - - // Complex Observation with nested structures - complexObs := &pbtypes.Observation{ - Id: complexId, - Observations: &pbvalues.List{ - Fields: []*pbvalues.Value{ - {Value: &pbvalues.Value_StringValue{StringValue: "complex-string-observation-with-lots-of-data"}}, - {Value: &pbvalues.Value_Int64Value{Int64Value: 9223372036854775807}}, // max int64 - {Value: &pbvalues.Value_Float64Value{Float64Value: 3.141592653589793}}, - {Value: &pbvalues.Value_BoolValue{BoolValue: true}}, - }, - }, - } - calculatedSize = calculateObservationSize(complexObs) - actualSize = proto.Size(complexObs) - if calculatedSize != actualSize { - t.Errorf("Complex Observation size mismatch: calculated=%d, actual=%d", calculatedSize, actualSize) - } - - // Complex Report with nested result - complexReport := &pbtypes.Report{ - Id: complexId, - Outcome: &pbtypes.AggregationOutcome{ - EncodableOutcome: &pbvalues.Map{ - Fields: map[string]*pbvalues.Value{ - "nested": { - Value: &pbvalues.Value_ListValue{ - ListValue: &pbvalues.List{ - Fields: []*pbvalues.Value{ - {Value: &pbvalues.Value_StringValue{StringValue: "nested-result-1"}}, - {Value: &pbvalues.Value_StringValue{StringValue: "nested-result-2"}}, - }, - }, - }, - }, - }, - }, - Metadata: []byte("complex-metadata"), - ShouldReport: true, - }, - } - calculatedSize = calculateReportSize(complexReport) - actualSize = proto.Size(complexReport) - if calculatedSize != actualSize { - t.Errorf("Complex Report size mismatch: calculated=%d, actual=%d", calculatedSize, actualSize) - } - }) - - // Test 4: Container message size calculations - t.Run("container messages", func(t *testing.T) { - // Test Query with mixed IDs (including empty) - emptyId := &pbtypes.Id{} - simpleId := &pbtypes.Id{ - WorkflowExecutionId: "exec-1", - WorkflowId: "workflow-1", - WorkflowOwner: "owner", - WorkflowName: "test", - ReportId: "report-1", - KeyId: "key-1", - } - - query := &pbtypes.Query{ - Ids: []*pbtypes.Id{emptyId, simpleId, emptyId}, - } - calculatedSize := calculateQuerySize(query.Ids) - actualSize := proto.Size(query) - if calculatedSize != actualSize { - t.Errorf("Mixed IDs Query size mismatch: calculated=%d, actual=%d", calculatedSize, actualSize) - } - - // Test Observations with mixed observations - emptyObs := &pbtypes.Observation{} - simpleObs := &pbtypes.Observation{ - Id: simpleId, - Observations: &pbvalues.List{ - Fields: []*pbvalues.Value{ - {Value: &pbvalues.Value_StringValue{StringValue: "test-observation"}}, - }, - }, - } - - observations := &pbtypes.Observations{ - Observations: []*pbtypes.Observation{emptyObs, simpleObs}, - } - calculatedSize = CalculateObservationsMessageSize(observations) - actualSize = proto.Size(observations) - if calculatedSize != actualSize { - t.Errorf("Mixed Observations size mismatch: calculated=%d, actual=%d", calculatedSize, actualSize) - } - - // Test Reports with mixed reports - emptyReport := &pbtypes.Report{} - simpleReport := &pbtypes.Report{ - Id: simpleId, - Outcome: &pbtypes.AggregationOutcome{ - EncodableOutcome: &pbvalues.Map{ - Fields: map[string]*pbvalues.Value{ - "result": {Value: &pbvalues.Value_StringValue{StringValue: "success"}}, - }, - }, - }, - } - - reports := []*pbtypes.Report{emptyReport, simpleReport} - calculatedSize = calculateReportsSize(reports) - - // For reports, we need to calculate the size as part of an Outcome message - outcome := &pbtypes.Outcome{ - CurrentReports: reports, - } - actualSize = proto.Size(outcome) - if calculatedSize != actualSize { - t.Errorf("Mixed Reports size mismatch: calculated=%d, actual=%d", calculatedSize, actualSize) - } - }) - - // Test 5: Edge cases - t.Run("edge cases", func(t *testing.T) { - // ID with zero numeric values (should be omitted in proto3) - zeroId := &pbtypes.Id{ - WorkflowExecutionId: "exec", - WorkflowId: "workflow", - WorkflowOwner: "owner", - WorkflowName: "name", - WorkflowDonId: 0, // zero value - WorkflowDonConfigVersion: 0, // zero value - ReportId: "report", - KeyId: "key", - } - calculatedSize := calculateIdSize(zeroId) - actualSize := proto.Size(zeroId) - if calculatedSize != actualSize { - t.Errorf("Zero values ID size mismatch: calculated=%d, actual=%d", calculatedSize, actualSize) - } - - // Observation with empty list - emptyListObs := &pbtypes.Observation{ - Id: &pbtypes.Id{ - WorkflowExecutionId: "exec-1", - WorkflowId: "workflow-1", - WorkflowOwner: "owner", - WorkflowName: "test", - ReportId: "report-1", - KeyId: "key-1", - }, - Observations: &pbvalues.List{Fields: []*pbvalues.Value{}}, // empty list - } - calculatedSize = calculateObservationSize(emptyListObs) - actualSize = proto.Size(emptyListObs) - if calculatedSize != actualSize { - t.Errorf("Empty list Observation size mismatch: calculated=%d, actual=%d", calculatedSize, actualSize) - } - }) - }) -} diff --git a/pkg/capabilities/consensus/ocr3/benchmark_test.go b/pkg/capabilities/consensus/ocr3/benchmark_test.go deleted file mode 100644 index 767b9c0498..0000000000 --- a/pkg/capabilities/consensus/ocr3/benchmark_test.go +++ /dev/null @@ -1,494 +0,0 @@ -package ocr3_test - -import ( - "context" - "fmt" - "runtime" - "strconv" - "testing" - "time" - - "github.com/shopspring/decimal" - "github.com/stretchr/testify/require" - "go.uber.org/zap/zapcore" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/timestamppb" - - ocrcommon "github.com/smartcontractkit/libocr/commontypes" - "github.com/smartcontractkit/libocr/offchainreporting2/types" - "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types" - - "github.com/smartcontractkit/chainlink-protos/cre/go/values" - - "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/ocr3" - "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/ocr3/datafeeds" - pbtypes "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/ocr3/types" - "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/requests" - "github.com/smartcontractkit/chainlink-common/pkg/capabilities/datastreams" - "github.com/smartcontractkit/chainlink-common/pkg/logger" -) - -// mockCapability implements CapabilityIface for testing -type mockCapability struct { - aggregators map[string]pbtypes.Aggregator -} - -func (m *mockCapability) GetAggregator(workflowID string) (pbtypes.Aggregator, error) { - return m.aggregators[workflowID], nil -} - -func (m *mockCapability) GetEncoderByWorkflowID(workflowID string) (pbtypes.Encoder, error) { - return nil, nil // Not used in benchmark -} - -func (m *mockCapability) GetEncoderByName(encoderName string, config *values.Map) (pbtypes.Encoder, error) { - return nil, nil // Not used in benchmark -} - -func (m *mockCapability) GetRegisteredWorkflowsIDs() []string { - ids := make([]string, 0, len(m.aggregators)) - for id := range m.aggregators { - ids = append(ids, id) - } - return ids -} - -func (m *mockCapability) UnregisterWorkflowID(workflowID string) { - delete(m.aggregators, workflowID) -} - -func BenchmarkReportingPlugin_Outcome_LLOAggregator(b *testing.B) { - // Define test matrix parameters - workflowCounts := []int{1, 2, 4, 8, 16, 32, 64, 128} - streamCounts := []int{32, 64, 128, 256, 512, 1024} - - // Create one logger for all benchmarks to reduce setup overhead - c := logger.Config{ - Level: zapcore.InfoLevel, // Set to InfoLevel for benchmarks to reduce log noise - } - lggr, err := c.New() - require.NoError(b, err, "failed to create logger for benchmark") - - // Run benchmarks for each combination - for _, numWorkflows := range workflowCounts { - for _, numStreamsPerWorkflow := range streamCounts { - benchName := fmt.Sprintf("workflows=%d/streams=%d", numWorkflows, numStreamsPerWorkflow) - b.Run(benchName, func(b *testing.B) { - runBenchmarkWithParams(b, lggr, numWorkflows, numStreamsPerWorkflow) - }) - } - } -} - -func BenchmarkReportingPlugin_Observation_LLOAggregator(b *testing.B) { - // Define test matrix parameters - workflowCounts := []int{1, 2, 4, 8, 16, 32, 64, 128} - streamCounts := []int{32, 64, 128, 256, 512, 1024} - - // Create one logger for all benchmarks to reduce setup overhead - c := logger.Config{ - Level: zapcore.InfoLevel, // Set to InfoLevel for benchmarks to reduce log noise - } - lggr, err := c.New() - require.NoError(b, err, "failed to create logger for benchmark") - - // Run benchmarks for each combination - for _, numWorkflows := range workflowCounts { - for _, numStreamsPerWorkflow := range streamCounts { - benchName := fmt.Sprintf("workflows=%d/streams=%d", numWorkflows, numStreamsPerWorkflow) - b.Run(benchName, func(b *testing.B) { - runObservationBenchmarkWithParams(b, lggr, numWorkflows, numStreamsPerWorkflow) - }) - } - } -} - -// runObservationBenchmarkWithParams runs a benchmark with the specified parameters -func runObservationBenchmarkWithParams(b *testing.B, lggr logger.Logger, numWorkflows, numStreamsPerWorkflow int) { - const ( - numOracles = 4 // Total nodes - f = 1 // Fault tolerance - ) - - // Create request store with requests for each workflow - store := requests.NewStore[*ocr3.ReportRequest]() - - // Create capability with LLO aggregators for each workflow - mockCap := &mockCapability{ - aggregators: make(map[string]pbtypes.Aggregator, numWorkflows), - } - - // Create LLO aggregators for each workflow and populate the store - for i := range numWorkflows { - workflowID := fmt.Sprintf("workflow-%d", i) - executionID := fmt.Sprintf("execution-%d", i) - - // Create aggregator - agg, err := createLLOAggregator(b, numStreamsPerWorkflow) - require.NoError(b, err) - mockCap.aggregators[workflowID] = agg - - // Populate store with observation data - lloEvent := createLLOEvent(b, numStreamsPerWorkflow, time.Now()) - wrappedEvent, err := values.Wrap(lloEvent) - require.NoError(b, err) - - // Create list with the LLO event - listVal, err := values.NewList([]any{wrappedEvent}) - require.NoError(b, err) - - // Create and add request to store - req := &ocr3.ReportRequest{ - WorkflowID: workflowID, - WorkflowExecutionID: executionID, - WorkflowName: fmt.Sprintf("Workflow %d", i), - WorkflowOwner: "test-owner", - WorkflowDonID: 1, - WorkflowDonConfigVersion: 1, - ReportID: fmt.Sprintf("report-%d", i), - KeyID: "test-key", - Observations: listVal, - } - - require.NoError(b, store.Add(req)) - } - - // Create reporting plugin - plugin, err := ocr3.NewReportingPlugin( - store, - mockCap, - numWorkflows, // batchSize matches numWorkflows - ocr3types.ReportingPluginConfig{ - N: numOracles, - F: f, - }, - &pbtypes.ReportingPluginConfig{ - OutcomePruningThreshold: 100, - }, - lggr, - ) - require.NoError(b, err) - - // Create test query with workflow IDs - query, err := createTestQuery(numWorkflows) - require.NoError(b, err) - - // Create outcome context (not really used for Observation) - outctx := ocr3types.OutcomeContext{ - SeqNr: 1, - PreviousOutcome: nil, // Not needed for Observation benchmark - } - - // Reset timer and enable memory allocation reporting - - b.ReportAllocs() - - // Preallocate memory stats variables - var memStatsBefore, memStatsAfter runtime.MemStats - - // Track cumulative metrics - var totalMemUsage uint64 - var totalObservationSize int - - // Run the benchmark - for b.Loop() { - runtime.GC() // Run garbage collection before measurement to reduce noise - runtime.ReadMemStats(&memStatsBefore) - - // Call Observation function - observation, err := plugin.Observation(context.Background(), outctx, query) - require.NoError(b, err) - - // Measure memory usage - runtime.ReadMemStats(&memStatsAfter) - memUsage := memStatsAfter.TotalAlloc - memStatsBefore.TotalAlloc - totalMemUsage += memUsage - - // Measure observation size - observationSize := len(observation) - totalObservationSize += observationSize - - // Basic validation of observation - var parsedObservation pbtypes.Observations - err = proto.Unmarshal(observation, &parsedObservation) - require.NoError(b, err) - require.Len(b, parsedObservation.Observations, numWorkflows) - } - - // Report average metrics - if b.N > 0 { - b.ReportMetric(float64(totalMemUsage)/float64(b.N), "B/memory") - b.ReportMetric(float64(totalObservationSize)/float64(b.N), "B/observation_size") - // Report streams per second metric to understand throughput - streamsProcessed := numWorkflows * numStreamsPerWorkflow - b.ReportMetric(float64(streamsProcessed), "streams/op") - } -} - -// runBenchmarkWithParams runs a benchmark with the specified parameters -func runBenchmarkWithParams(b *testing.B, lggr logger.Logger, numWorkflows, numStreamsPerWorkflow int) { - // Test parameters - const ( - numOracles = 4 // Total nodes - f = 1 // Fault tolerance - ) - - // Create request store - store := requests.NewStore[*ocr3.ReportRequest]() - - // Create capability with LLO aggregators for each workflow - mockCap := &mockCapability{ - aggregators: make(map[string]pbtypes.Aggregator, numWorkflows), - } - - // Create LLO aggregators for each workflow - for i := range numWorkflows { - workflowID := fmt.Sprintf("workflow-%d", i) - agg, err := createLLOAggregator(b, numStreamsPerWorkflow) - require.NoError(b, err) - mockCap.aggregators[workflowID] = agg - } - - // Create reporting plugin - plugin, err := ocr3.NewReportingPlugin( - store, - mockCap, - numWorkflows, // batchSize - ocr3types.ReportingPluginConfig{ - N: numOracles, - F: f, - }, - &pbtypes.ReportingPluginConfig{ - OutcomePruningThreshold: 100, - }, - lggr, - ) - require.NoError(b, err) - - // Create test query with 10 workflow IDs - query, err := createTestQuery(numWorkflows) - require.NoError(b, err) - - // Create previous outcome with the same 10 workflow IDs - previousOutcome, err := createTestPreviousOutcome(numWorkflows, numStreamsPerWorkflow) - require.NoError(b, err) - - // Create attributed observations from all oracles - aos := createTestAttributedObservations(b, numOracles, numWorkflows, numStreamsPerWorkflow) - - // Create outcome context - outctx := ocr3types.OutcomeContext{ - SeqNr: 1, - PreviousOutcome: previousOutcome, - } - - // Reset timer and enable memory allocation reporting - - b.ReportAllocs() - - // Run the benchmark - for b.Loop() { - var memStatsBefore, memStatsAfter runtime.MemStats - runtime.ReadMemStats(&memStatsBefore) - - // Call Outcome function - outcome, err := plugin.Outcome(context.Background(), outctx, query, aos) - require.NoError(b, err) - - // Measure memory usage - runtime.ReadMemStats(&memStatsAfter) - memUsage := memStatsAfter.TotalAlloc - memStatsBefore.TotalAlloc - - // Measure outcome size - outcomeSize := len(outcome) - - // Report custom metrics - b.ReportMetric(float64(memUsage), "B/memory") - b.ReportMetric(float64(outcomeSize), "B/outcome_size") - - // Validate outcome contents - var parsedOutcome pbtypes.Outcome - err = proto.Unmarshal(outcome, &parsedOutcome) - require.NoError(b, err) - require.Len(b, parsedOutcome.Outcomes, numWorkflows) - } -} - -// Helper functions - -// createTestQuery generates a query with the specified number of workflow IDs -func createTestQuery(numWorkflows int) ([]byte, error) { - ids := make([]*pbtypes.Id, numWorkflows) - for i := range numWorkflows { - ids[i] = &pbtypes.Id{ - WorkflowExecutionId: fmt.Sprintf("execution-%d", i), - WorkflowId: fmt.Sprintf("workflow-%d", i), - WorkflowOwner: "test-owner", - WorkflowName: fmt.Sprintf("Workflow %d", i), - WorkflowDonId: 1, - WorkflowDonConfigVersion: 1, - ReportId: fmt.Sprintf("report-%d", i), - KeyId: "test-key", - } - } - - query := &pbtypes.Query{ - Ids: ids, - } - - return proto.MarshalOptions{Deterministic: true}.Marshal(query) -} - -// createTestPreviousOutcome generates a previous outcome with consistent LLOOutcomeMetadata -func createTestPreviousOutcome(numWorkflows, numStreamsPerWorkflow int) ([]byte, error) { - outcome := &pbtypes.Outcome{ - Outcomes: make(map[string]*pbtypes.AggregationOutcome, numWorkflows), - CurrentReports: []*pbtypes.Report{}, - } - - // Create an identical LLOOutcomeMetadata for all workflows - baseMetadata := &datafeeds.LLOOutcomeMetadata{ - StreamInfo: make(map[uint32]*datafeeds.LLOStreamInfo, numStreamsPerWorkflow), - } - - // Populate with stream info - baseTime := time.Now().Add(-10 * time.Minute).UnixNano() - zeroPrice, _ := decimal.Zero.MarshalBinary() - - for i := range numStreamsPerWorkflow { - streamID := uint32(i) - baseMetadata.StreamInfo[streamID] = &datafeeds.LLOStreamInfo{ - Timestamp: baseTime, - Price: zeroPrice, - } - } - - // Marshal once - metadataBytes, err := proto.Marshal(baseMetadata) - if err != nil { - return nil, err - } - - // Create outcome entries for each workflow, using the same metadata - for i := range numWorkflows { - workflowID := fmt.Sprintf("workflow-%d", i) - outcome.Outcomes[workflowID] = &pbtypes.AggregationOutcome{ - Metadata: metadataBytes, - LastSeenAt: 1, - ShouldReport: false, - Timestamp: timestamppb.Now(), - EncodableOutcome: nil, // Not needed for benchmark - } - } - - return proto.MarshalOptions{Deterministic: true}.Marshal(outcome) -} - -// createTestAttributedObservations generates attributed observations from multiple oracles -func createTestAttributedObservations(b *testing.B, numOracles, numWorkflows, numStreamsPerWorkflow int) []types.AttributedObservation { - aos := make([]types.AttributedObservation, numOracles) - ts := timestamppb.Now() // Use a consistent timestamp for all observations to ensure consensus - for oracle := range numOracles { - observationsProto := &pbtypes.Observations{ - Observations: make([]*pbtypes.Observation, numWorkflows), - RegisteredWorkflowIds: make([]string, numWorkflows), - Timestamp: ts, - } - - // Create an observation for each workflow - for i := range numWorkflows { - workflowID := fmt.Sprintf("workflow-%d", i) - executionID := fmt.Sprintf("execution-%d", i) - observationsProto.RegisteredWorkflowIds[i] = workflowID - - // Create LLO events - lloEvent := createLLOEvent(b, numStreamsPerWorkflow, ts.AsTime()) - wrappedEvent, err := values.Wrap(lloEvent) - require.NoError(b, err) - - // Create list value with the LLO event - listVal, err := values.NewList([]any{wrappedEvent}) - require.NoError(b, err) - - listProto := values.Proto(listVal).GetListValue() - require.NotNil(b, listProto, "listProto should not be nil") // Ensure listProto is not nil - - // Add observation for this workflow - observationsProto.Observations[i] = &pbtypes.Observation{ - Id: &pbtypes.Id{ - WorkflowExecutionId: executionID, - WorkflowId: workflowID, - WorkflowOwner: "test-owner", - WorkflowName: fmt.Sprintf("Workflow %d", i), - WorkflowDonId: 1, - WorkflowDonConfigVersion: 1, - ReportId: fmt.Sprintf("report-%d", i), - KeyId: "test-key", - }, - Observations: listProto, - } - } - - // Marshal the observations - obsBytes, err := proto.MarshalOptions{Deterministic: true}.Marshal(observationsProto) - require.NoError(b, err) - - // Create attributed observation - aos[oracle] = types.AttributedObservation{ - Observation: obsBytes, - Observer: ocrcommon.OracleID(oracle), - } - } - - return aos -} - -// createLLOEvent creates an LLO event with the specified number of streams -func createLLOEvent(b *testing.B, numStreams int, ts time.Time) *datastreams.LLOStreamsTriggerEvent { - timestamp := uint64(ts.UnixNano()) - event := &datastreams.LLOStreamsTriggerEvent{ - ObservationTimestampNanoseconds: timestamp, - Payload: make([]*datastreams.LLOStreamDecimal, 0, numStreams), - } - - // Create stream values with consistent prices - for i := range numStreams { - price := decimal.NewFromInt(int64(100 + i%10)) // Use a few different price values - binary, err := price.MarshalBinary() - require.NoError(b, err) - - event.Payload = append(event.Payload, &datastreams.LLOStreamDecimal{ - StreamID: uint32(i), - Decimal: binary, - }) - } - - return event -} - -// createLLOAggregator creates an LLO aggregator with the specified number of streams -func createLLOAggregator(b *testing.B, numStreams int) (pbtypes.Aggregator, error) { - // Create feed configs for all streams - streamConfigs := make(map[string]datafeeds.FeedConfig, numStreams) - for i := range numStreams { - streamConfigs[strconv.Itoa(i)] = datafeeds.FeedConfig{ - // Deviation: decimal.NewFromFloat(0.01), // 1% deviation threshold - Heartbeat: 3600, // 1 hour heartbeat - RemappedIDHex: fmt.Sprintf("0x%064x", i+1000), // Unique remapped ID - } - } - - // Create LLO config - c := datafeeds.LLOAggregatorConfig{ - Streams: streamConfigs, - } - - // Create LLO aggregator - //return datafeeds.NewLLOAggregator(configMap) - m, err := c.ToMap() - if err != nil { - // Handle error in creating LLO aggregator - return nil, fmt.Errorf("failed to create LLO aggregator: %w", err) - } - return datafeeds.NewLLOAggregator(*m) -} diff --git a/pkg/capabilities/consensus/ocr3/capability.go b/pkg/capabilities/consensus/ocr3/capability.go deleted file mode 100644 index 4d30ae3d7c..0000000000 --- a/pkg/capabilities/consensus/ocr3/capability.go +++ /dev/null @@ -1,303 +0,0 @@ -package ocr3 - -import ( - "context" - "fmt" - "strconv" - "sync" - "time" - - "github.com/jonboulle/clockwork" - "google.golang.org/protobuf/proto" - - "github.com/smartcontractkit/chainlink-common/pkg/capabilities" - "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/ocr3/types" - "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/requests" - "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/metering" - "github.com/smartcontractkit/chainlink-common/pkg/services" - "github.com/smartcontractkit/chainlink-protos/cre/go/values" -) - -const ( - ocrCapabilityID = "offchain_reporting@1.0.0" - - methodStartRequest = "start_request" - methodSendResponse = "send_response" - methodHeader = "method" - transmissionHeader = "transmission" - terminateHeader = "terminate" -) - -var info = capabilities.MustNewCapabilityInfo( - ocrCapabilityID, - capabilities.CapabilityTypeConsensus, - "OCR3 consensus exposed as a capability.", -) - -type capability struct { - services.Service - eng *services.Engine - - capabilities.CapabilityInfo - capabilities.Validator[config, inputs, ReportResponse] - - reqHandler *requests.Handler[*ReportRequest, ReportResponse] - - requestTimeout time.Duration - requestTimeoutLock sync.RWMutex - - clock clockwork.Clock - - aggregatorFactory types.AggregatorFactory - aggregators map[string]types.Aggregator - - encoderFactory types.EncoderFactory - encoders map[string]types.Encoder - - callbackChannelBufferSize int - - registeredWorkflowsIDs map[string]bool - mu sync.RWMutex -} - -var _ CapabilityIface = (*capability)(nil) -var _ capabilities.ExecutableCapability = (*capability)(nil) - -func NewCapability(s *requests.Store[*ReportRequest], clock clockwork.Clock, requestTimeout time.Duration, aggregatorFactory types.AggregatorFactory, encoderFactory types.EncoderFactory, lggr logger.Logger, - callbackChannelBufferSize int) *capability { - o := &capability{ - CapabilityInfo: info, - Validator: capabilities.NewValidator[config, inputs, ReportResponse](capabilities.ValidatorArgs{Info: info}), - clock: clock, - requestTimeout: requestTimeout, - aggregatorFactory: aggregatorFactory, - aggregators: map[string]types.Aggregator{}, - encoderFactory: encoderFactory, - encoders: map[string]types.Encoder{}, - - callbackChannelBufferSize: callbackChannelBufferSize, - registeredWorkflowsIDs: map[string]bool{}, - } - o.Service, o.eng = services.Config{ - Name: "OCR3CapabilityClient", - NewSubServices: func(l logger.Logger) []services.Service { - o.reqHandler = requests.NewHandler(lggr, s, clock, requestTimeout) - return []services.Service{o.reqHandler} - }, - }.NewServiceEngine(lggr) - return o -} - -func (o *capability) RegisterToWorkflow(ctx context.Context, request capabilities.RegisterToWorkflowRequest) error { - c, err := o.ValidateConfig(request.Config) - if err != nil { - return err - } - - o.mu.Lock() - defer o.mu.Unlock() - agg, err := o.aggregatorFactory(c.AggregationMethod, *c.AggregationConfig, o.eng) - if err != nil { - return err - } - o.aggregators[request.Metadata.WorkflowID] = agg - - encoder, err := o.encoderFactory(c.Encoder, c.EncoderConfig, o.eng) - if err != nil { - return err - } - o.encoders[request.Metadata.WorkflowID] = encoder - o.registeredWorkflowsIDs[request.Metadata.WorkflowID] = true - return nil -} - -func (o *capability) GetAggregator(workflowID string) (types.Aggregator, error) { - agg, ok := o.aggregators[workflowID] - if !ok { - return nil, fmt.Errorf("no aggregator found for workflowID %s", workflowID) - } - - return agg, nil -} - -func (o *capability) GetEncoderByWorkflowID(workflowID string) (types.Encoder, error) { - enc, ok := o.encoders[workflowID] - if !ok { - return nil, fmt.Errorf("no encoder found for workflowID %s", workflowID) - } - - return enc, nil -} - -func (o *capability) GetEncoderByName(encoderName string, config *values.Map) (types.Encoder, error) { - return o.encoderFactory(encoderName, config, o.eng) -} - -func (o *capability) GetRegisteredWorkflowsIDs() []string { - o.mu.RLock() - defer o.mu.RUnlock() - - workflows := make([]string, 0, len(o.registeredWorkflowsIDs)) - for wf := range o.registeredWorkflowsIDs { - workflows = append(workflows, wf) - } - return workflows -} - -func (o *capability) UnregisterWorkflowID(workflowID string) { - o.mu.Lock() - defer o.mu.Unlock() - delete(o.registeredWorkflowsIDs, workflowID) -} - -func (o *capability) UnregisterFromWorkflow(ctx context.Context, request capabilities.UnregisterFromWorkflowRequest) error { - o.mu.Lock() - defer o.mu.Unlock() - delete(o.registeredWorkflowsIDs, request.Metadata.WorkflowID) - delete(o.aggregators, request.Metadata.WorkflowID) - delete(o.encoders, request.Metadata.WorkflowID) - return nil -} - -func (o *capability) setRequestTimeout(timeout time.Duration) { - o.requestTimeoutLock.Lock() - defer o.requestTimeoutLock.Unlock() - o.requestTimeout = timeout -} - -// Execute enqueues a new consensus request, passing it to the reporting plugin as needed. -// IMPORTANT: OCR3 only exposes signatures via the contractTransmitter, which is located -// in a separate process to the reporting plugin LOOPP. However, only the reporting plugin -// LOOPP is able to transmit responses back to the workflow engine. As a workaround to this, we've implemented a custom contract transmitter which fetches this capability from the -// registry and calls Execute with the response, setting "method = `methodSendResponse`". -func (o *capability) Execute(ctx context.Context, r capabilities.CapabilityRequest) (capabilities.CapabilityResponse, error) { - m := struct { - Method string - Transmission map[string]any - Terminate bool - }{ - Method: methodStartRequest, - } - err := r.Inputs.UnwrapTo(&m) - if err != nil { - o.eng.Warnf("could not unwrap method from CapabilityRequest, using default: %v", err) - } - - switch m.Method { - case methodSendResponse: - inputs, err := values.NewMap(m.Transmission) - if err != nil { - return capabilities.CapabilityResponse{}, fmt.Errorf("failed to create map for response inputs: %w", err) - } - o.eng.Debugw("Execute - sending response", "workflowExecutionID", r.Metadata.WorkflowExecutionID, "inputs", inputs, "terminate", m.Terminate) - var responseErr error - if m.Terminate { - o.eng.Debugw("Execute - terminating execution", "workflowExecutionID", r.Metadata.WorkflowExecutionID) - responseErr = capabilities.ErrStopExecution - } - out := ReportResponse{ - WorkflowExecutionID: r.Metadata.WorkflowExecutionID, - Value: inputs, - Err: responseErr, - } - o.reqHandler.SendResponse(ctx, out) - - // Return a dummy response back to the caller - // This allows the transmitter to block on a response before - // returning from Transmit() - return capabilities.CapabilityResponse{}, nil - case methodStartRequest: - // Receives and stores an observation to do consensus on - // Receives an aggregation method; at this point the method has been validated - // Returns the consensus result over a channel - inputs, err := o.ValidateInputs(r.Inputs) - if err != nil { - return capabilities.CapabilityResponse{}, err - } - inputLenBytes := byteSizeOfMap(r.Inputs) - - config, err := o.ValidateConfig(r.Config) - if err != nil { - return capabilities.CapabilityResponse{}, err - } - - ch, err := o.queueRequestForProcessing(ctx, r.Metadata, inputs, config) - if err != nil { - return capabilities.CapabilityResponse{}, err - } - - select { - case <-ctx.Done(): - return capabilities.CapabilityResponse{}, ctx.Err() - case response := <-ch: - outputLenBytes := byteSizeOfMap(response.Value) - return capabilities.CapabilityResponse{ - Value: response.Value, - Metadata: capabilities.ResponseMetadata{ - Metering: []capabilities.MeteringNodeDetail{ - {SpendUnit: metering.PayloadUnit.Name, SpendValue: strconv.Itoa(inputLenBytes + outputLenBytes)}, - }, - }, - }, response.Err - } - } - - return capabilities.CapabilityResponse{}, fmt.Errorf("unknown method: %s", m.Method) -} - -// queueRequestForProcessing queues a request for processing by the worker -// goroutine by adding the request to its store. -// -// When a request is queued, a timer is started to ensure that the request does not exceed its expiry time. -func (o *capability) queueRequestForProcessing( - ctx context.Context, - metadata capabilities.RequestMetadata, - i *inputs, - c *config, -) (<-chan ReportResponse, error) { - callbackCh := make(chan ReportResponse, o.callbackChannelBufferSize) - - // Use the capability-level request timeout unless the request's config specifies - // its own timeout, in which case we'll use that instead. This allows the workflow spec - // to configure more granular timeouts depending on the circumstances. - o.requestTimeoutLock.RLock() - requestTimeout := o.requestTimeout - if c.RequestTimeoutMS != 0 { - requestTimeout = time.Duration(c.RequestTimeoutMS) * time.Millisecond - } - o.requestTimeoutLock.RUnlock() - - r := &ReportRequest{ - StopCh: make(chan struct{}), - CallbackCh: callbackCh, - WorkflowExecutionID: metadata.WorkflowExecutionID, - WorkflowID: metadata.WorkflowID, - WorkflowOwner: metadata.WorkflowOwner, - WorkflowName: metadata.WorkflowName, - ReportID: c.ReportID, - WorkflowDonID: metadata.WorkflowDonID, - WorkflowDonConfigVersion: metadata.WorkflowDonConfigVersion, - Observations: i.Observations, - OverriddenEncoderName: i.EncoderName, - OverriddenEncoderConfig: i.EncoderConfig, - KeyID: c.KeyID, - ExpiresAt: o.clock.Now().Add(requestTimeout), - } - - o.eng.Debugw("Execute - adding to store", "workflowID", r.WorkflowID, "workflowExecutionID", r.WorkflowExecutionID, "observations", r.Observations) - - o.reqHandler.SendRequest(ctx, r) - return callbackCh, nil -} - -// byteSizeOfMap is a utility to get the wire-size -// of a values.Map. -func byteSizeOfMap(m *values.Map) int { - if m == nil { - return 0 - } - pbVal := values.Proto(m) - size := proto.Size(pbVal) - return size -} diff --git a/pkg/capabilities/consensus/ocr3/capability_test.go b/pkg/capabilities/consensus/ocr3/capability_test.go deleted file mode 100644 index aaaf406efe..0000000000 --- a/pkg/capabilities/consensus/ocr3/capability_test.go +++ /dev/null @@ -1,541 +0,0 @@ -package ocr3 - -import ( - "context" - "os" - "testing" - "time" - - "github.com/google/uuid" - "github.com/jonboulle/clockwork" - "github.com/shopspring/decimal" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/smartcontractkit/chainlink-common/pkg/capabilities" - "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/ocr3/types" - "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/requests" - "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/utils" - "github.com/smartcontractkit/chainlink-protos/cre/go/values" -) - -const workflowTestID = "consensus-workflow-test-id-1" -const workflowTestID2 = "consensus-workflow-test-id-2" -const workflowTestID3 = "consensus-workflow-test-id-3" -const workflowExecutionTestID = "consensus-workflow-execution-test-id-1" -const workflowTestName = "consensus-workflow-test-name-1" -const reportTestID = "rep-id-1" - -type mockAggregator struct { - types.Aggregator -} - -func mockAggregatorFactory(_ string, _ values.Map, _ logger.Logger) (types.Aggregator, error) { - return &mockAggregator{}, nil -} - -type encoder struct { - types.Encoder -} - -func mockEncoderFactory(_ string, _ *values.Map, _ logger.Logger) (types.Encoder, error) { - return &encoder{}, nil -} - -func TestOCR3Capability_Schema(t *testing.T) { - n := time.Now() - fc := clockwork.NewFakeClockAt(n) - lggr := logger.Nop() - - s := requests.NewStore[*ReportRequest]() - - cp := NewCapability(s, fc, 1*time.Second, mockAggregatorFactory, mockEncoderFactory, lggr, 10) - schema, err := cp.Schema() - require.NoError(t, err) - - var shouldUpdate = false - if shouldUpdate { - err = os.WriteFile("./testdata/fixtures/capability/schema.json", []byte(schema), 0600) - require.NoError(t, err) - } - - fixture, err := os.ReadFile("./testdata/fixtures/capability/schema.json") - require.NoError(t, err) - - utils.AssertJSONEqual(t, fixture, []byte(schema)) -} - -func TestOCR3Capability(t *testing.T) { - cases := []struct { - name string - aggregationMethod string - }{ - { - name: "success - aggregation_method data_feeds", - aggregationMethod: "data_feeds", - }, - { - name: "success - aggregation_method reduce", - aggregationMethod: "reduce", - }, - } - for _, tt := range cases { - t.Run(tt.name, func(t *testing.T) { - n := time.Now() - fc := clockwork.NewFakeClockAt(n) - lggr := logger.Test(t) - - ctx := t.Context() - - s := requests.NewStore[*ReportRequest]() - - cp := NewCapability(s, fc, 1*time.Second, mockAggregatorFactory, mockEncoderFactory, lggr, 10) - require.NoError(t, cp.Start(ctx)) - - config, err := values.NewMap( - map[string]any{ - "aggregation_method": tt.aggregationMethod, - "aggregation_config": map[string]any{}, - "encoder_config": map[string]any{}, - "encoder": "evm", - "report_id": "ffff", - "key_id": "evm", - }, - ) - require.NoError(t, err) - - ethUsdValStr := "1.123456" - ethUsdValue, err := decimal.NewFromString(ethUsdValStr) - require.NoError(t, err) - observationKey := "ETH_USD" - obs := []any{map[string]any{observationKey: ethUsdValue}} - inputs, err := values.NewMap(map[string]any{"observations": obs}) - require.NoError(t, err) - - executeReq := capabilities.CapabilityRequest{ - Metadata: capabilities.RequestMetadata{ - WorkflowID: workflowTestID, - WorkflowExecutionID: workflowExecutionTestID, - }, - Config: config, - Inputs: inputs, - } - - respCh := executeAsync(ctx, executeReq, cp.Execute) - - obsv, err := values.NewList(obs) - require.NoError(t, err) - - // Mock the oracle returning a response - mresp, err := values.NewMap(map[string]any{"observations": obsv}) - cp.reqHandler.SendResponse(ctx, ReportResponse{ - Value: mresp, - WorkflowExecutionID: workflowExecutionTestID, - }) - require.NoError(t, err) - - resp := <-respCh - assert.NoError(t, resp.Err) - - assert.Equal(t, mresp, resp.Value) - assert.Equal(t, "RESOURCE_TYPE_NETWORK", resp.Metadata.Metering[0].SpendUnit) - assert.Equal(t, "122", resp.Metadata.Metering[0].SpendValue) - }) - } -} - -func TestOCR3Capability_Eviction(t *testing.T) { - n := time.Now() - fc := clockwork.NewFakeClockAt(n) - lggr := logger.Test(t) - - ctx := t.Context() - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - rea := time.Second - s := requests.NewStore[*ReportRequest]() - cp := NewCapability(s, fc, rea, mockAggregatorFactory, mockEncoderFactory, lggr, 10) - require.NoError(t, cp.Start(ctx)) - - config, err := values.NewMap( - map[string]any{ - "aggregation_method": "data_feeds", - "aggregation_config": map[string]any{}, - "encoder_config": map[string]any{}, - "encoder": "evm", - "report_id": "aaaa", - "key_id": "evm", - }, - ) - require.NoError(t, err) - - ethUsdValue, err := decimal.NewFromString("1.123456") - require.NoError(t, err) - inputs, err := values.NewMap(map[string]any{"observations": []any{map[string]any{"ETH_USD": ethUsdValue}}}) - require.NoError(t, err) - - rid := uuid.New().String() - executeReq := capabilities.CapabilityRequest{ - Metadata: capabilities.RequestMetadata{ - WorkflowID: workflowTestID, - WorkflowExecutionID: rid, - }, - Config: config, - Inputs: inputs, - } - - done := make(chan struct{}) - t.Cleanup(func() { <-done }) - go func() { - defer close(done) - ticker := time.NewTicker(1 * time.Second) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - fc.Advance(1 * time.Hour) - } - } - }() - - respCh := executeAsync(ctx, executeReq, cp.Execute) - - resp := <-respCh - assert.ErrorContains(t, resp.Err, "timeout exceeded: could not process request before expiry") - - request := s.Get(rid) - assert.Nil(t, request) - - assert.NoError(t, err) -} - -func TestOCR3Capability_EvictionUsingConfig(t *testing.T) { - n := time.Now() - fc := clockwork.NewFakeClockAt(n) - lggr := logger.Test(t) - - ctx := t.Context() - ctx, cancel := context.WithCancel(ctx) - defer cancel() - // This is the default expired at - rea := time.Hour - s := requests.NewStore[*ReportRequest]() - cp := NewCapability(s, fc, rea, mockAggregatorFactory, mockEncoderFactory, lggr, 10) - require.NoError(t, cp.Start(ctx)) - - config, err := values.NewMap( - map[string]any{ - "aggregation_method": "data_feeds", - "aggregation_config": map[string]any{}, - "encoder_config": map[string]any{}, - "encoder": "evm", - "report_id": "aaaa", - "key_id": "evm", - "request_timeout_ms": 10000, - }, - ) - require.NoError(t, err) - - ethUsdValue, err := decimal.NewFromString("1.123456") - require.NoError(t, err) - inputs, err := values.NewMap(map[string]any{"observations": []any{map[string]any{"ETH_USD": ethUsdValue}}}) - require.NoError(t, err) - - rid := uuid.New().String() - executeReq := capabilities.CapabilityRequest{ - Metadata: capabilities.RequestMetadata{ - WorkflowID: workflowTestID, - WorkflowExecutionID: rid, - }, - Config: config, - Inputs: inputs, - } - - // 1 minute is more than the config timeout we provided, but less than - // the hardcoded timeout. - done := make(chan struct{}) - t.Cleanup(func() { <-done }) - go func() { - defer close(done) - ticker := time.NewTicker(1 * time.Second) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - // 1 minute is more than the config timeout we provided, but less than - // the hardcoded timeout. - fc.Advance(1 * time.Minute) - } - } - }() - - _, err = cp.Execute(ctx, executeReq) - - assert.ErrorContains(t, err, "timeout exceeded: could not process request before expiry") - - reqs := s.GetByIDs([]string{rid}) - - assert.Empty(t, reqs) -} - -func TestOCR3Capability_Registration(t *testing.T) { - n := time.Now() - fc := clockwork.NewFakeClockAt(n) - lggr := logger.Test(t) - - ctx := t.Context() - s := requests.NewStore[*ReportRequest]() - cp := NewCapability(s, fc, 1*time.Second, mockAggregatorFactory, mockEncoderFactory, lggr, 10) - require.NoError(t, cp.Start(ctx)) - - config, err := values.NewMap(map[string]any{ - "aggregation_method": "data_feeds", - "aggregation_config": map[string]any{}, - "encoder": "", - "encoder_config": map[string]any{}, - "report_id": "000f", - "key_id": "evm", - }) - require.NoError(t, err) - - registerReq := capabilities.RegisterToWorkflowRequest{ - Metadata: capabilities.RegistrationMetadata{ - WorkflowID: workflowTestID, - }, - Config: config, - } - - err = cp.RegisterToWorkflow(ctx, registerReq) - require.NoError(t, err) - - agg, err := cp.GetAggregator(workflowTestID) - require.NoError(t, err) - assert.NotNil(t, agg) - - unregisterReq := capabilities.UnregisterFromWorkflowRequest{ - Metadata: capabilities.RegistrationMetadata{ - WorkflowID: workflowTestID, - }, - } - - err = cp.UnregisterFromWorkflow(ctx, unregisterReq) - require.NoError(t, err) - - _, err = cp.GetAggregator(workflowTestID) - assert.ErrorContains(t, err, "no aggregator found for") -} - -func TestOCR3Capability_ValidateConfig(t *testing.T) { - n := time.Now() - fc := clockwork.NewFakeClockAt(n) - lggr := logger.Test(t) - - s := requests.NewStore[*ReportRequest]() - - o := NewCapability(s, fc, 1*time.Second, mockAggregatorFactory, mockEncoderFactory, lggr, 10) - - t.Run("ValidConfig", func(t *testing.T) { - config, err := values.NewMap(map[string]any{ - "aggregation_method": "data_feeds", - "aggregation_config": map[string]any{}, - "encoder": "", - "encoder_config": map[string]any{}, - "report_id": "aaaa", - "key_id": "evm", - }) - require.NoError(t, err) - - c, err := o.ValidateConfig(config) - require.NoError(t, err) - require.NotNil(t, c) - }) - - t.Run("InvalidConfig null", func(t *testing.T) { - config, err := values.NewMap(map[string]any{ - "aggregation_method": "data_feeds", - "report_id": "aaaa", - "key_id": "evm", - }) - require.NoError(t, err) - - c, err := o.ValidateConfig(config) - require.Error(t, err) - assert.Contains(t, err.Error(), "expected object, but got null") // taken from the error json schema error message - require.Nil(t, c) - }) - - t.Run("InvalidConfig illegal report_id", func(t *testing.T) { - config, err := values.NewMap(map[string]any{ - "aggregation_method": "data_feeds", - "aggregation_config": map[string]any{}, - "encoder": "", - "encoder_config": map[string]any{}, - "report_id": "aa", - "key_id": "evm", - }) - require.NoError(t, err) - - c, err := o.ValidateConfig(config) - require.Error(t, err) - assert.Contains(t, err.Error(), "does not match pattern") // taken from the error json schema error message - require.Nil(t, c) - }) - - t.Run("InvalidConfig no key_id", func(t *testing.T) { - config, err := values.NewMap(map[string]any{ - "aggregation_method": "data_feeds", - "aggregation_config": map[string]any{}, - "encoder": "", - "encoder_config": map[string]any{}, - "report_id": "aaaa", - }) - require.NoError(t, err) - - c, err := o.ValidateConfig(config) - require.Error(t, err) - assert.Contains(t, err.Error(), "missing properties: 'key_id'") // taken from the error json schema error message - require.Nil(t, c) - }) -} - -func TestOCR3Capability_RespondsToLateRequest(t *testing.T) { - n := time.Now() - fc := clockwork.NewFakeClockAt(n) - lggr := logger.Test(t) - - ctx := t.Context() - - s := requests.NewStore[*ReportRequest]() - - cp := NewCapability(s, fc, 1*time.Second, mockAggregatorFactory, mockEncoderFactory, lggr, 10) - require.NoError(t, cp.Start(ctx)) - - config, err := values.NewMap( - map[string]any{ - "aggregation_method": "data_feeds", - "aggregation_config": map[string]any{}, - "encoder_config": map[string]any{}, - "encoder": "evm", - "report_id": "ffff", - "key_id": "evm", - }, - ) - require.NoError(t, err) - - ethUsdValStr := "1.123456" - ethUsdValue, err := decimal.NewFromString(ethUsdValStr) - require.NoError(t, err) - observationKey := "ETH_USD" - obs := map[string]any{observationKey: ethUsdValue} - inputs, err := values.NewMap(map[string]any{"observations": []any{obs}}) - require.NoError(t, err) - - obsv, err := values.NewMap(obs) - require.NoError(t, err) - - // Mock the oracle returning a response prior to the request being sent - cp.reqHandler.SendResponse(ctx, ReportResponse{ - Value: obsv, - WorkflowExecutionID: workflowExecutionTestID, - }) - require.NoError(t, err) - - executeReq := capabilities.CapabilityRequest{ - Metadata: capabilities.RequestMetadata{ - WorkflowID: workflowTestID, - WorkflowExecutionID: workflowExecutionTestID, - }, - Config: config, - Inputs: inputs, - } - response, err := cp.Execute(ctx, executeReq) - require.NoError(t, err) - - expectedCapabilityResponse := capabilities.CapabilityResponse{ - Value: obsv, - } - - assert.Equal(t, expectedCapabilityResponse.Value, response.Value) -} - -func TestOCR3Capability_RespondingToLateRequestDoesNotBlockOnSlowResponseConsumer(t *testing.T) { - n := time.Now() - fc := clockwork.NewFakeClockAt(n) - lggr := logger.Test(t) - - ctx := t.Context() - - s := requests.NewStore[*ReportRequest]() - - cp := NewCapability(s, fc, 1*time.Second, mockAggregatorFactory, mockEncoderFactory, lggr, 0) - require.NoError(t, cp.Start(ctx)) - - config, err := values.NewMap( - map[string]any{ - "aggregation_method": "data_feeds", - "aggregation_config": map[string]any{}, - "encoder_config": map[string]any{}, - "encoder": "evm", - "report_id": "ffff", - "key_id": "evm", - }, - ) - require.NoError(t, err) - - ethUsdValStr := "1.123456" - ethUsdValue, err := decimal.NewFromString(ethUsdValStr) - require.NoError(t, err) - observationKey := "ETH_USD" - obs := map[string]any{observationKey: ethUsdValue} - inputs, err := values.NewMap(map[string]any{"observations": []any{obs}}) - require.NoError(t, err) - - obsv, err := values.NewMap(obs) - require.NoError(t, err) - - // Mock the oracle returning a response prior to the request being sent - cp.reqHandler.SendResponse(ctx, ReportResponse{ - Value: obsv, - WorkflowExecutionID: workflowExecutionTestID, - }) - require.NoError(t, err) - - executeReq := capabilities.CapabilityRequest{ - Metadata: capabilities.RequestMetadata{ - WorkflowID: workflowTestID, - WorkflowExecutionID: workflowExecutionTestID, - }, - Config: config, - Inputs: inputs, - } - resp, err := cp.Execute(ctx, executeReq) - require.NoError(t, err) - - expectedCapabilityResponse := capabilities.CapabilityResponse{ - Value: obsv, - } - - assert.Equal(t, expectedCapabilityResponse.Value, resp.Value) -} - -type asyncCapabilityResponse struct { - capabilities.CapabilityResponse - Err error -} - -func executeAsync(ctx context.Context, request capabilities.CapabilityRequest, toExecute func(ctx context.Context, request capabilities.CapabilityRequest) (capabilities.CapabilityResponse, error)) <-chan asyncCapabilityResponse { - respCh := make(chan asyncCapabilityResponse, 1) - go func() { - resp, err := toExecute(ctx, request) - respCh <- asyncCapabilityResponse{CapabilityResponse: capabilities.CapabilityResponse{Value: resp.Value, Metadata: resp.Metadata}, Err: err} - close(respCh) - }() - - return respCh -} diff --git a/pkg/capabilities/consensus/ocr3/factory.go b/pkg/capabilities/consensus/ocr3/factory.go deleted file mode 100644 index d349ce5c3d..0000000000 --- a/pkg/capabilities/consensus/ocr3/factory.go +++ /dev/null @@ -1,104 +0,0 @@ -package ocr3 - -import ( - "context" - "time" - - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/durationpb" - - "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types" - - "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/ocr3/types" - "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/requests" - "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/services" -) - -const ( - defaultMaxPhaseOutputBytes = 1000000 // 1 MB - defaultMaxReportCount = 20 - defaultBatchSize = 20 - defaultOutcomePruningThreshold = 3600 - defaultRequestExpiry = 20 * time.Second -) - -type factory struct { - store *requests.Store[*ReportRequest] - capability *capability - lggr logger.Logger - - services.StateMachine -} - -func newFactory(s *requests.Store[*ReportRequest], c *capability, lggr logger.Logger) (*factory, error) { - return &factory{ - store: s, - capability: c, - lggr: logger.Named(lggr, "OCR3ReportingPluginFactory"), - }, nil -} - -func (o *factory) NewReportingPlugin(_ context.Context, config ocr3types.ReportingPluginConfig) (ocr3types.ReportingPlugin[[]byte], ocr3types.ReportingPluginInfo, error) { - var configProto types.ReportingPluginConfig - err := proto.Unmarshal(config.OffchainConfig, &configProto) - if err != nil { - // an empty byte array will be unmarshalled into zero values without error - return nil, ocr3types.ReportingPluginInfo{}, err - } - if configProto.MaxQueryLengthBytes <= 0 { - configProto.MaxQueryLengthBytes = defaultMaxPhaseOutputBytes - } - if configProto.MaxObservationLengthBytes <= 0 { - configProto.MaxObservationLengthBytes = defaultMaxPhaseOutputBytes - } - if configProto.MaxOutcomeLengthBytes <= 0 { - configProto.MaxOutcomeLengthBytes = defaultMaxPhaseOutputBytes - } - if configProto.MaxReportLengthBytes <= 0 { - configProto.MaxReportLengthBytes = defaultMaxPhaseOutputBytes - } - if configProto.MaxBatchSize <= 0 { - configProto.MaxBatchSize = defaultBatchSize - } - if configProto.OutcomePruningThreshold <= 0 { - configProto.OutcomePruningThreshold = defaultOutcomePruningThreshold - } - if configProto.MaxReportCount <= 0 { - configProto.MaxReportCount = defaultMaxReportCount - } - if configProto.RequestTimeout == nil { - configProto.RequestTimeout = durationpb.New(defaultRequestExpiry) - } - o.capability.setRequestTimeout(configProto.RequestTimeout.AsDuration()) - rp, err := NewReportingPlugin(o.store, o.capability, int(configProto.MaxBatchSize), config, &configProto, o.lggr) - rpInfo := ocr3types.ReportingPluginInfo{ - Name: "OCR3 Capability Plugin", - Limits: ocr3types.ReportingPluginLimits{ - MaxQueryLength: int(configProto.MaxQueryLengthBytes), - MaxObservationLength: int(configProto.MaxObservationLengthBytes), - MaxOutcomeLength: int(configProto.MaxOutcomeLengthBytes), - MaxReportLength: int(configProto.MaxReportLengthBytes), - MaxReportCount: int(configProto.MaxReportCount), - }, - } - return rp, rpInfo, err -} - -func (o *factory) Start(ctx context.Context) error { - return o.StartOnce("OCR3ReportingPlugin", func() error { - return nil - }) -} - -func (o *factory) Close() error { - return o.StopOnce("OCR3ReportingPlugin", func() error { - return nil - }) -} - -func (o *factory) Name() string { return o.lggr.Name() } - -func (o *factory) HealthReport() map[string]error { - return map[string]error{o.Name(): o.Healthy()} -} diff --git a/pkg/capabilities/consensus/ocr3/models.go b/pkg/capabilities/consensus/ocr3/models.go deleted file mode 100644 index 1080b8f584..0000000000 --- a/pkg/capabilities/consensus/ocr3/models.go +++ /dev/null @@ -1,22 +0,0 @@ -package ocr3 - -import ( - "github.com/smartcontractkit/chainlink-protos/cre/go/values" -) - -type config struct { - AggregationMethod string `mapstructure:"aggregation_method" json:"aggregation_method" jsonschema:"enum=data_feeds,enum=llo_streams,enum=identical,enum=reduce,enum=secure_mint"` - AggregationConfig *values.Map `mapstructure:"aggregation_config" json:"aggregation_config"` - Encoder string `mapstructure:"encoder" json:"encoder"` - EncoderConfig *values.Map `mapstructure:"encoder_config" json:"encoder_config"` - ReportID string `mapstructure:"report_id" json:"report_id" jsonschema:"required,pattern=^[a-f0-9]{4}$"` - RequestTimeoutMS int64 `mapstructure:"request_timeout_ms" json:"request_timeout_ms"` - - KeyID string `mapstructure:"key_id" json:"key_id,omitempty" jsonschema:"required"` -} - -type inputs struct { - Observations *values.List `json:"observations" jsonschema:""` - EncoderName string `mapstructure:"encoder" json:"encoder,omitempty"` - EncoderConfig *values.Map `mapstructure:"encoder_config" json:"encoder_config,omitempty"` -} diff --git a/pkg/capabilities/consensus/ocr3/ocr3.go b/pkg/capabilities/consensus/ocr3/ocr3.go deleted file mode 100644 index c22c189f61..0000000000 --- a/pkg/capabilities/consensus/ocr3/ocr3.go +++ /dev/null @@ -1,112 +0,0 @@ -package ocr3 - -import ( - "context" - "errors" - "time" - - "github.com/jonboulle/clockwork" - - "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/ocr3/types" - "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/requests" - "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/loop" - "github.com/smartcontractkit/chainlink-common/pkg/loop/reportingplugins" - ocr3rp "github.com/smartcontractkit/chainlink-common/pkg/loop/reportingplugins/ocr3" - commontypes "github.com/smartcontractkit/chainlink-common/pkg/types" - "github.com/smartcontractkit/chainlink-common/pkg/types/core" -) - -var _ ocr3rp.ProviderServer[commontypes.PluginProvider] = (*Capability)(nil) - -type Capability struct { - loop.Plugin - reportingplugins.PluginProviderServer - config Config - capabilityRegistry core.CapabilitiesRegistry -} - -type Config struct { - RequestTimeout *time.Duration - Logger logger.Logger - AggregatorFactory types.AggregatorFactory - EncoderFactory types.EncoderFactory - SendBufferSize int - - store *requests.Store[*ReportRequest] - capability *capability - clock clockwork.Clock -} - -const ( - defaultSendBufferSize = 10 -) - -func NewOCR3(config Config) *Capability { - if config.RequestTimeout == nil { - dre := defaultRequestExpiry - config.RequestTimeout = &dre - } - - if config.SendBufferSize == 0 { - config.SendBufferSize = defaultSendBufferSize - } - - if config.clock == nil { - config.clock = clockwork.NewRealClock() - } - - if config.store == nil { - config.store = requests.NewStore[*ReportRequest]() - } - - if config.capability == nil { - ci := NewCapability(config.store, config.clock, *config.RequestTimeout, config.AggregatorFactory, config.EncoderFactory, config.Logger, - config.SendBufferSize) - config.capability = ci - } - - cp := &Capability{ - Plugin: loop.Plugin{Logger: config.Logger}, - PluginProviderServer: reportingplugins.PluginProviderServer{}, - config: config, - } - - cp.SubService(config.capability) - return cp -} - -func (o *Capability) NewReportingPluginFactory(ctx context.Context, cfg core.ReportingPluginServiceConfig, - provider commontypes.PluginProvider, pipelineRunner core.PipelineRunnerService, telemetry core.TelemetryClient, - errorLog core.ErrorLog, capabilityRegistry core.CapabilitiesRegistry, keyValueStore core.KeyValueStore, - relayerSet core.RelayerSet) (core.OCR3ReportingPluginFactory, error) { - f, err := newFactory(o.config.store, o.config.capability, o.config.Logger) - if err != nil { - return nil, err - } - - err = capabilityRegistry.Add(ctx, o.config.capability) - if err != nil { - return nil, err - } - - o.capabilityRegistry = capabilityRegistry - - return f, err -} - -func (o *Capability) NewValidationService(ctx context.Context) (core.ValidationService, error) { - s := &validationService{lggr: o.Logger} - o.SubService(s) - return s, nil -} - -func (o *Capability) Close() error { - err := o.Plugin.Close() - - if o.capabilityRegistry != nil { - err = errors.Join(err, o.capabilityRegistry.Remove(context.TODO(), o.config.capability.ID)) - } - - return err -} diff --git a/pkg/capabilities/consensus/ocr3/ocr3_test.go b/pkg/capabilities/consensus/ocr3/ocr3_test.go deleted file mode 100644 index 81ec655440..0000000000 --- a/pkg/capabilities/consensus/ocr3/ocr3_test.go +++ /dev/null @@ -1,71 +0,0 @@ -package ocr3 - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - - "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/types" - "github.com/smartcontractkit/chainlink-common/pkg/types/core" - "github.com/smartcontractkit/chainlink-common/pkg/types/core/mocks" -) - -func TestOCR3_ReportingFactoryAddsCapability(t *testing.T) { - ctx := t.Context() - - cfg := Config{ - EncoderFactory: mockEncoderFactory, - Logger: logger.Test(t), - } - o := NewOCR3(cfg) - require.NoError(t, o.Start(ctx)) - - var p types.PluginProvider - var pr core.PipelineRunnerService - var tc core.TelemetryClient - var el core.ErrorLog - var kv core.KeyValueStore - var rs core.RelayerSet - r := mocks.NewCapabilitiesRegistry(t) - r.On("Add", mock.Anything, o.config.capability).Return(nil) - - _, err := o.NewReportingPluginFactory(ctx, core.ReportingPluginServiceConfig{}, p, pr, tc, el, r, kv, rs) - require.NoError(t, err) -} - -func TestOCR3_ReportingFactoryIsAService(t *testing.T) { - ctx := t.Context() - - cfg := Config{ - EncoderFactory: mockEncoderFactory, - Logger: logger.Test(t), - } - o := NewOCR3(cfg) - require.NoError(t, o.Start(ctx)) - - var p types.PluginProvider - var pr core.PipelineRunnerService - var tc core.TelemetryClient - var el core.ErrorLog - var kv core.KeyValueStore - var rs core.RelayerSet - r := mocks.NewCapabilitiesRegistry(t) - r.On("Add", mock.Anything, o.config.capability).Return(nil) - r.On("Remove", mock.Anything, o.config.capability.ID).Return(nil) - - factory, err := o.NewReportingPluginFactory(ctx, core.ReportingPluginServiceConfig{}, p, pr, tc, el, r, kv, rs) - require.NoError(t, err) - - require.NoError(t, factory.Start(ctx)) - - r.AssertCalled(t, "Add", mock.Anything, o.config.capability) - assert.NoError(t, factory.Ready()) - - err = o.Close() - require.NoError(t, err) - - r.AssertCalled(t, "Remove", mock.Anything, o.config.capability.ID) -} diff --git a/pkg/capabilities/consensus/ocr3/reporting_plugin.go b/pkg/capabilities/consensus/ocr3/reporting_plugin.go deleted file mode 100644 index cc0e2688a8..0000000000 --- a/pkg/capabilities/consensus/ocr3/reporting_plugin.go +++ /dev/null @@ -1,611 +0,0 @@ -package ocr3 - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "fmt" - "slices" - "time" - - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/timestamppb" - - "github.com/smartcontractkit/libocr/quorumhelper" - - "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/requests" - ocrcommon "github.com/smartcontractkit/libocr/commontypes" - "github.com/smartcontractkit/libocr/offchainreporting2/types" - "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types" - - "google.golang.org/protobuf/types/known/structpb" - - "github.com/smartcontractkit/chainlink-protos/cre/go/values" - "github.com/smartcontractkit/chainlink-protos/cre/go/values/pb" - - pbtypes "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/ocr3/types" - "github.com/smartcontractkit/chainlink-common/pkg/logger" -) - -var _ ocr3types.ReportingPlugin[[]byte] = (*reportingPlugin)(nil) - -type CapabilityIface interface { - GetAggregator(workflowID string) (pbtypes.Aggregator, error) - GetEncoderByWorkflowID(workflowID string) (pbtypes.Encoder, error) - GetEncoderByName(encoderName string, config *values.Map) (pbtypes.Encoder, error) - GetRegisteredWorkflowsIDs() []string - UnregisterWorkflowID(workflowID string) -} - -type reportingPlugin struct { - batchSize int - s *requests.Store[*ReportRequest] - r CapabilityIface - config ocr3types.ReportingPluginConfig - limits *pbtypes.ReportingPluginConfig - lggr logger.Logger -} - -func NewReportingPlugin(s *requests.Store[*ReportRequest], r CapabilityIface, batchSize int, config ocr3types.ReportingPluginConfig, - limits *pbtypes.ReportingPluginConfig, lggr logger.Logger) (*reportingPlugin, error) { - return &reportingPlugin{ - s: s, - r: r, - batchSize: batchSize, - config: config, - limits: limits, - lggr: logger.Named(lggr, "OCR3ConsensusReportingPlugin"), - }, nil -} - -func (r *reportingPlugin) Query(ctx context.Context, outctx ocr3types.OutcomeContext) (types.Query, error) { - batch, err := r.s.FirstN(r.batchSize) - - if err != nil { - r.lggr.Errorw("could not retrieve batch", "error", err) - return nil, err - } - - ids := []*pbtypes.Id{} - allExecutionIDs := []string{} - seenIds := make(map[idKey]bool) - cachedQuerySize := 0 - - for _, rq := range batch { - key := GetIDKey(rq) - newId := &pbtypes.Id{ - WorkflowExecutionId: rq.WorkflowExecutionID, - WorkflowId: rq.WorkflowID, - WorkflowOwner: rq.WorkflowOwner, - WorkflowName: rq.WorkflowName, - WorkflowDonId: rq.WorkflowDonID, - WorkflowDonConfigVersion: rq.WorkflowDonConfigVersion, - ReportId: rq.ReportID, - KeyId: rq.KeyID, - } - - // Simple duplicate elimination using a map - if seenIds[key] { - continue - } - - // If the new id would exceed the max query size, stop adding more ids - ok, newSize := QueryBatchHasCapacity(cachedQuerySize, newId, int(r.limits.MaxQueryLengthBytes)) - if !ok { - break - } - - seenIds[key] = true - ids = append(ids, newId) - allExecutionIDs = append(allExecutionIDs, rq.WorkflowExecutionID) - cachedQuerySize = newSize - } - - r.lggr.Debugw("Query complete", "len", len(ids), "allExecutionIDs", allExecutionIDs) - return proto.MarshalOptions{Deterministic: true}.Marshal(&pbtypes.Query{ - Ids: ids, - }) -} - -func (r *reportingPlugin) Observation(ctx context.Context, outctx ocr3types.OutcomeContext, query types.Query) (types.Observation, error) { - queryReq := &pbtypes.Query{} - err := proto.Unmarshal(query, queryReq) - if err != nil { - return nil, err - } - - weids := []string{} - for _, q := range queryReq.Ids { - if q == nil { - r.lggr.Debugw("skipping nil id for query", "query", queryReq) - continue - } - weids = append(weids, q.WorkflowExecutionId) - } - - reqs := r.s.GetByIDs(weids) - reqMap := map[string]*ReportRequest{} - for _, req := range reqs { - reqMap[req.WorkflowExecutionID] = req - } - - obs := &pbtypes.Observations{ - RegisteredWorkflowIds: r.r.GetRegisteredWorkflowsIDs(), - Timestamp: timestamppb.New(time.Now()), - } - allExecutionIDs := []string{} - - // Initialize cached size with the base message size (RegisteredWorkflowIds and Timestamp) - cachedObsSize := CalculateObservationsMessageSize(obs) - - for _, weid := range weids { - rq, ok := reqMap[weid] - if !ok { - r.lggr.Debugw("could not find local observations for weid requested in the query", "executionID", weid) - continue - } - - lggr := logger.With( - r.lggr, - "executionID", rq.WorkflowExecutionID, - "workflowID", rq.WorkflowID, - ) - - listProto := values.Proto(rq.Observations).GetListValue() - if listProto == nil { - lggr.Errorw("observations are not a list") - continue - } - - var cfgProto *pb.Map - if rq.OverriddenEncoderConfig != nil { - cp := values.Proto(rq.OverriddenEncoderConfig).GetMapValue() - cfgProto = cp - } - - newOb := &pbtypes.Observation{ - Observations: listProto, - Id: &pbtypes.Id{ - WorkflowExecutionId: rq.WorkflowExecutionID, - WorkflowId: rq.WorkflowID, - WorkflowOwner: rq.WorkflowOwner, - WorkflowName: rq.WorkflowName, - WorkflowDonId: rq.WorkflowDonID, - WorkflowDonConfigVersion: rq.WorkflowDonConfigVersion, - ReportId: rq.ReportID, - KeyId: rq.KeyID, - }, - OverriddenEncoderName: rq.OverriddenEncoderName, - OverriddenEncoderConfig: cfgProto, - } - - ok, newSize := ObservationsBatchHasCapacity(cachedObsSize, newOb, int(r.limits.MaxObservationLengthBytes)) - if !ok { - break - } - - obs.Observations = append(obs.Observations, newOb) - allExecutionIDs = append(allExecutionIDs, rq.WorkflowExecutionID) - cachedObsSize = newSize - } - - r.lggr.Debugw("Observation complete", "len", len(obs.Observations), "queryLen", len(queryReq.Ids), "allExecutionIDs", allExecutionIDs) - return proto.MarshalOptions{Deterministic: true}.Marshal(obs) -} - -func (r *reportingPlugin) ValidateObservation(ctx context.Context, outctx ocr3types.OutcomeContext, query types.Query, ao types.AttributedObservation) error { - return nil -} - -func (r *reportingPlugin) ObservationQuorum(ctx context.Context, outctx ocr3types.OutcomeContext, query types.Query, aos []types.AttributedObservation) (bool, error) { - return quorumhelper.ObservationCountReachesObservationQuorum(quorumhelper.QuorumTwoFPlusOne, r.config.N, r.config.F, aos), nil -} - -func shaForOverriddenEncoder(obs *pbtypes.Observation) (string, error) { - hash := sha256.New() - _, err := hash.Write([]byte(obs.OverriddenEncoderName)) - if err != nil { - return "", fmt.Errorf("could not write encoder name to hash: %w", err) - } - - marshalled, err := proto.MarshalOptions{Deterministic: true}.Marshal(obs.OverriddenEncoderConfig) - if err != nil { - return "", fmt.Errorf("could not marshal overridden encoder: %w", err) - } - - _, err = hash.Write(marshalled) - if err != nil { - return "", fmt.Errorf("could not write encoder config to hash: %w", err) - } - - return string(hash.Sum([]byte{})), nil -} - -type encoderConfig struct { - name string - config *pb.Map -} - -func (r *reportingPlugin) Outcome(ctx context.Context, outctx ocr3types.OutcomeContext, query types.Query, attributedObservations []types.AttributedObservation) (ocr3types.Outcome, error) { - // execution ID -> oracle ID -> list of observations - execIDToOracleObservations := map[string]map[ocrcommon.OracleID][]values.Value{} - seenWorkflowIDs := map[string]int{} - var sortedTimestamps []*timestamppb.Timestamp - var finalTimestamp *timestamppb.Timestamp - execIDToEncoderShaToCount := map[string]map[string]int{} - shaToEncoder := map[string]encoderConfig{} - for _, attributedObservation := range attributedObservations { - obs := &pbtypes.Observations{} - err := proto.Unmarshal(attributedObservation.Observation, obs) - if err != nil { - r.lggr.Errorw("could not unmarshal observation", "error", err, "observation", obs) - continue - } - - countedWorkflowIDs := map[string]bool{} - for _, id := range obs.RegisteredWorkflowIds { - // Skip if we've already counted this workflow ID. we want to avoid duplicates in the seen workflow IDs. - if _, ok := countedWorkflowIDs[id]; ok { - continue - } - - // Count how many times a workflow ID is seen from Observations, no need for initial value since it's 0 by default. - seenWorkflowIDs[id]++ - - countedWorkflowIDs[id] = true - } - - sortedTimestamps = append(sortedTimestamps, obs.Timestamp) - - seenExecutionIDs := map[string]bool{} - for _, request := range obs.Observations { - if request == nil { - r.lggr.Debugw("skipping nil request in observations", "observations", obs.Observations) - continue - } - - if request.Id == nil { - r.lggr.Debugw("skipping nil id in request", "request", request) - continue - } - - weid := request.Id.WorkflowExecutionId - if seenExecutionIDs[weid] { - r.lggr.Debugw("skipping duplicate workflow execution id in observation", "executionID", weid) - continue - } - seenExecutionIDs[weid] = true - - obsList, innerErr := values.FromListValueProto(request.Observations) - if obsList == nil || innerErr != nil { - r.lggr.Errorw("observations are not a list", "weID", weid, "oracleID", attributedObservation.Observer, "err", innerErr) - continue - } - - if _, ok := execIDToOracleObservations[weid]; !ok { - execIDToOracleObservations[weid] = make(map[ocrcommon.OracleID][]values.Value) - } - execIDToOracleObservations[weid][attributedObservation.Observer] = obsList.Underlying - - sha, err := shaForOverriddenEncoder(request) - if err != nil { - r.lggr.Errorw("could not calculate sha for overridden encoder", "error", err, "observation", obs) - continue - } - - shaToEncoder[sha] = encoderConfig{ - name: request.OverriddenEncoderName, - config: request.OverriddenEncoderConfig, - } - if _, ok := execIDToEncoderShaToCount[weid]; !ok { - execIDToEncoderShaToCount[weid] = map[string]int{} - } - execIDToEncoderShaToCount[weid][sha]++ - } - } - - // Since we will most likely get N different timestamps, each with frequency=1, we get the median instead of the mode. - slices.SortFunc(sortedTimestamps, func(a, b *timestamppb.Timestamp) int { - if a.AsTime().Before(b.AsTime()) { - return -1 - } - if a.AsTime().After(b.AsTime()) { - return 1 - } - return 0 - }) - timestampCount := len(sortedTimestamps) - mid := timestampCount / 2 - if timestampCount%2 == 1 { - finalTimestamp = sortedTimestamps[mid] - } else { - a := sortedTimestamps[mid-1].AsTime().Unix() - b := sortedTimestamps[mid].AsTime().Unix() - // a + (b-a) / 2 to avoid overflows - finalTimestamp = timestamppb.New(time.Unix(a+(b-a)/2, 0)) - } - - q := &pbtypes.Query{} - err := proto.Unmarshal(query, q) - if err != nil { - return nil, err - } - - previousOutcome := &pbtypes.Outcome{} - err = proto.Unmarshal(outctx.PreviousOutcome, previousOutcome) - if err != nil { - return nil, err - } - if previousOutcome.Outcomes == nil { - previousOutcome.Outcomes = map[string]*pbtypes.AggregationOutcome{} - } - - // Wipe out the CurrentReports. This gets regenerated - // every time since we only want to transmit reports that - // are part of the current Query. - previousOutcome.CurrentReports = []*pbtypes.Report{} - var allExecutionIDs []string - cachedReportSize := 0 - - for _, weid := range q.Ids { - if weid == nil { - r.lggr.Debugw("skipping nil id in query", "query", q) - continue - } - lggr := logger.With(r.lggr, "executionID", weid.WorkflowExecutionId, "workflowID", weid.WorkflowId) - obs, ok := execIDToOracleObservations[weid.WorkflowExecutionId] - if !ok { - lggr.Debugw("could not find any observations matching weid requested in the query") - continue - } - - workflowOutcome, ok := previousOutcome.Outcomes[weid.WorkflowId] - if !ok { - lggr.Debugw("could not find existing outcome for workflow, aggregator will create a new one") - } - - if len(obs) < (2*r.config.F + 1) { - lggr.Debugw("insufficient observations for workflow execution id") - continue - } - - agg, err2 := r.r.GetAggregator(weid.WorkflowId) - if err2 != nil { - lggr.Errorw("could not retrieve aggregator for workflow", "error", err2) - continue - } - - outcome, err2 := agg.Aggregate(lggr, workflowOutcome, obs, r.config.F) - if err2 != nil { - lggr.Errorw("error aggregating outcome", "error", err2) - continue - } - - // Only if the previous outcome exists: - // We carry the last seen round from the previous outcome, since the aggregation does carry it. - // So each `Aggregate()` call will return an outcome with a zero value for LastSeenAt. - if workflowOutcome != nil { - outcome.LastSeenAt = workflowOutcome.LastSeenAt - } - - outcome.Timestamp = finalTimestamp - - shaToCount, ok := execIDToEncoderShaToCount[weid.WorkflowExecutionId] - if !ok { - lggr.Debugw("could not find any encoder shas matching weid requested in the query") - continue - } - - // Note: no need to check the observation count here, - // we've checked this above when we checked the observations count. - var encCfg *encoderConfig - for sha, count := range shaToCount { - if count >= 2*r.config.F+1 { - encoderCfg, ok := shaToEncoder[sha] - if !ok { - lggr.Debugw("could not find encoder matching sha") - continue - } - - lggr.Debugw("consensus reached on overridden encoder", "encoderName", encoderCfg.name) - encCfg = &encoderCfg - break - } - } - - if encCfg != nil { - lggr.Debugw("overridden encoder set", "name", encCfg.name, "cfg", encCfg.config) - outcome.EncoderName = encCfg.name - outcome.EncoderConfig = encCfg.config - } - - report := &pbtypes.Report{ - Outcome: outcome, - Id: weid, - } - - ok, newSize := ReportBatchHasCapacity(cachedReportSize, report, int(r.limits.MaxOutcomeLengthBytes)) - if !ok { - break - } - - previousOutcome.CurrentReports = append(previousOutcome.CurrentReports, report) - allExecutionIDs = append(allExecutionIDs, weid.WorkflowExecutionId) - cachedReportSize = newSize - - previousOutcome.Outcomes[weid.WorkflowId] = outcome - } - - // We need to prune outcomes from previous workflows that are no longer relevant. - for workflowID, outcome := range previousOutcome.Outcomes { - // Update the last seen round for this outcome. But this should only happen if the workflow is seen by F+1 nodes. - if seenWorkflowIDs[workflowID] >= (r.config.F + 1) { - r.lggr.Debugw("updating last seen round of outcome for workflow", "workflowID", workflowID) - outcome.LastSeenAt = outctx.SeqNr - } else if outctx.SeqNr-outcome.LastSeenAt > r.limits.OutcomePruningThreshold { - r.lggr.Debugw("pruning outcome for workflow", "workflowID", workflowID, "SeqNr", outctx.SeqNr, "lastSeenAt", outcome.LastSeenAt) - delete(previousOutcome.Outcomes, workflowID) - r.r.UnregisterWorkflowID(workflowID) - } - } - - rawOutcome, err := proto.MarshalOptions{Deterministic: true}.Marshal(previousOutcome) - h := sha256.New() - h.Write(rawOutcome) - outcomeHash := h.Sum(nil) - r.lggr.Debugw("Outcome complete", "len", len(previousOutcome.Outcomes), "nAggregatedWorkflowExecutions", len(previousOutcome.CurrentReports), "allExecutionIDs", allExecutionIDs, "outcomeHash", hex.EncodeToString(outcomeHash), "err", err) - return rawOutcome, err -} - -func marshalReportInfo(info *pbtypes.ReportInfo, keyID string) ([]byte, error) { - p, err := proto.MarshalOptions{Deterministic: true}.Marshal(info) - if err != nil { - return nil, err - } - - infos, err := structpb.NewStruct(map[string]any{ - "keyBundleName": keyID, - "reportInfo": p, - }) - if err != nil { - return nil, err - } - - ip, err := proto.MarshalOptions{Deterministic: true}.Marshal(infos) - if err != nil { - return nil, err - } - - return ip, nil -} - -func (r *reportingPlugin) Reports(ctx context.Context, seqNr uint64, outcome ocr3types.Outcome) ([]ocr3types.ReportPlus[[]byte], error) { - o := &pbtypes.Outcome{} - err := proto.Unmarshal(outcome, o) - if err != nil { - return nil, err - } - - reports := []ocr3types.ReportPlus[[]byte]{} - - for _, report := range o.CurrentReports { - if report == nil { - r.lggr.Debugw("skipping nil report in outcome", "outcome", o) - continue - } - - if report.Id == nil { - r.lggr.Debugw("skipping report with nil id in outcome", "report", report) - continue - } - - if report.Outcome == nil { - r.lggr.Debugw("skipping report with nil outcome", "report", report) - continue - } - - lggr := logger.With( - r.lggr, - "workflowID", report.Id.WorkflowId, - "executionID", report.Id.WorkflowExecutionId, - "shouldReport", report.Outcome.ShouldReport, - ) - lggr.Debugw("generating reports", "len", len(o.CurrentReports)) - - outcome, id := report.Outcome, report.Id - - info := &pbtypes.ReportInfo{ - Id: id, - ShouldReport: outcome.ShouldReport, - } - - var rawReport []byte - if info.ShouldReport { - meta := &pbtypes.Metadata{ - Version: 1, - ExecutionID: id.WorkflowExecutionId, - Timestamp: uint32(outcome.Timestamp.AsTime().Unix()), - DONID: id.WorkflowDonId, - DONConfigVersion: id.WorkflowDonConfigVersion, - WorkflowID: id.WorkflowId, - WorkflowName: id.WorkflowName, - WorkflowOwner: id.WorkflowOwner, - ReportID: id.ReportId, - } - newOutcome, err := pbtypes.AppendMetadata(outcome, meta) - if err != nil { - lggr.Errorw("could not append IDs") - continue - } - - var encoder pbtypes.Encoder - if newOutcome.EncoderName != "" { - lggr.Debugw("using encoder from outcome", "encoderName", newOutcome.EncoderName, "executionID", report.Id.WorkflowExecutionId) - encoderConfig, err2 := values.FromMapValueProto(newOutcome.EncoderConfig) - if err2 != nil { - lggr.Errorw("could not convert desired encoder config to values.Map", "error", err2, "executionID", report.Id.WorkflowExecutionId) - } else { - encoder, err2 = r.r.GetEncoderByName(newOutcome.EncoderName, encoderConfig) - if err2 != nil { - lggr.Errorw("could not retrieve desired encoder, will use per-workflow default", "error", err2, "executionID", report.Id.WorkflowExecutionId) - } - } - } - - if encoder == nil { - encoder, err = r.r.GetEncoderByWorkflowID(id.WorkflowId) - if err != nil { - lggr.Errorw("could not retrieve encoder for workflow", "error", err) - continue - } - } - - mv, err := values.FromMapValueProto(newOutcome.EncodableOutcome) - if err != nil { - lggr.Errorw("could not decode map from map value proto", "error", err) - continue - } - - rawReport, err = encoder.Encode(ctx, *mv) - if err != nil { - if cerr := ctx.Err(); cerr != nil { - lggr.Errorw("report encoding cancelled", "err", cerr) - return nil, cerr - } - lggr.Errorw("could not encode report for workflow", "error", err) - continue - } - } - - infob, err := marshalReportInfo(info, id.KeyId) - if err != nil { - lggr.Errorw("could not marshal id into ReportWithInfo", "error", err) - continue - } - - // Append every report, even if shouldReport = false, to let the transmitter mark the step as complete. - reports = append(reports, ocr3types.ReportPlus[[]byte]{ - ReportWithInfo: ocr3types.ReportWithInfo[[]byte]{ - Report: rawReport, - Info: infob, - }, - }) - } - - r.lggr.Debugw("Reports complete", "len", len(reports)) - return reports, nil -} - -func (r *reportingPlugin) ShouldAcceptAttestedReport(ctx context.Context, seqNr uint64, rwi ocr3types.ReportWithInfo[[]byte]) (bool, error) { - // True because we always want to transmit a report, even if shouldReport = false. - return true, nil -} - -func (r *reportingPlugin) ShouldTransmitAcceptedReport(ctx context.Context, seqNr uint64, rwi ocr3types.ReportWithInfo[[]byte]) (bool, error) { - // True because we always want to transmit a report, even if shouldReport = false. - return true, nil -} - -func (r *reportingPlugin) Close() error { - return nil -} diff --git a/pkg/capabilities/consensus/ocr3/reporting_plugin_test.go b/pkg/capabilities/consensus/ocr3/reporting_plugin_test.go deleted file mode 100644 index 9a99b5a55c..0000000000 --- a/pkg/capabilities/consensus/ocr3/reporting_plugin_test.go +++ /dev/null @@ -1,1302 +0,0 @@ -package ocr3 - -import ( - "context" - "errors" - "slices" - "testing" - "time" - - "github.com/google/uuid" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/timestamppb" - - "github.com/smartcontractkit/libocr/commontypes" - "github.com/smartcontractkit/libocr/offchainreporting2/types" - "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types" - - pbtypes "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/ocr3/types" - "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/requests" - "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-protos/cre/go/values" - "github.com/smartcontractkit/chainlink-protos/cre/go/values/pb" -) - -func TestReportingPlugin_Query_ErrorInQueueCall(t *testing.T) { - ctx := t.Context() - lggr := logger.Test(t) - s := requests.NewStore[*ReportRequest]() - batchSize := 0 - rp, err := NewReportingPlugin(s, nil, batchSize, ocr3types.ReportingPluginConfig{}, defaultLimits(), lggr) - require.NoError(t, err) - - outcomeCtx := ocr3types.OutcomeContext{ - PreviousOutcome: []byte(""), - } - _, err = rp.Query(ctx, outcomeCtx) - assert.Error(t, err) -} - -func TestReportingPlugin_Query(t *testing.T) { - ctx := t.Context() - lggr := logger.Test(t) - s := requests.NewStore[*ReportRequest]() - rp, err := NewReportingPlugin(s, nil, defaultBatchSize, ocr3types.ReportingPluginConfig{}, defaultLimits(), lggr) - require.NoError(t, err) - - eid := uuid.New().String() - wowner := uuid.New().String() - - err = s.Add(&ReportRequest{ - WorkflowID: workflowTestID, - WorkflowExecutionID: eid, - WorkflowOwner: wowner, - WorkflowName: workflowTestName, - ReportID: reportTestID, - }) - require.NoError(t, err) - outcomeCtx := ocr3types.OutcomeContext{ - PreviousOutcome: []byte(""), - } - - q, err := rp.Query(ctx, outcomeCtx) - require.NoError(t, err) - - qry := &pbtypes.Query{} - err = proto.Unmarshal(q, qry) - require.NoError(t, err) - - assert.Len(t, qry.Ids, 1) - assert.Equal(t, workflowTestID, qry.Ids[0].WorkflowId) - assert.Equal(t, qry.Ids[0].WorkflowExecutionId, eid) -} - -type mockCapability struct { - t *testing.T - aggregator pbtypes.Aggregator - encoder *enc - registeredWorkflows map[string]bool - expectedEncoderName string -} - -type aggregator struct { - gotObs map[commontypes.OracleID][]values.Value - outcome *pbtypes.AggregationOutcome -} - -func (a *aggregator) Aggregate(lggr logger.Logger, pout *pbtypes.AggregationOutcome, observations map[commontypes.OracleID][]values.Value, _ int) (*pbtypes.AggregationOutcome, error) { - a.gotObs = observations - nm, err := values.NewMap( - map[string]any{ - "aggregated": "outcome", - }, - ) - if err != nil { - return nil, err - } - a.outcome = &pbtypes.AggregationOutcome{ - EncodableOutcome: values.Proto(nm).GetMapValue(), - } - return a.outcome, nil -} - -type erroringAggregator struct { - aggregator - count int -} - -func (a *erroringAggregator) Aggregate(lggr logger.Logger, pout *pbtypes.AggregationOutcome, observations map[commontypes.OracleID][]values.Value, i int) (*pbtypes.AggregationOutcome, error) { - defer func() { a.count += 1 }() - if a.count == 0 { - return nil, errors.New("failed to aggregate") - } - - return a.aggregator.Aggregate(lggr, pout, observations, i) -} - -type enc struct { - gotInput values.Map -} - -func (e *enc) Encode(ctx context.Context, input values.Map) ([]byte, error) { - e.gotInput = input - return proto.Marshal(values.Proto(&input)) -} - -func (mc *mockCapability) GetAggregator(workflowID string) (pbtypes.Aggregator, error) { - return mc.aggregator, nil -} - -func (mc *mockCapability) GetEncoderByWorkflowID(workflowID string) (pbtypes.Encoder, error) { - return mc.encoder, nil -} - -func (mc *mockCapability) GetEncoderByName(encoderName string, config *values.Map) (pbtypes.Encoder, error) { - require.Equal(mc.t, mc.expectedEncoderName, encoderName) - return mc.encoder, nil -} - -func (mc *mockCapability) GetRegisteredWorkflowsIDs() []string { - workflows := make([]string, 0, len(mc.registeredWorkflows)) - for wf := range mc.registeredWorkflows { - workflows = append(workflows, wf) - } - return workflows -} - -func (mc *mockCapability) UnregisterWorkflowID(workflowID string) { - delete(mc.registeredWorkflows, workflowID) -} - -func TestReportingPlugin_Observation(t *testing.T) { - ctx := t.Context() - lggr := logger.Test(t) - s := requests.NewStore[*ReportRequest]() - mcap := &mockCapability{ - aggregator: &aggregator{}, - encoder: &enc{}, - registeredWorkflows: map[string]bool{ - workflowTestID: true, - workflowTestID2: true, - }, - } - rp, err := NewReportingPlugin(s, mcap, defaultBatchSize, ocr3types.ReportingPluginConfig{}, defaultLimits(), lggr) - require.NoError(t, err) - - o, err := values.NewList([]any{"hello"}) - require.NoError(t, err) - - eid := uuid.New().String() - wowner := uuid.New().String() - err = s.Add(&ReportRequest{ - WorkflowID: workflowTestID, - WorkflowExecutionID: eid, - WorkflowOwner: wowner, - WorkflowName: workflowTestName, - ReportID: reportTestID, - Observations: o, - }) - require.NoError(t, err) - outcomeCtx := ocr3types.OutcomeContext{ - PreviousOutcome: []byte(""), - } - - q, err := rp.Query(ctx, outcomeCtx) - require.NoError(t, err) - - obs, err := rp.Observation(ctx, outcomeCtx, q) - require.NoError(t, err) - - obspb := &pbtypes.Observations{} - err = proto.Unmarshal(obs, obspb) - require.NoError(t, err) - - assert.Len(t, obspb.Observations, 1) - fo := obspb.Observations[0] - assert.Equal(t, fo.Id.WorkflowExecutionId, eid) - assert.Equal(t, workflowTestID, fo.Id.WorkflowId) - lvp, err := values.FromListValueProto(fo.Observations) - require.NoError(t, err) - assert.Equal(t, o, lvp) - expected := []string{workflowTestID, workflowTestID2} - actual := obspb.RegisteredWorkflowIds - slices.Sort(actual) - slices.Sort(expected) - assert.Equal(t, expected, actual) -} - -func TestReportingPlugin_Observation_NilIds(t *testing.T) { - ctx := t.Context() - lggr := logger.Test(t) - s := requests.NewStore[*ReportRequest]() - mcap := &mockCapability{ - aggregator: &aggregator{}, - encoder: &enc{}, - registeredWorkflows: map[string]bool{ - workflowTestID: true, - workflowTestID2: true, - }, - } - rp, err := NewReportingPlugin(s, mcap, defaultBatchSize, ocr3types.ReportingPluginConfig{}, defaultLimits(), lggr) - require.NoError(t, err) - - outcomeCtx := ocr3types.OutcomeContext{ - PreviousOutcome: []byte(""), - } - - q := &pbtypes.Query{ - Ids: []*pbtypes.Id{ - nil, - { - WorkflowExecutionId: uuid.New().String(), - }, - }, - } - qb, err := proto.Marshal(q) - require.NoError(t, err) - - _, err = rp.Observation(ctx, outcomeCtx, qb) - require.NoError(t, err) -} - -func TestReportingPlugin_Observation_NoResults(t *testing.T) { - ctx := t.Context() - lggr := logger.Test(t) - s := requests.NewStore[*ReportRequest]() - mcap := &mockCapability{ - aggregator: &aggregator{}, - encoder: &enc{}, - } - rp, err := NewReportingPlugin(s, mcap, defaultBatchSize, ocr3types.ReportingPluginConfig{}, defaultLimits(), lggr) - require.NoError(t, err) - - outcomeCtx := ocr3types.OutcomeContext{ - PreviousOutcome: []byte(""), - } - - q, err := rp.Query(ctx, outcomeCtx) - require.NoError(t, err) - - obs, err := rp.Observation(ctx, outcomeCtx, q) - require.NoError(t, err) - - obspb := &pbtypes.Observations{} - err = proto.Unmarshal(obs, obspb) - require.NoError(t, err) - - assert.Empty(t, obspb.Observations) -} - -func TestReportingPlugin_Outcome(t *testing.T) { - lggr := logger.Test(t) - s := requests.NewStore[*ReportRequest]() - aggregator := &aggregator{} - mcap := &mockCapability{ - aggregator: aggregator, - encoder: &enc{}, - } - rp, err := NewReportingPlugin(s, mcap, defaultBatchSize, ocr3types.ReportingPluginConfig{}, defaultLimits(), lggr) - require.NoError(t, err) - - weid := uuid.New().String() - wowner := uuid.New().String() - id := &pbtypes.Id{ - WorkflowExecutionId: weid, - WorkflowId: workflowTestID, - WorkflowOwner: wowner, - WorkflowName: workflowTestName, - ReportId: reportTestID, - } - q := &pbtypes.Query{ - Ids: []*pbtypes.Id{id}, - } - qb, err := proto.Marshal(q) - require.NoError(t, err) - o, err := values.NewList([]any{"hello"}) - require.NoError(t, err) - obs := &pbtypes.Observations{ - Observations: []*pbtypes.Observation{ - { - Id: id, - Observations: values.Proto(o).GetListValue(), - }, - }, - } - - rawObs, err := proto.Marshal(obs) - require.NoError(t, err) - aos := []types.AttributedObservation{ - { - Observation: rawObs, - Observer: commontypes.OracleID(1), - }, - } - - outcome, err := rp.Outcome(t.Context(), ocr3types.OutcomeContext{}, qb, aos) - require.NoError(t, err) - - opb := &pbtypes.Outcome{} - err = proto.Unmarshal(outcome, opb) - require.NoError(t, err) - - assert.Len(t, opb.CurrentReports, 1) - - cr := opb.CurrentReports[0] - assert.EqualExportedValues(t, cr.Id, id) - assert.EqualExportedValues(t, cr.Outcome, aggregator.outcome) - assert.EqualExportedValues(t, opb.Outcomes[workflowTestID], aggregator.outcome) -} - -func TestReportingPlugin_Outcome_AggregatorErrorDoesntInterruptOtherWorkflows(t *testing.T) { - lggr := logger.Test(t) - s := requests.NewStore[*ReportRequest]() - aggregator := &erroringAggregator{} - mcap := &mockCapability{ - aggregator: aggregator, - encoder: &enc{}, - } - rp, err := NewReportingPlugin(s, mcap, defaultBatchSize, ocr3types.ReportingPluginConfig{}, defaultLimits(), lggr) - require.NoError(t, err) - - weid := uuid.New().String() - wowner := uuid.New().String() - id := &pbtypes.Id{ - WorkflowExecutionId: weid, - WorkflowId: workflowTestID, - WorkflowOwner: wowner, - WorkflowName: workflowTestName, - ReportId: reportTestID, - } - weid2 := uuid.New().String() - id2 := &pbtypes.Id{ - WorkflowExecutionId: weid2, - WorkflowId: workflowTestID, - WorkflowOwner: wowner, - WorkflowName: workflowTestName, - ReportId: reportTestID, - } - q := &pbtypes.Query{ - Ids: []*pbtypes.Id{id, id2}, - } - qb, err := proto.Marshal(q) - require.NoError(t, err) - o, err := values.NewList([]any{"hello"}) - require.NoError(t, err) - - o2, err := values.NewList([]any{"world"}) - require.NoError(t, err) - obs := &pbtypes.Observations{ - Observations: []*pbtypes.Observation{ - { - Id: id, - Observations: values.Proto(o).GetListValue(), - }, - { - Id: id2, - Observations: values.Proto(o2).GetListValue(), - }, - }, - } - - rawObs, err := proto.Marshal(obs) - require.NoError(t, err) - aos := []types.AttributedObservation{ - { - Observation: rawObs, - Observer: commontypes.OracleID(1), - }, - } - - outcome, err := rp.Outcome(t.Context(), ocr3types.OutcomeContext{}, qb, aos) - require.NoError(t, err) - - opb := &pbtypes.Outcome{} - err = proto.Unmarshal(outcome, opb) - require.NoError(t, err) - - assert.Len(t, opb.CurrentReports, 1) - - cr := opb.CurrentReports[0] - assert.EqualExportedValues(t, cr.Id, id2) - assert.EqualExportedValues(t, cr.Outcome, aggregator.outcome) - assert.EqualExportedValues(t, opb.Outcomes[workflowTestID], aggregator.outcome) -} - -func TestReportingPlugin_Outcome_NilDerefs(t *testing.T) { - ctx := t.Context() - lggr := logger.Test(t) - s := requests.NewStore[*ReportRequest]() - mcap := &mockCapability{ - aggregator: &aggregator{}, - encoder: &enc{}, - } - rp, err := NewReportingPlugin(s, mcap, defaultBatchSize, ocr3types.ReportingPluginConfig{}, defaultLimits(), lggr) - require.NoError(t, err) - - weid := uuid.New().String() - wowner := uuid.New().String() - id := &pbtypes.Id{ - WorkflowExecutionId: weid, - WorkflowId: workflowTestID, - WorkflowOwner: wowner, - WorkflowName: workflowTestName, - ReportId: reportTestID, - } - q := &pbtypes.Query{ - Ids: []*pbtypes.Id{ - id, - nil, - }, - } - qb, err := proto.Marshal(q) - require.NoError(t, err) - aos := []types.AttributedObservation{ - { - Observer: commontypes.OracleID(1), - }, - {}, - } - - _, err = rp.Outcome(ctx, ocr3types.OutcomeContext{}, qb, aos) - require.NoError(t, err) - - obs := &pbtypes.Observations{ - Observations: []*pbtypes.Observation{ - nil, - {}, - }, - RegisteredWorkflowIds: nil, - } - obsb, err := proto.Marshal(obs) - require.NoError(t, err) - - aos = []types.AttributedObservation{ - { - Observation: obsb, - Observer: commontypes.OracleID(1), - }, - } - _, err = rp.Outcome(ctx, ocr3types.OutcomeContext{}, qb, aos) - require.NoError(t, err) -} - -func TestReportingPlugin_Outcome_AggregatorErrorDoesntInterruptOtherIDs(t *testing.T) { - ctx := t.Context() - lggr := logger.Test(t) - s := requests.NewStore[*ReportRequest]() - mcap := &mockCapability{ - aggregator: &aggregator{}, - encoder: &enc{}, - } - rp, err := NewReportingPlugin(s, mcap, defaultBatchSize, ocr3types.ReportingPluginConfig{}, defaultLimits(), lggr) - require.NoError(t, err) - - weid := uuid.New().String() - wowner := uuid.New().String() - id1 := &pbtypes.Id{ - WorkflowExecutionId: weid, - WorkflowId: workflowTestID, - WorkflowOwner: wowner, - WorkflowName: workflowTestName, - ReportId: reportTestID, - } - - weid2 := uuid.New().String() - id2 := &pbtypes.Id{ - WorkflowExecutionId: weid2, - WorkflowId: workflowTestID, - WorkflowOwner: wowner, - WorkflowName: workflowTestName, - ReportId: reportTestID, - } - q := &pbtypes.Query{ - Ids: []*pbtypes.Id{ - id1, - id2, - }, - } - qb, err := proto.Marshal(q) - require.NoError(t, err) - aos := []types.AttributedObservation{ - { - Observer: commontypes.OracleID(1), - }, - {}, - } - - _, err = rp.Outcome(ctx, ocr3types.OutcomeContext{}, qb, aos) - require.NoError(t, err) - - obs := &pbtypes.Observations{ - Observations: []*pbtypes.Observation{ - nil, - {}, - }, - RegisteredWorkflowIds: nil, - } - obsb, err := proto.Marshal(obs) - require.NoError(t, err) - - aos = []types.AttributedObservation{ - { - Observation: obsb, - Observer: commontypes.OracleID(1), - }, - } - _, err = rp.Outcome(ctx, ocr3types.OutcomeContext{}, qb, aos) - require.NoError(t, err) -} - -func TestReportingPlugin_Reports_ShouldReportFalse(t *testing.T) { - lggr := logger.Test(t) - s := requests.NewStore[*ReportRequest]() - mcap := &mockCapability{ - aggregator: &aggregator{}, - encoder: &enc{}, - } - rp, err := NewReportingPlugin(s, mcap, defaultBatchSize, ocr3types.ReportingPluginConfig{}, defaultLimits(), lggr) - require.NoError(t, err) - - var sqNr uint64 - weid := uuid.New().String() - wowner := uuid.New().String() - id := &pbtypes.Id{ - WorkflowExecutionId: weid, - WorkflowId: workflowTestID, - WorkflowOwner: wowner, - } - nm, err := values.NewMap( - map[string]any{ - "our": "aggregation", - }, - ) - require.NoError(t, err) - outcome := &pbtypes.Outcome{ - CurrentReports: []*pbtypes.Report{ - { - Id: id, - Outcome: &pbtypes.AggregationOutcome{ - EncodableOutcome: values.Proto(nm).GetMapValue(), - }, - }, - }, - } - pl, err := proto.Marshal(outcome) - require.NoError(t, err) - reports, err := rp.Reports(t.Context(), sqNr, pl) - require.NoError(t, err) - - assert.Len(t, reports, 1) - gotRep := reports[0] - assert.Empty(t, gotRep.ReportWithInfo.Report) - - ib := gotRep.ReportWithInfo.Info - info, err := extractReportInfo(ib) - require.NoError(t, err) - - assert.EqualExportedValues(t, id, info.Id) - assert.False(t, info.ShouldReport) - - require.Nil(t, gotRep.TransmissionScheduleOverride) -} - -func TestReportingPlugin_Reports_NilDerefs(t *testing.T) { - ctx := t.Context() - lggr := logger.Test(t) - s := requests.NewStore[*ReportRequest]() - mcap := &mockCapability{ - aggregator: &aggregator{}, - encoder: &enc{}, - } - rp, err := NewReportingPlugin(s, mcap, defaultBatchSize, ocr3types.ReportingPluginConfig{}, defaultLimits(), lggr) - require.NoError(t, err) - - var sqNr uint64 - weid := uuid.New().String() - wowner := uuid.New().String() - id := &pbtypes.Id{ - WorkflowExecutionId: weid, - WorkflowId: workflowTestID, - WorkflowOwner: wowner, - } - require.NoError(t, err) - outcome := &pbtypes.Outcome{ - CurrentReports: []*pbtypes.Report{ - { - Id: id, - Outcome: &pbtypes.AggregationOutcome{ - EncodableOutcome: nil, - }, - }, - {}, - { - Outcome: &pbtypes.AggregationOutcome{}, - }, - { - Id: id, - }, - }, - } - pl, err := proto.Marshal(outcome) - require.NoError(t, err) - _, err = rp.Reports(ctx, sqNr, pl) - require.NoError(t, err) -} - -func TestReportingPlugin_Reports_ShouldReportTrue(t *testing.T) { - lggr := logger.Test(t) - dynamicEncoderName := "special_encoder" - s := requests.NewStore[*ReportRequest]() - mcap := &mockCapability{ - t: t, - aggregator: &aggregator{}, - encoder: &enc{}, - expectedEncoderName: dynamicEncoderName, - } - rp, err := NewReportingPlugin(s, mcap, defaultBatchSize, ocr3types.ReportingPluginConfig{}, defaultLimits(), lggr) - require.NoError(t, err) - - var sqNr uint64 - weid := uuid.New().String() - wowner := uuid.New().String() - donID := uint32(1) - id := &pbtypes.Id{ - WorkflowExecutionId: weid, - WorkflowId: workflowTestID, - WorkflowOwner: wowner, - WorkflowName: workflowTestName, - ReportId: reportTestID, - WorkflowDonId: donID, - } - nm, err := values.NewMap( - map[string]any{ - "our": "aggregation", - }, - ) - nmp := values.Proto(nm).GetMapValue() - require.NoError(t, err) - outcome := &pbtypes.Outcome{ - CurrentReports: []*pbtypes.Report{ - { - Id: id, - Outcome: &pbtypes.AggregationOutcome{ - EncodableOutcome: nmp, - ShouldReport: true, - EncoderName: dynamicEncoderName, - }, - }, - }, - } - pl, err := proto.Marshal(outcome) - require.NoError(t, err) - reports, err := rp.Reports(t.Context(), sqNr, pl) - require.NoError(t, err) - - assert.Len(t, reports, 1) - gotRep := reports[0] - - rep := &pb.Value{} - err = proto.Unmarshal(gotRep.ReportWithInfo.Report, rep) - require.NoError(t, err) - - // The workflow ID and execution ID get added to the report. - nm.Underlying[pbtypes.MetadataFieldName], err = values.NewMap(map[string]any{ - "Version": 1, - "ExecutionID": weid, - "Timestamp": 0, - "DONID": donID, - "DONConfigVersion": 0, - "WorkflowID": workflowTestID, - "WorkflowName": workflowTestName, - "WorkflowOwner": wowner, - "ReportID": reportTestID, - }) - require.NoError(t, err) - fp, err := values.FromProto(rep) - require.NoError(t, err) - require.Equal(t, nm, fp) - - ib := gotRep.ReportWithInfo.Info - info, err := extractReportInfo(ib) - require.NoError(t, err) - - assert.EqualExportedValues(t, info.Id, id) - assert.True(t, info.ShouldReport) - - require.Nil(t, gotRep.TransmissionScheduleOverride) -} - -func TestReportingPlugin_Outcome_ShouldPruneOldOutcomes(t *testing.T) { - ctx := t.Context() - lggr := logger.Test(t) - s := requests.NewStore[*ReportRequest]() - mcap := &mockCapability{ - aggregator: &aggregator{}, - encoder: &enc{}, - registeredWorkflows: map[string]bool{ - workflowTestID: true, - workflowTestID2: true, - }, - } - rp, err := NewReportingPlugin(s, mcap, defaultBatchSize, ocr3types.ReportingPluginConfig{}, defaultLimits(), lggr) - require.NoError(t, err) - - weid := uuid.New().String() - wowner := uuid.New().String() - id := &pbtypes.Id{ - WorkflowExecutionId: weid, - WorkflowId: workflowTestID, - WorkflowOwner: wowner, - WorkflowName: workflowTestName, - ReportId: reportTestID, - } - id2 := &pbtypes.Id{ - WorkflowExecutionId: weid, - WorkflowId: workflowTestID2, - WorkflowOwner: wowner, - WorkflowName: workflowTestName, - ReportId: reportTestID, - } - id3 := &pbtypes.Id{ - WorkflowExecutionId: weid, - WorkflowId: workflowTestID3, - WorkflowOwner: wowner, - WorkflowName: workflowTestName, - ReportId: reportTestID, - } - q := &pbtypes.Query{ - Ids: []*pbtypes.Id{id, id2, id3}, - } - qb, err := proto.Marshal(q) - require.NoError(t, err) - o, err := values.NewList([]any{"hello"}) - require.NoError(t, err) - obs := &pbtypes.Observations{ - Observations: []*pbtypes.Observation{ - { - Id: id, - Observations: values.Proto(o).GetListValue(), - }, - { - Id: id2, - Observations: values.Proto(o).GetListValue(), - }, - { - Id: id3, - Observations: values.Proto(o).GetListValue(), - }, - }, - RegisteredWorkflowIds: []string{workflowTestID, workflowTestID2}, - } - obs2 := &pbtypes.Observations{ - Observations: []*pbtypes.Observation{ - { - Id: id, - Observations: values.Proto(o).GetListValue(), - }, - { - Id: id2, - Observations: values.Proto(o).GetListValue(), - }, - { - Id: id3, - Observations: values.Proto(o).GetListValue(), - }, - }, - RegisteredWorkflowIds: []string{workflowTestID}, - } - - rawObs, err := proto.Marshal(obs) - require.NoError(t, err) - rawObs2, err := proto.Marshal(obs2) - require.NoError(t, err) - aos := []types.AttributedObservation{ - { - Observation: rawObs, - Observer: commontypes.OracleID(1), - }, - } - aos2 := []types.AttributedObservation{ - { - Observation: rawObs2, - Observer: commontypes.OracleID(1), - }, - } - - outcome1, err := rp.Outcome(ctx, ocr3types.OutcomeContext{SeqNr: 100}, qb, aos) - require.NoError(t, err) - opb1 := &pbtypes.Outcome{} - err = proto.Unmarshal(outcome1, opb1) - require.NoError(t, err) - - outcome2, err := rp.Outcome(ctx, ocr3types.OutcomeContext{SeqNr: defaultOutcomePruningThreshold + 100, PreviousOutcome: outcome1}, qb, aos2) - require.NoError(t, err) - opb2 := &pbtypes.Outcome{} - err = proto.Unmarshal(outcome2, opb2) - require.NoError(t, err) - - assert.Equal(t, uint64(100), opb1.Outcomes[workflowTestID].LastSeenAt) - assert.Equal(t, uint64(100), opb1.Outcomes[workflowTestID2].LastSeenAt) - assert.Equal(t, uint64(0), opb1.Outcomes[workflowTestID3].LastSeenAt) - assert.Equal(t, uint64(defaultOutcomePruningThreshold+100), opb2.Outcomes[workflowTestID].LastSeenAt) - assert.Equal(t, uint64(100), opb2.Outcomes[workflowTestID2].LastSeenAt) - assert.Zero(t, opb2.Outcomes[workflowTestID3]) // This outcome was pruned -} - -func TestReportPlugin_Outcome_ShouldReturnMedianTimestamp(t *testing.T) { - ctx := t.Context() - lggr := logger.Test(t) - s := requests.NewStore[*ReportRequest]() - mcap := &mockCapability{ - aggregator: &aggregator{}, - encoder: &enc{}, - registeredWorkflows: map[string]bool{ - workflowTestID: true, - workflowTestID2: true, - }, - } - rp, err := NewReportingPlugin(s, mcap, defaultBatchSize, ocr3types.ReportingPluginConfig{}, defaultLimits(), lggr) - require.NoError(t, err) - - weid := uuid.New().String() - wowner := uuid.New().String() - id := &pbtypes.Id{ - WorkflowExecutionId: weid, - WorkflowId: workflowTestID, - WorkflowOwner: wowner, - WorkflowName: workflowTestName, - ReportId: reportTestID, - } - id2 := &pbtypes.Id{ - WorkflowExecutionId: weid, - WorkflowId: workflowTestID2, - WorkflowOwner: wowner, - WorkflowName: workflowTestName, - ReportId: reportTestID, - } - id3 := &pbtypes.Id{ - WorkflowExecutionId: weid, - WorkflowId: workflowTestID3, - WorkflowOwner: wowner, - WorkflowName: workflowTestName, - ReportId: reportTestID, - } - q := &pbtypes.Query{ - Ids: []*pbtypes.Id{id, id2, id3}, - } - qb, err := proto.Marshal(q) - require.NoError(t, err) - o, err := values.NewList([]any{"hello"}) - require.NoError(t, err) - time1 := time.Now().Add(time.Second * 1) - time2 := time.Now().Add(time.Second * 2) - time3 := time.Now().Add(time.Second * 3) - obs := &pbtypes.Observations{ - Observations: []*pbtypes.Observation{ - { - Id: id, - Observations: values.Proto(o).GetListValue(), - }, - { - Id: id2, - Observations: values.Proto(o).GetListValue(), - }, - { - Id: id3, - Observations: values.Proto(o).GetListValue(), - }, - }, - RegisteredWorkflowIds: []string{workflowTestID, workflowTestID2}, - Timestamp: timestamppb.New(time1), - } - obs2 := &pbtypes.Observations{ - Observations: []*pbtypes.Observation{ - { - Id: id, - Observations: values.Proto(o).GetListValue(), - }, - { - Id: id2, - Observations: values.Proto(o).GetListValue(), - }, - { - Id: id3, - Observations: values.Proto(o).GetListValue(), - }, - }, - RegisteredWorkflowIds: []string{workflowTestID}, - Timestamp: timestamppb.New(time2), - } - obs3 := &pbtypes.Observations{ - Observations: []*pbtypes.Observation{ - { - Id: id, - Observations: values.Proto(o).GetListValue(), - }, - { - Id: id2, - Observations: values.Proto(o).GetListValue(), - }, - { - Id: id3, - Observations: values.Proto(o).GetListValue(), - }, - }, - RegisteredWorkflowIds: []string{workflowTestID}, - Timestamp: timestamppb.New(time3), - } - - rawObs, err := proto.Marshal(obs) - require.NoError(t, err) - rawObs2, err := proto.Marshal(obs2) - require.NoError(t, err) - rawObs3, err := proto.Marshal(obs3) - require.NoError(t, err) - aos := []types.AttributedObservation{ - { - Observation: rawObs, - Observer: commontypes.OracleID(1), - }, - { - Observation: rawObs2, - Observer: commontypes.OracleID(2), - }, - { - Observation: rawObs3, - Observer: commontypes.OracleID(3), - }, - } - - outcome, err := rp.Outcome(ctx, ocr3types.OutcomeContext{SeqNr: 100}, qb, aos) - require.NoError(t, err) - opb1 := &pbtypes.Outcome{} - err = proto.Unmarshal(outcome, opb1) - require.NoError(t, err) - - assert.Equal(t, timestamppb.New(time2), opb1.Outcomes[workflowTestID].Timestamp) -} - -func TestReportPlugin_Outcome_ShouldReturnOverriddenEncoder(t *testing.T) { - lggr := logger.Test(t) - s := requests.NewStore[*ReportRequest]() - mcap := &mockCapability{ - aggregator: &aggregator{}, - encoder: &enc{}, - registeredWorkflows: map[string]bool{ - workflowTestID: true, - workflowTestID2: true, - }, - } - rp, err := NewReportingPlugin(s, mcap, defaultBatchSize, ocr3types.ReportingPluginConfig{F: 1}, defaultLimits(), lggr) - require.NoError(t, err) - - wowner := uuid.New().String() - id := &pbtypes.Id{ - WorkflowExecutionId: uuid.New().String(), - WorkflowId: workflowTestID, - WorkflowOwner: wowner, - WorkflowName: workflowTestName, - ReportId: reportTestID, - } - id2 := &pbtypes.Id{ - WorkflowExecutionId: uuid.New().String(), - WorkflowId: workflowTestID2, - WorkflowOwner: wowner, - WorkflowName: workflowTestName, - ReportId: reportTestID, - } - id3 := &pbtypes.Id{ - WorkflowExecutionId: uuid.New().String(), - WorkflowId: workflowTestID3, - WorkflowOwner: wowner, - WorkflowName: workflowTestName, - ReportId: reportTestID, - } - q := &pbtypes.Query{ - Ids: []*pbtypes.Id{id, id2, id3}, - } - qb, err := proto.Marshal(q) - require.NoError(t, err) - o, err := values.NewList([]any{"hello"}) - require.NoError(t, err) - time1 := time.Now().Add(time.Second * 1) - time2 := time.Now().Add(time.Second * 2) - time3 := time.Now().Add(time.Second * 3) - m, err := values.NewMap(map[string]any{"foo": "bar"}) - require.NoError(t, err) - mc := values.ProtoMap(m) - obs := &pbtypes.Observations{ - Observations: []*pbtypes.Observation{ - { - Id: id, - Observations: values.Proto(o).GetListValue(), - OverriddenEncoderName: "evm", - OverriddenEncoderConfig: mc, - }, - { - Id: id2, - Observations: values.Proto(o).GetListValue(), - OverriddenEncoderName: "evm", - OverriddenEncoderConfig: mc, - }, - { - Id: id3, - Observations: values.Proto(o).GetListValue(), - }, - }, - RegisteredWorkflowIds: []string{workflowTestID, workflowTestID2}, - Timestamp: timestamppb.New(time1), - } - obs2 := &pbtypes.Observations{ - Observations: []*pbtypes.Observation{ - { - Id: id, - Observations: values.Proto(o).GetListValue(), - OverriddenEncoderName: "evm", - OverriddenEncoderConfig: mc, - }, - { - Id: id2, - Observations: values.Proto(o).GetListValue(), - OverriddenEncoderName: "evm", - OverriddenEncoderConfig: mc, - }, - { - Id: id3, - Observations: values.Proto(o).GetListValue(), - }, - }, - RegisteredWorkflowIds: []string{workflowTestID}, - Timestamp: timestamppb.New(time2), - } - obs3 := &pbtypes.Observations{ - Observations: []*pbtypes.Observation{ - { - Id: id, - Observations: values.Proto(o).GetListValue(), - OverriddenEncoderName: "evm", - OverriddenEncoderConfig: mc, - }, - { - Id: id2, - Observations: values.Proto(o).GetListValue(), - OverriddenEncoderName: "solana", - OverriddenEncoderConfig: mc, - }, - { - Id: id3, - Observations: values.Proto(o).GetListValue(), - }, - }, - RegisteredWorkflowIds: []string{workflowTestID}, - Timestamp: timestamppb.New(time3), - } - - rawObs, err := proto.Marshal(obs) - require.NoError(t, err) - rawObs2, err := proto.Marshal(obs2) - require.NoError(t, err) - rawObs3, err := proto.Marshal(obs3) - require.NoError(t, err) - aos := []types.AttributedObservation{ - { - Observation: rawObs, - Observer: commontypes.OracleID(1), - }, - { - Observation: rawObs2, - Observer: commontypes.OracleID(2), - }, - { - Observation: rawObs3, - Observer: commontypes.OracleID(3), - }, - } - - outcome, err := rp.Outcome(t.Context(), ocr3types.OutcomeContext{SeqNr: 100}, qb, aos) - require.NoError(t, err) - opb1 := &pbtypes.Outcome{} - err = proto.Unmarshal(outcome, opb1) - require.NoError(t, err) - - assert.Equal(t, "evm", opb1.Outcomes[workflowTestID].EncoderName) - ec, err := values.FromMapValueProto(opb1.Outcomes[workflowTestID].EncoderConfig) - require.NoError(t, err) - assert.Equal(t, ec, m) - - // No consensus on outcome 2 - assert.Empty(t, opb1.Outcomes[workflowTestID2].EncoderName) - assert.Nil(t, opb1.Outcomes[workflowTestID2].EncoderConfig) - - // Outcome 3 doesn't set the encoder - assert.Empty(t, opb1.Outcomes[workflowTestID3].EncoderName) - assert.Nil(t, opb1.Outcomes[workflowTestID3].EncoderConfig) -} - -func TestDuplicateEliminationLogic(t *testing.T) { - // This test specifically addresses the condition `if seenIds[key]` in the Query phase - // to verify that duplicate IDs are properly handled and don't affect size calculations - - // Helper function to create a ReportRequest for testing GetIDKey - createReportRequest := func(workflowExecutionId, workflowId, reportId string) *ReportRequest { - return &ReportRequest{ - WorkflowExecutionID: workflowExecutionId, - WorkflowID: workflowId, - WorkflowOwner: "owner", - WorkflowName: "test", - WorkflowDonID: 123, - WorkflowDonConfigVersion: 456, - ReportID: reportId, - KeyID: "key-1", - } - } - - // Helper function to create a pbtypes.Id from ReportRequest - createIdFromRequest := func(rq *ReportRequest) *pbtypes.Id { - return &pbtypes.Id{ - WorkflowExecutionId: rq.WorkflowExecutionID, - WorkflowId: rq.WorkflowID, - WorkflowOwner: rq.WorkflowOwner, - WorkflowName: rq.WorkflowName, - WorkflowDonId: rq.WorkflowDonID, - WorkflowDonConfigVersion: rq.WorkflowDonConfigVersion, - ReportId: rq.ReportID, - KeyId: rq.KeyID, - } - } - - t.Run("duplicate keys are properly detected", func(t *testing.T) { - // Create two ReportRequests that should generate the same key - rq1 := createReportRequest("exec-1", "workflow-1", "report-1") - rq2 := createReportRequest("exec-1", "workflow-1", "report-1") // Same as rq1 - - key1 := GetIDKey(rq1) - key2 := GetIDKey(rq2) - - if key1 != key2 { - t.Errorf("Expected identical keys for identical requests, got %+v != %+v", key1, key2) - } - }) - - t.Run("different keys are properly distinguished", func(t *testing.T) { - // Create two ReportRequests that should generate different keys - rq1 := createReportRequest("exec-1", "workflow-1", "report-1") - rq2 := createReportRequest("exec-2", "workflow-1", "report-1") // Different execution ID - - key1 := GetIDKey(rq1) - key2 := GetIDKey(rq2) - - if key1 == key2 { - t.Errorf("Expected different keys for different requests, got %+v == %+v", key1, key2) - } - }) - - t.Run("duplicate elimination in query batching simulation", func(t *testing.T) { - // Simulate the Query phase logic with duplicates - seenIds := make(map[idKey]bool) - var ids []*pbtypes.Id - var allExecutionIDs []string - cachedQuerySize := 0 - sizeLimit := 1000 - - // Create a batch with duplicates - batch := []*ReportRequest{ - createReportRequest("exec-1", "workflow-1", "report-1"), - createReportRequest("exec-2", "workflow-1", "report-1"), - createReportRequest("exec-1", "workflow-1", "report-1"), // Duplicate of first - createReportRequest("exec-3", "workflow-1", "report-1"), - createReportRequest("exec-2", "workflow-1", "report-1"), // Duplicate of second - } - - // Simulate the logic from reporting_plugin.go Query method - for _, rq := range batch { - key := GetIDKey(rq) - newId := createIdFromRequest(rq) - - // This is the condition we're specifically testing - if seenIds[key] { - continue // Skip duplicates - } - - // Check size limit (this should only be called for non-duplicates) - canAdd, newSize := QueryBatchHasCapacity(cachedQuerySize, newId, sizeLimit) - if !canAdd { - break - } - - seenIds[key] = true - ids = append(ids, newId) - allExecutionIDs = append(allExecutionIDs, rq.WorkflowExecutionID) - cachedQuerySize = newSize - } - - // Verify results - expectedUniqueIds := 3 // exec-1, exec-2, exec-3 - if len(ids) != expectedUniqueIds { - t.Errorf("Expected %d unique IDs, got %d", expectedUniqueIds, len(ids)) - } - - if len(allExecutionIDs) != expectedUniqueIds { - t.Errorf("Expected %d unique execution IDs, got %d", expectedUniqueIds, len(allExecutionIDs)) - } - - if len(seenIds) != expectedUniqueIds { - t.Errorf("Expected %d entries in seenIds map, got %d", expectedUniqueIds, len(seenIds)) - } - - // Verify that the correct execution IDs are present - expectedExecutionIDs := map[string]bool{ - "exec-1": true, - "exec-2": true, - "exec-3": true, - } - - for _, execID := range allExecutionIDs { - if !expectedExecutionIDs[execID] { - t.Errorf("Unexpected execution ID in results: %s", execID) - } - delete(expectedExecutionIDs, execID) - } - - if len(expectedExecutionIDs) > 0 { - t.Errorf("Missing expected execution IDs: %v", expectedExecutionIDs) - } - - // Verify that size calculation was only done for unique items - // Calculate expected size manually - expectedSize := 0 - for _, id := range ids { - idSize := calculateIdSize(id) - if idSize > 0 { - tagSize := varintSize(uint64(1<<3 | 2)) - lengthSize := varintSize(uint64(idSize)) - expectedSize += tagSize + lengthSize + idSize - } - } - - if cachedQuerySize != expectedSize { - t.Errorf("Expected cached size %d, got %d", expectedSize, cachedQuerySize) - } - }) - - t.Run("seenIds map prevents processing of duplicates", func(t *testing.T) { - // Test the exact condition: if seenIds[key] { continue } - seenIds := make(map[idKey]bool) - - rq := createReportRequest("exec-1", "workflow-1", "report-1") - key := GetIDKey(rq) - - // Initially, key should not be in seenIds - if seenIds[key] { - t.Error("Key should not be in seenIds initially") - } - - // Add key to seenIds - seenIds[key] = true - - // Now the condition should be true - if !seenIds[key] { - t.Error("Key should be in seenIds after adding") - } - - // Test with a different key - rq2 := createReportRequest("exec-2", "workflow-1", "report-1") - key2 := GetIDKey(rq2) - - // This key should not be in seenIds - if seenIds[key2] { - t.Error("Different key should not be in seenIds") - } - }) -} - -func defaultLimits() *pbtypes.ReportingPluginConfig { - return &pbtypes.ReportingPluginConfig{ - OutcomePruningThreshold: defaultOutcomePruningThreshold, - MaxQueryLengthBytes: defaultMaxPhaseOutputBytes, - MaxObservationLengthBytes: defaultMaxPhaseOutputBytes, - MaxOutcomeLengthBytes: defaultMaxPhaseOutputBytes, - } -} diff --git a/pkg/capabilities/consensus/ocr3/testdata/fixtures/capability/schema.json b/pkg/capabilities/consensus/ocr3/testdata/fixtures/capability/schema.json deleted file mode 100644 index 31c20ed685..0000000000 --- a/pkg/capabilities/consensus/ocr3/testdata/fixtures/capability/schema.json +++ /dev/null @@ -1,140 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/smartcontractkit/chainlink/capabilities/offchain_reporting@1.0.0/root", - "properties": { - "config": { - "properties": { - "aggregation_method": { - "type": "string", - "enum": [ - "data_feeds", - "llo_streams", - "identical", - "reduce", - "secure_mint" - ] - }, - "aggregation_config": { - "properties": { - "Underlying": { - "type": "object" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "Underlying" - ] - }, - "encoder": { - "type": "string" - }, - "encoder_config": { - "properties": { - "Underlying": { - "type": "object" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "Underlying" - ] - }, - "report_id": { - "type": "string", - "pattern": "^[a-f0-9]{4}$" - }, - "request_timeout_ms": { - "type": "integer" - }, - "key_id": { - "type": "string" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "aggregation_method", - "aggregation_config", - "encoder", - "encoder_config", - "report_id", - "request_timeout_ms", - "key_id" - ] - }, - "inputs": { - "properties": { - "observations": { - "properties": { - "Underlying": { - "items": true, - "type": "array" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "Underlying" - ] - }, - "encoder": { - "type": "string" - }, - "encoder_config": { - "properties": { - "Underlying": { - "type": "object" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "Underlying" - ] - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "observations" - ] - }, - "outputs": { - "properties": { - "WorkflowExecutionID": { - "type": "string" - }, - "Value": { - "properties": { - "Underlying": { - "type": "object" - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "Underlying" - ] - }, - "Err": true - }, - "additionalProperties": false, - "type": "object", - "required": [ - "WorkflowExecutionID", - "Value", - "Err" - ] - } - }, - "additionalProperties": false, - "type": "object", - "required": [ - "config", - "inputs", - "outputs" - ], - "description": "OCR3 consensus exposed as a capability." -} \ No newline at end of file diff --git a/pkg/capabilities/consensus/ocr3/testdata/fixtures/capability/test.yaml b/pkg/capabilities/consensus/ocr3/testdata/fixtures/capability/test.yaml deleted file mode 100644 index 6f2212f659..0000000000 --- a/pkg/capabilities/consensus/ocr3/testdata/fixtures/capability/test.yaml +++ /dev/null @@ -1,21 +0,0 @@ -config: - aggregation_method: data_feeds - aggregation_config: - Underlying: {} - - encoder: "" - encoder_config: - Underlying: {} - -inputs: - observations: - Underlying: - - - -outputs: - WorkflowExecutionID: "" - Value: - Err: - - -# yaml-language-server: $schema=./schema.json diff --git a/pkg/capabilities/consensus/ocr3/transmitter.go b/pkg/capabilities/consensus/ocr3/transmitter.go deleted file mode 100644 index 31d82c5bae..0000000000 --- a/pkg/capabilities/consensus/ocr3/transmitter.go +++ /dev/null @@ -1,159 +0,0 @@ -package ocr3 - -import ( - "context" - "encoding/base64" - "encoding/binary" - "encoding/hex" - "errors" - "fmt" - "strconv" - - "google.golang.org/protobuf/proto" - - "github.com/smartcontractkit/libocr/offchainreporting2/types" - "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types" - - "github.com/smartcontractkit/chainlink-common/pkg/capabilities" - pbtypes "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/ocr3/types" - "github.com/smartcontractkit/chainlink-common/pkg/custmsg" - "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/types/core" - "github.com/smartcontractkit/chainlink-protos/cre/go/values" - - "google.golang.org/protobuf/types/known/structpb" -) - -var _ ocr3types.ContractTransmitter[[]byte] = (*ContractTransmitter)(nil) - -// ContractTransmitter is a custom transmitter for the OCR3 capability. -// When called it will forward the report + its signatures back to the -// OCR3 capability by making a call to Execute with a special "method" -// parameter. -type ContractTransmitter struct { - lggr logger.Logger - registry core.CapabilitiesRegistry - capability capabilities.ExecutableCapability - fromAccount string - emitter custmsg.MessageEmitter -} - -func extractReportInfo(data []byte) (*pbtypes.ReportInfo, error) { - info := &structpb.Struct{} - err := proto.Unmarshal(data, info) - if err != nil { - return nil, err - } - - im := info.AsMap() - ri, ok := im["reportInfo"] - if !ok { - return nil, errors.New("could not fetch reportInfo from structpb") - } - - ris, ok := ri.(string) - if !ok { - return nil, errors.New("reportInfo is not bytes") - } - - rib, err := base64.StdEncoding.DecodeString(ris) - if err != nil { - return nil, err - } - - reportInfo := &pbtypes.ReportInfo{} - err = proto.Unmarshal(rib, reportInfo) - return reportInfo, err -} - -func (c *ContractTransmitter) Transmit(ctx context.Context, configDigest types.ConfigDigest, seqNr uint64, rwi ocr3types.ReportWithInfo[[]byte], signatures []types.AttributedOnchainSignature) error { - info, err := extractReportInfo(rwi.Info) - if err != nil { - c.lggr.Error("could not unmarshal info") - return err - } - - signedReport := &pbtypes.SignedReport{} - if info.ShouldReport { - signedReport.Report = rwi.Report - - // report context is the config digest + the sequence number padded with zeros - // (see OCR3OnchainKeyringAdapter in core) - seqToEpoch := make([]byte, 32) - binary.BigEndian.PutUint32(seqToEpoch[32-5:32-1], uint32(seqNr)) - zeros := make([]byte, 32) - repContext := append(append(configDigest[:], seqToEpoch[:]...), zeros...) - signedReport.Context = repContext - - var sigs [][]byte - for _, s := range signatures { - sigs = append(sigs, s.Signature) - } - signedReport.Signatures = sigs - reportIDBytes, err2 := hex.DecodeString(info.Id.ReportId) - if err2 != nil { - return fmt.Errorf("could not decode report id: %w", err2) - } - signedReport.ID = reportIDBytes - c.lggr.Debugw("ContractTransmitter added signatures and context", "nSignatures", len(sigs), "contextLen", len(repContext)) - } - - resp := map[string]any{ - methodHeader: methodSendResponse, - transmissionHeader: signedReport, - terminateHeader: !info.ShouldReport, - } - inputs, err := values.Wrap(resp) - if err != nil { - c.lggr.Error("could not wrap report", "payload", resp) - return err - } - - c.lggr.Debugw("ContractTransmitter transmitting", "shouldReport", info.ShouldReport, "len", len(rwi.Report)) - if c.capability == nil { - cp, innerErr := c.registry.Get(ctx, ocrCapabilityID) - if innerErr != nil { - return fmt.Errorf("failed to fetch ocr3 capability from registry: %w", innerErr) - } - - c.capability = cp.(capabilities.ExecutableCapability) - } - - msg := "report with id " + info.Id.ReportId + " should be reported: " + strconv.FormatBool(info.ShouldReport) - err = c.emitter.With( - "workflowExecutionID", info.Id.WorkflowExecutionId, - "workflowID", info.Id.WorkflowId, - "workflowOwner", info.Id.WorkflowOwner, - "workflowName", info.Id.WorkflowName, - "reportId", info.Id.ReportId, - ).Emit(ctx, msg) - if err != nil { - c.lggr.Errorw("could not emit message: "+msg, "error", err) - } - - _, err = c.capability.Execute(ctx, capabilities.CapabilityRequest{ - Metadata: capabilities.RequestMetadata{ - WorkflowExecutionID: info.Id.WorkflowExecutionId, - WorkflowID: info.Id.WorkflowId, - WorkflowDonID: info.Id.WorkflowDonId, - }, - Inputs: inputs.(*values.Map), - }) - if err != nil { - c.lggr.Errorw("could not transmit response", "error", err, "weid", info.Id.WorkflowExecutionId) - } - c.lggr.Debugw("ContractTransmitter transmitting done", "shouldReport", info.ShouldReport, "len", len(rwi.Report)) - return err -} - -func (c *ContractTransmitter) FromAccount(_ context.Context) (types.Account, error) { - return types.Account(c.fromAccount), nil -} - -func (c *ContractTransmitter) SetCapability(capability capabilities.ExecutableCapability) { - c.capability = capability -} - -func NewContractTransmitter(lggr logger.Logger, registry core.CapabilitiesRegistry, fromAccount string) *ContractTransmitter { - return &ContractTransmitter{lggr: lggr, registry: registry, fromAccount: fromAccount, emitter: custmsg.NewLabeler()} -} diff --git a/pkg/capabilities/consensus/ocr3/transmitter_test.go b/pkg/capabilities/consensus/ocr3/transmitter_test.go deleted file mode 100644 index 3952e71533..0000000000 --- a/pkg/capabilities/consensus/ocr3/transmitter_test.go +++ /dev/null @@ -1,194 +0,0 @@ -package ocr3 - -import ( - "encoding/hex" - "testing" - "time" - - "github.com/google/uuid" - "github.com/jonboulle/clockwork" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - "google.golang.org/protobuf/proto" - - "github.com/smartcontractkit/libocr/offchainreporting2/types" - "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types" - - "github.com/smartcontractkit/chainlink-common/pkg/capabilities" - pbtypes "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/ocr3/types" - "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/requests" - "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" - "github.com/smartcontractkit/chainlink-common/pkg/types/core/mocks" - "github.com/smartcontractkit/chainlink-protos/cre/go/values" -) - -func TestTransmitter(t *testing.T) { - wid := "consensus-workflow-test-id-1" - wowner := "foo-owner" - repID := []byte{0xf0, 0xe0} - ctx := t.Context() - lggr := logger.Test(t) - s := requests.NewStore[*ReportRequest]() - - weid := uuid.New().String() - - cp := NewCapability( - s, - clockwork.NewFakeClock(), - 10*time.Second, - mockAggregatorFactory, - func(_ string, _ *values.Map, _ logger.Logger) (pbtypes.Encoder, error) { - return &encoder{}, nil - }, - lggr, - 10, - ) - servicetest.Run(t, cp) - - payload, err := values.NewMap(map[string]any{"observations": []string{"something happened"}}) - require.NoError(t, err) - config, err := values.NewMap(map[string]any{ - "aggregation_method": "data_feeds", - "aggregation_config": map[string]any{}, - "encoder": "", - "encoder_config": map[string]any{}, - "report_id": hex.EncodeToString(repID), - "key_id": "evm", - }) - require.NoError(t, err) - - gotCh := executeAsync(ctx, capabilities.CapabilityRequest{ - Metadata: capabilities.RequestMetadata{ - WorkflowExecutionID: weid, - WorkflowID: wid, - }, - Config: config, - Inputs: payload, - }, cp.Execute) - - require.NoError(t, err) - - r := mocks.NewCapabilitiesRegistry(t) - r.On("Get", mock.Anything, ocrCapabilityID).Return(cp, nil) - - info := &pbtypes.ReportInfo{ - Id: &pbtypes.Id{ - WorkflowExecutionId: weid, - WorkflowId: wid, - WorkflowOwner: wowner, - ReportId: hex.EncodeToString(repID), - }, - ShouldReport: true, - } - infob, err := marshalReportInfo(info, "evm") - require.NoError(t, err) - - sp := values.Proto(values.NewString("hello")) - spb, err := proto.Marshal(sp) - require.NoError(t, err) - rep := ocr3types.ReportWithInfo[[]byte]{ - Info: infob, - Report: spb, - } - - transmitter := NewContractTransmitter(lggr, r, "fromAccountString") - - var sqNr uint64 - sigs := []types.AttributedOnchainSignature{ - {Signature: []byte("a-signature")}, - } - err = transmitter.Transmit(ctx, types.ConfigDigest{}, sqNr, rep, sigs) - require.NoError(t, err) - - resp := <-gotCh - assert.NoError(t, resp.Err) - - signedReport := pbtypes.SignedReport{} - require.NoError(t, resp.Value.UnwrapTo(&signedReport)) - - assert.Equal(t, spb, signedReport.Report) - assert.Len(t, signedReport.Signatures, 1) - assert.Len(t, signedReport.Context, 96) - assert.Equal(t, repID, signedReport.ID) -} - -func TestTransmitter_ShouldReportFalse(t *testing.T) { - wid := "consensus-workflow-test-id-1" - wowner := "foo-owner" - ctx := t.Context() - lggr := logger.Test(t) - s := requests.NewStore[*ReportRequest]() - - weid := uuid.New().String() - - cp := NewCapability( - s, - clockwork.NewFakeClock(), - 10*time.Second, - mockAggregatorFactory, - func(_ string, _ *values.Map, _ logger.Logger) (pbtypes.Encoder, error) { - return &encoder{}, nil - }, - lggr, - 10, - ) - servicetest.Run(t, cp) - - payload, err := values.NewMap(map[string]any{"observations": []string{"something happened"}}) - require.NoError(t, err) - config, err := values.NewMap(map[string]any{ - "aggregation_method": "data_feeds", - "aggregation_config": map[string]any{}, - "encoder": "", - "encoder_config": map[string]any{}, - "report_id": "aaff", - "key_id": "evm", - }) - require.NoError(t, err) - - gotCh := executeAsync(ctx, capabilities.CapabilityRequest{ - Metadata: capabilities.RequestMetadata{ - WorkflowExecutionID: weid, - WorkflowID: wid, - }, - Inputs: payload, - Config: config, - }, cp.Execute) - - r := mocks.NewCapabilitiesRegistry(t) - r.On("Get", mock.Anything, ocrCapabilityID).Return(cp, nil) - - info := &pbtypes.ReportInfo{ - Id: &pbtypes.Id{ - WorkflowExecutionId: weid, - WorkflowId: wid, - WorkflowOwner: wowner, - }, - ShouldReport: false, - } - infob, err := marshalReportInfo(info, "evm") - require.NoError(t, err) - - sp := values.Proto(values.NewString("hello")) - spb, err := proto.Marshal(sp) - require.NoError(t, err) - rep := ocr3types.ReportWithInfo[[]byte]{ - Info: infob, - Report: spb, - } - - transmitter := NewContractTransmitter(lggr, r, "fromAccountString") - - var sqNr uint64 - sigs := []types.AttributedOnchainSignature{ - {Signature: []byte("a-signature")}, - } - err = transmitter.Transmit(ctx, types.ConfigDigest{}, sqNr, rep, sigs) - require.NoError(t, err) - - resp := <-gotCh - assert.Error(t, resp.Err) - assert.ErrorIs(t, resp.Err, capabilities.ErrStopExecution) -} diff --git a/pkg/capabilities/consensus/ocr3/validation_service.go b/pkg/capabilities/consensus/ocr3/validation_service.go deleted file mode 100644 index 3a7a5806fe..0000000000 --- a/pkg/capabilities/consensus/ocr3/validation_service.go +++ /dev/null @@ -1,38 +0,0 @@ -package ocr3 - -import ( - "context" - - "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/services" - "github.com/smartcontractkit/chainlink-common/pkg/types/core" -) - -var _ core.ValidationService = (*validationService)(nil) - -type validationService struct { - lggr logger.Logger - services.StateMachine -} - -func (v *validationService) ValidateConfig(ctx context.Context, config map[string]any) error { - return nil -} - -func (v *validationService) Start(ctx context.Context) error { - return v.StartOnce("OCR3ReportingPluginValidation", func() error { - return nil - }) -} - -func (v *validationService) Close() error { - return v.StopOnce("OCR3ReportingPluginValidation", func() error { - return nil - }) -} - -func (v *validationService) Name() string { return v.lggr.Name() } - -func (v *validationService) HealthReport() map[string]error { - return map[string]error{v.Name(): v.Healthy()} -} diff --git a/pkg/capabilities/consensus/ocr3/value_map_encoder.go b/pkg/capabilities/consensus/ocr3/value_map_encoder.go deleted file mode 100644 index 1e96dc6f71..0000000000 --- a/pkg/capabilities/consensus/ocr3/value_map_encoder.go +++ /dev/null @@ -1,19 +0,0 @@ -package ocr3 - -import ( - "context" - - "google.golang.org/protobuf/proto" - - "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/ocr3/types" - "github.com/smartcontractkit/chainlink-protos/cre/go/values" -) - -type ValueMapEncoder struct{} - -func (v ValueMapEncoder) Encode(_ context.Context, input values.Map) ([]byte, error) { - opts := proto.MarshalOptions{Deterministic: true} - return opts.Marshal(values.Proto(&input)) -} - -var _ types.Encoder = (*ValueMapEncoder)(nil) diff --git a/pkg/capabilities/consensus/ocr3/value_map_encoder_test.go b/pkg/capabilities/consensus/ocr3/value_map_encoder_test.go deleted file mode 100644 index 3e13f0ab9b..0000000000 --- a/pkg/capabilities/consensus/ocr3/value_map_encoder_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package ocr3_test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "google.golang.org/protobuf/proto" - - "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/ocr3" - "github.com/smartcontractkit/chainlink-protos/cre/go/values" - "github.com/smartcontractkit/chainlink-protos/cre/go/values/pb" -) - -func Test_ValuesEncoder_Encode(t *testing.T) { - t.Parallel() - input := map[string]any{ - "foo": "bar", - "baz": int64(42), - "x": map[string]any{"y": "z"}, - } - inputWrapped, err := values.NewMap(input) - require.NoError(t, err) - - expectedProto := &pb.Value{ - Value: &pb.Value_MapValue{ - MapValue: &pb.Map{ - Fields: map[string]*pb.Value{ - "foo": {Value: &pb.Value_StringValue{StringValue: "bar"}}, - "baz": {Value: &pb.Value_Int64Value{Int64Value: 42}}, - "x": { - Value: &pb.Value_MapValue{ - MapValue: &pb.Map{ - Fields: map[string]*pb.Value{ - "y": {Value: &pb.Value_StringValue{StringValue: "z"}}, - }, - }, - }, - }, - }, - }, - }, - } - - encoder := ocr3.ValueMapEncoder{} - actual, err := encoder.Encode(t.Context(), *inputWrapped) - require.NoError(t, err) - - opts := proto.MarshalOptions{Deterministic: true} - expected, err := opts.Marshal(expectedProto) - require.NoError(t, err) - - assert.Equal(t, expected, actual) - - decoded := &pb.Value{} - require.NoError(t, proto.Unmarshal(actual, decoded)) - - val, err := values.FromProto(decoded) - require.NoError(t, err) - - output := map[string]any{} - require.NoError(t, val.UnwrapTo(&output)) - - assert.Equal(t, input, output) -} diff --git a/pkg/capabilities/consensus/ocr3/report_request.go b/pkg/capabilities/consensus/requests/fixtures_test.go similarity index 67% rename from pkg/capabilities/consensus/ocr3/report_request.go rename to pkg/capabilities/consensus/requests/fixtures_test.go index f70cfd6907..fa86fa6d9b 100644 --- a/pkg/capabilities/consensus/ocr3/report_request.go +++ b/pkg/capabilities/consensus/requests/fixtures_test.go @@ -1,24 +1,26 @@ -package ocr3 +package requests_test import ( "context" "fmt" "time" - "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-protos/cre/go/values" ) -type ReportRequest struct { - Observations *values.List `mapstructure:"-"` +// testReportRequest/testReportResponse stand in for a real ConsensusRequest/ConsensusResponse +// implementation (e.g. ocr3.ReportRequest/ReportResponse) so the generic Store/Handler can be +// exercised without depending on a specific capability. +type testReportRequest struct { + Observations *values.List OverriddenEncoderName string OverriddenEncoderConfig *values.Map ExpiresAt time.Time // CallbackCh is a channel to send a response back to the requester // after the request has been processed or timed out. - CallbackCh chan ReportResponse - StopCh services.StopChan + CallbackCh chan testReportResponse + StopCh chan struct{} WorkflowExecutionID string WorkflowID string @@ -31,15 +33,15 @@ type ReportRequest struct { KeyID string } -func (r *ReportRequest) ID() string { +func (r *testReportRequest) ID() string { return r.WorkflowExecutionID } -func (r *ReportRequest) ExpiryTime() time.Time { +func (r *testReportRequest) ExpiryTime() time.Time { return r.ExpiresAt } -func (r *ReportRequest) SendResponse(ctx context.Context, resp ReportResponse) { +func (r *testReportRequest) SendResponse(ctx context.Context, resp testReportResponse) { select { case <-ctx.Done(): return @@ -48,16 +50,15 @@ func (r *ReportRequest) SendResponse(ctx context.Context, resp ReportResponse) { } } -func (r *ReportRequest) SendTimeout(ctx context.Context) { - timeoutResponse := ReportResponse{ +func (r *testReportRequest) SendTimeout(ctx context.Context) { + r.SendResponse(ctx, testReportResponse{ WorkflowExecutionID: r.WorkflowExecutionID, Err: fmt.Errorf("timeout exceeded: could not process request before expiry, workflowExecutionID %s", r.WorkflowExecutionID), - } - r.SendResponse(ctx, timeoutResponse) + }) } -func (r *ReportRequest) Copy() *ReportRequest { - return &ReportRequest{ +func (r *testReportRequest) Copy() *testReportRequest { + return &testReportRequest{ Observations: r.Observations.CopyList(), OverriddenEncoderConfig: r.OverriddenEncoderConfig.CopyMap(), @@ -79,12 +80,12 @@ func (r *ReportRequest) Copy() *ReportRequest { } } -type ReportResponse struct { +type testReportResponse struct { WorkflowExecutionID string Value *values.Map Err error } -func (r ReportResponse) RequestID() string { +func (r testReportResponse) RequestID() string { return r.WorkflowExecutionID } diff --git a/pkg/capabilities/consensus/requests/handler_test.go b/pkg/capabilities/consensus/requests/handler_test.go index 1e3de0a7df..77d9f46fb7 100644 --- a/pkg/capabilities/consensus/requests/handler_test.go +++ b/pkg/capabilities/consensus/requests/handler_test.go @@ -8,7 +8,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/ocr3" "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/requests" "github.com/smartcontractkit/chainlink-common/pkg/logger" @@ -21,11 +20,11 @@ func Test_Handler_SendsResponse(t *testing.T) { lggr := logger.Test(t) ctx := t.Context() - h := requests.NewHandler(lggr, requests.NewStore[*ocr3.ReportRequest](), clockwork.NewFakeClockAt(time.Now()), 1*time.Second) + h := requests.NewHandler(lggr, requests.NewStore[*testReportRequest](), clockwork.NewFakeClockAt(time.Now()), 1*time.Second) servicetest.Run(t, h) - responseCh := make(chan ocr3.ReportResponse, 10) - h.SendRequest(ctx, &ocr3.ReportRequest{ + responseCh := make(chan testReportResponse, 10) + h.SendRequest(ctx, &testReportRequest{ WorkflowExecutionID: "test", CallbackCh: responseCh, ExpiresAt: time.Now().Add(1 * time.Hour), @@ -34,7 +33,7 @@ func Test_Handler_SendsResponse(t *testing.T) { testVal, err := values.NewMap(map[string]any{"result": "testval"}) require.NoError(t, err) - h.SendResponse(ctx, ocr3.ReportResponse{ + h.SendResponse(ctx, testReportResponse{ WorkflowExecutionID: "test", Value: testVal, Err: nil, @@ -48,19 +47,19 @@ func Test_Handler_SendsResponseToLateRequest(t *testing.T) { lggr := logger.Test(t) ctx := t.Context() - h := requests.NewHandler(lggr, requests.NewStore[*ocr3.ReportRequest](), clockwork.NewFakeClockAt(time.Now()), 1*time.Second) + h := requests.NewHandler(lggr, requests.NewStore[*testReportRequest](), clockwork.NewFakeClockAt(time.Now()), 1*time.Second) servicetest.Run(t, h) testVal, err := values.NewMap(map[string]any{"result": "testval"}) require.NoError(t, err) - h.SendResponse(ctx, ocr3.ReportResponse{ + h.SendResponse(ctx, testReportResponse{ WorkflowExecutionID: "test", Value: testVal, Err: nil, }) - responseCh := make(chan ocr3.ReportResponse, 10) - h.SendRequest(ctx, &ocr3.ReportRequest{ + responseCh := make(chan testReportResponse, 10) + h.SendRequest(ctx, &testReportRequest{ WorkflowExecutionID: "test", CallbackCh: responseCh, ExpiresAt: time.Now().Add(1 * time.Hour), @@ -74,20 +73,20 @@ func Test_Handler_SendsResponseToLateRequestOnlyOnce(t *testing.T) { lggr := logger.Test(t) ctx := t.Context() - h := requests.NewHandler(lggr, requests.NewStore[*ocr3.ReportRequest](), clockwork.NewFakeClockAt(time.Now()), 1*time.Second) + h := requests.NewHandler(lggr, requests.NewStore[*testReportRequest](), clockwork.NewFakeClockAt(time.Now()), 1*time.Second) servicetest.Run(t, h) testVal, err := values.NewMap(map[string]any{"result": "testval"}) require.NoError(t, err) - h.SendResponse(ctx, ocr3.ReportResponse{ + h.SendResponse(ctx, testReportResponse{ WorkflowExecutionID: "test", Value: testVal, Err: nil, }) - responseCh := make(chan ocr3.ReportResponse, 10) - h.SendRequest(ctx, &ocr3.ReportRequest{ + responseCh := make(chan testReportResponse, 10) + h.SendRequest(ctx, &testReportRequest{ WorkflowExecutionID: "test", CallbackCh: responseCh, ExpiresAt: time.Now().Add(1 * time.Hour), @@ -98,8 +97,8 @@ func Test_Handler_SendsResponseToLateRequestOnlyOnce(t *testing.T) { resp := <-responseCh require.Equal(t, testVal, resp.Value) - responseCh = make(chan ocr3.ReportResponse, 10) - h.SendRequest(ctx, &ocr3.ReportRequest{ + responseCh = make(chan testReportResponse, 10) + h.SendRequest(ctx, &testReportRequest{ WorkflowExecutionID: "test", CallbackCh: responseCh, ExpiresAt: time.Now().Add(1 * time.Hour), @@ -117,11 +116,11 @@ func Test_Handler_PendingRequestsExpiry(t *testing.T) { lggr := logger.Test(t) clock := clockwork.NewFakeClockAt(time.Now()) - h := requests.NewHandler(lggr, requests.NewStore[*ocr3.ReportRequest](), clock, 1*time.Second) + h := requests.NewHandler(lggr, requests.NewStore[*testReportRequest](), clock, 1*time.Second) servicetest.Run(t, h) - responseCh := make(chan ocr3.ReportResponse, 10) - h.SendRequest(ctx, &ocr3.ReportRequest{ + responseCh := make(chan testReportResponse, 10) + h.SendRequest(ctx, &testReportRequest{ WorkflowExecutionID: "test", CallbackCh: responseCh, ExpiresAt: time.Now().Add(1 * time.Second), diff --git a/pkg/capabilities/consensus/requests/store_stats_test.go b/pkg/capabilities/consensus/requests/store_stats_test.go index 6908c4aad0..9f5cbae41b 100644 --- a/pkg/capabilities/consensus/requests/store_stats_test.go +++ b/pkg/capabilities/consensus/requests/store_stats_test.go @@ -7,7 +7,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/ocr3" "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/requests" ) @@ -23,9 +22,9 @@ func TestOCR3Store_Stats(t *testing.T) { // Create a new store with stats collector statsCollector := &testStatsCollector{} - s := requests.NewStoreWithStatsCollector[*ocr3.ReportRequest](statsCollector) + s := requests.NewStoreWithStatsCollector[*testReportRequest](statsCollector) rid := uuid.New().String() - req := &ocr3.ReportRequest{ + req := &testReportRequest{ WorkflowExecutionID: rid, } diff --git a/pkg/capabilities/consensus/requests/store_test.go b/pkg/capabilities/consensus/requests/store_test.go index c730b84426..72c3bab224 100644 --- a/pkg/capabilities/consensus/requests/store_test.go +++ b/pkg/capabilities/consensus/requests/store_test.go @@ -9,7 +9,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/ocr3" "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/requests" "github.com/smartcontractkit/chainlink-protos/cre/go/values" ) @@ -17,9 +16,9 @@ import ( func TestOCR3Store(t *testing.T) { n := time.Now() - s := requests.NewStore[*ocr3.ReportRequest]() + s := requests.NewStore[*testReportRequest]() rid := uuid.New().String() - req := &ocr3.ReportRequest{ + req := &testReportRequest{ WorkflowExecutionID: rid, ExpiresAt: n.Add(10 * time.Second), } @@ -61,7 +60,7 @@ func TestOCR3Store(t *testing.T) { t.Run("firstN, batchSize larger than queue", func(t *testing.T) { for range 10 { - err := s.Add(&ocr3.ReportRequest{WorkflowExecutionID: uuid.New().String(), ExpiresAt: n.Add(1 * time.Hour)}) + err := s.Add(&testReportRequest{WorkflowExecutionID: uuid.New().String(), ExpiresAt: n.Add(1 * time.Hour)}) require.NoError(t, err) } items, err := s.FirstN(100) @@ -76,7 +75,7 @@ func TestOCR3Store(t *testing.T) { }) t.Run("rangeN", func(t *testing.T) { - err := s.Add(&ocr3.ReportRequest{WorkflowExecutionID: uuid.New().String(), ExpiresAt: n.Add(1 * time.Hour)}) + err := s.Add(&testReportRequest{WorkflowExecutionID: uuid.New().String(), ExpiresAt: n.Add(1 * time.Hour)}) require.NoError(t, err) r, err := s.RangeN(0, 1) assert.NoError(t, err) @@ -90,7 +89,7 @@ func TestOCR3Store(t *testing.T) { t.Run("rangeN, batchSize larger than queue with start offset", func(t *testing.T) { for range 10 { - err := s.Add(&ocr3.ReportRequest{WorkflowExecutionID: uuid.New().String(), ExpiresAt: n.Add(1 * time.Hour)}) + err := s.Add(&testReportRequest{WorkflowExecutionID: uuid.New().String(), ExpiresAt: n.Add(1 * time.Hour)}) require.NoError(t, err) } items, err := s.RangeN(5, 100) @@ -108,9 +107,9 @@ func TestOCR3Store(t *testing.T) { } func TestOCR3Store_ManagesStateConsistently(t *testing.T) { - s := requests.NewStore[*ocr3.ReportRequest]() + s := requests.NewStore[*testReportRequest]() rid := uuid.New().String() - req := &ocr3.ReportRequest{ + req := &testReportRequest{ WorkflowExecutionID: rid, } @@ -138,15 +137,15 @@ func TestOCR3Store_ManagesStateConsistently(t *testing.T) { } func TestOCR3Store_ReadRequestsCopy(t *testing.T) { - s := requests.NewStore[*ocr3.ReportRequest]() + s := requests.NewStore[*testReportRequest]() rid := uuid.New().String() - cb := make(chan ocr3.ReportResponse, 1) + cb := make(chan testReportResponse, 1) stopCh := make(chan struct{}, 1) obs, err := values.NewList( []any{"hello", 1}, ) require.NoError(t, err) - req := &ocr3.ReportRequest{ + req := &testReportRequest{ WorkflowExecutionID: rid, WorkflowID: "wid", WorkflowName: "name", @@ -165,17 +164,17 @@ func TestOCR3Store_ReadRequestsCopy(t *testing.T) { testCases := []struct { name string - get func(ctx context.Context, rid string) *ocr3.ReportRequest + get func(ctx context.Context, rid string) *testReportRequest }{ { name: "get", - get: func(ctx context.Context, rid string) *ocr3.ReportRequest { + get: func(ctx context.Context, rid string) *testReportRequest { return s.Get(rid) }, }, { name: "firstN", - get: func(ctx context.Context, rid string) *ocr3.ReportRequest { + get: func(ctx context.Context, rid string) *testReportRequest { rs, err2 := s.FirstN(1) require.NoError(t, err2) assert.Len(t, rs, 1) @@ -184,7 +183,7 @@ func TestOCR3Store_ReadRequestsCopy(t *testing.T) { }, { name: "getByIDs", - get: func(ctx context.Context, rid string) *ocr3.ReportRequest { + get: func(ctx context.Context, rid string) *testReportRequest { rs := s.GetByIDs([]string{rid}) assert.Len(t, rs, 1) return rs[0] @@ -211,7 +210,7 @@ func TestOCR3Store_ReadRequestsCopy(t *testing.T) { gr.StopCh <- struct{}{} <-stopCh - gr.CallbackCh <- ocr3.ReportResponse{} + gr.CallbackCh <- testReportResponse{} <-cb }) }