diff --git a/cli/golem-cli/command-output-schema/command-output.schema.json b/cli/golem-cli/command-output-schema/command-output.schema.json index 2ad3e5ef83..1282b272cc 100644 --- a/cli/golem-cli/command-output-schema/command-output.schema.json +++ b/cli/golem-cli/command-output-schema/command-output.schema.json @@ -1782,6 +1782,7 @@ "required": [ "$type", "index", + "attribution", "entry" ], "properties": { @@ -1792,6 +1793,9 @@ "type": "integer", "minimum": 0 }, + "attribution": { + "$ref": "#/definitions/PublicOplogEntryAttribution" + }, "entry": { "$ref": "#/definitions/PublicOplogEntry" } @@ -4572,6 +4576,78 @@ }, "additionalProperties": false }, + "PublicOplogEntryAttribution": { + "oneOf": [ + { + "type": "object", + "required": ["type"], + "properties": { + "type": { "const": "Agent" } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": ["type", "invocation", "ancestors"], + "properties": { + "type": { "const": "Entity" }, + "invocation": { "$ref": "#/definitions/PublicEntityInvocation" }, + "ancestors": { + "type": "array", + "items": { "$ref": "#/definitions/PublicEntityInvocation" } + } + }, + "additionalProperties": false + } + ] + }, + "PublicEntityInvocation": { + "type": "object", + "required": ["entity", "startIndex", "callMode", "operation"], + "properties": { + "entity": { "$ref": "#/definitions/PublicAgentEntity" }, + "startIndex": { "type": "integer", "minimum": 1 }, + "callMode": { "type": "string", "enum": ["synchronous", "asynchronous", "fireAndForget"] }, + "operation": { + "oneOf": [ + { "$ref": "#/definitions/PublicEntityInvocationOperation" }, + { "type": "null" } + ] + } + }, + "additionalProperties": false + }, + "PublicAgentEntity": { + "type": "object", + "required": ["kind", "name"], + "properties": { + "kind": { "type": "string", "enum": ["tool", "toolMiddleware"] }, + "name": { "type": "string" } + }, + "additionalProperties": false + }, + "PublicEntityInvocationOperation": { + "oneOf": [ + { + "type": "object", + "required": ["type", "commandPath", "hasStdin", "hasStdout", "declaresStdout"], + "properties": { + "type": { "const": "Tool" }, + "commandPath": { "type": "array", "items": { "type": "string" } }, + "hasStdin": { "type": "boolean" }, + "hasStdout": { + "type": "boolean", + "description": "Whether a live stdout attachment was requested. Stdout bytes are not recorded in the oplog." + }, + "declaresStdout": { + "type": "boolean", + "description": "Whether the tool declares stdout support. Stdout bytes are not recorded in the oplog." + } + }, + "additionalProperties": false + } + ] + }, "PublicOplogEntry": { "oneOf": [ { "$ref": "#/definitions/PublicOplogEntryCreate" }, @@ -7799,6 +7875,7 @@ }, "SecretValuePayload": { "type": "object", + "description": "Secret handle identity and resolution metadata. Plaintext secret material is never included.", "required": ["secretId", "version", "resolvedAt"], "properties": { "secretId": { "type": "string" }, diff --git a/cli/golem-cli/src/command_handler/agent/mod.rs b/cli/golem-cli/src/command_handler/agent/mod.rs index 3a9719a86a..3e83f800e8 100644 --- a/cli/golem-cli/src/command_handler/agent/mod.rs +++ b/cli/golem-cli/src/command_handler/agent/mod.rs @@ -70,7 +70,7 @@ use golem_common::model::component::ComponentName; use golem_common::model::component::{ComponentId, ComponentRevision}; use golem_common::model::component_metadata::{ParsedFunctionName, ParsedFunctionSite}; use golem_common::model::environment::EnvironmentName; -use golem_common::model::oplog::{OplogCursor, PublicOplogEntry}; +use golem_common::model::oplog::{OplogCursor, PublicOplogEntryWithIndex}; use golem_common::model::worker::{ AgentConfigEntryDto, RevertLastInvocations, RevertToOplogIndex, UpdateRecord, }; @@ -782,7 +782,7 @@ impl AgentCommandHandler { let mut cursor = Option::::None; let mut had_entries = false; loop { - let mut entries = Vec::<(u64, PublicOplogEntry)>::new(); + let mut entries = Vec::::new(); cursor = { let clients = self.ctx.golem_clients().await?; @@ -799,21 +799,18 @@ impl AgentCommandHandler { .await .map_service_error()?; - entries.extend( - result - .entries - .into_iter() - .map(|entry| (entry.oplog_index.as_u64(), entry.entry)), - ); + entries.extend(result.entries); result.next }; if !entries.is_empty() { had_entries = true; - for (index, entry) in entries { - self.ctx - .log_handler() - .log_output(AgentOplogEntryView { index, entry })?; + for entry in entries { + self.ctx.log_handler().log_output(AgentOplogEntryView { + index: entry.oplog_index.as_u64(), + attribution: entry.attribution, + entry: entry.entry, + })?; } } diff --git a/cli/golem-cli/src/model/agent/oplog.rs b/cli/golem-cli/src/model/agent/oplog.rs index 991286de27..037457945a 100644 --- a/cli/golem-cli/src/model/agent/oplog.rs +++ b/cli/golem-cli/src/model/agent/oplog.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::agent_id_display::SourceLanguage; use crate::log::logln; use crate::model::cli_output::StructuredOutput; use crate::model::text_format::*; @@ -21,12 +20,15 @@ use base64::prelude::BASE64_STANDARD; use golem_common::model::Timestamp; use golem_common::model::oplog::{ MultipartPartData, PluginInstallationDescription, PublicAgentInvocation, - PublicAgentInvocationResult, PublicAttributeValue, PublicOplogEntry, PublicSnapshotData, - PublicUpdateDescription, StringAttributeValue, + PublicAgentInvocationResult, PublicAttributeValue, PublicEntityCallMode, + PublicEntityInvocation, PublicEntityInvocationOperation, PublicOplogEntry, + PublicOplogEntryAttribution, PublicSnapshotData, PublicUpdateDescription, StringAttributeValue, }; use golem_common::schema::TypedSchemaValue; use serde::{Deserialize, Serialize}; +#[cfg(test)] +use crate::agent_id_display::SourceLanguage; #[cfg(test)] use crate::model::agent::{AgentMetadataView, AgentsMetadataResponseView, RawAgentId}; #[cfg(test)] @@ -40,6 +42,7 @@ use std::collections::HashMap; #[serde(rename_all = "camelCase")] pub struct AgentOplogEntryView { pub index: u64, + pub attribution: PublicOplogEntryAttribution, pub entry: PublicOplogEntry, } @@ -53,10 +56,65 @@ impl TextOutput for AgentOplogEntryView { "{}: ", format_main_id(&format!("#{:0>5}", self.index)) )); + for line in render_oplog_attribution_lines(&self.attribution) { + logln(line); + } self.entry.log() } } +fn render_oplog_attribution_lines(attribution: &PublicOplogEntryAttribution) -> Vec { + let pad = " "; + match attribution { + PublicOplogEntryAttribution::Agent(_) => vec![format!("{pad}owner: agent")], + PublicOplogEntryAttribution::Entity(context) => { + let mut lines = vec![ + format!("{pad}owner: entity"), + format!("{pad}entity chain:"), + ]; + for ancestor in &context.ancestors { + lines.push(format_entity_invocation(pad, ancestor, false)); + } + lines.push(format_entity_invocation(pad, &context.invocation, true)); + if let Some(PublicEntityInvocationOperation::Tool(tool)) = &context.invocation.operation + { + lines.push(format!( + "{pad}tool command: {}", + tool.command_path.join(" ") + )); + lines.push(format!("{pad}stdin requested: {}", tool.has_stdin)); + lines.push(format!("{pad}stdout requested: {}", tool.has_stdout)); + lines.push(format!("{pad}stdout declared: {}", tool.declares_stdout)); + lines.push(format!( + "{pad}stdout recording: none (live attachment only)" + )); + } + lines + } + } +} + +fn format_entity_invocation( + pad: &str, + invocation: &PublicEntityInvocation, + current: bool, +) -> String { + let kind = match invocation.entity.kind { + golem_common::model::oplog::PublicAgentEntityKind::Tool => "tool", + golem_common::model::oplog::PublicAgentEntityKind::ToolMiddleware => "tool middleware", + }; + let call_mode = match invocation.call_mode { + PublicEntityCallMode::Synchronous => "synchronous", + PublicEntityCallMode::Asynchronous => "asynchronous", + PublicEntityCallMode::FireAndForget => "fire and forget", + }; + let current = if current { " [current]" } else { "" }; + format!( + "{pad} - {kind} {} (start #{}, {call_mode}){current}", + invocation.entity.name, invocation.start_index + ) +} + impl TextOutput for PublicOplogEntry { fn log(&self) { let pad = " "; @@ -208,7 +266,7 @@ impl TextOutput for PublicOplogEntry { PublicAgentInvocationResult::AgentInitialization(output) | PublicAgentInvocationResult::AgentMethod(output) => { logln(format!("{pad}output:")); - log_typed_schema_value(pad, &output.output, &SourceLanguage::default()); + log_typed_schema_value(pad, &output.output); } PublicAgentInvocationResult::ManualUpdate(_) => {} PublicAgentInvocationResult::LoadSnapshot(fallible) => { @@ -952,7 +1010,6 @@ fn render_agent_invocation( lines.push(render_typed_schema_value_line( pad, ¶ms.constructor_parameters, - &SourceLanguage::default(), )); } PublicAgentInvocation::AgentMethodInvocation(params) => { @@ -961,11 +1018,7 @@ fn render_agent_invocation( format_id(¶ms.idempotency_key) )); lines.push(format!("{pad}input:")); - lines.push(render_typed_schema_value_line( - pad, - ¶ms.function_input, - &SourceLanguage::default(), - )); + lines.push(render_typed_schema_value_line(pad, ¶ms.function_input)); } PublicAgentInvocation::SaveSnapshot(_) => {} PublicAgentInvocation::LoadSnapshot(params) => { @@ -1067,20 +1120,20 @@ fn render_agent_invocation_header( } fn typed_schema_value_to_string(value: &TypedSchemaValue) -> String { - golem_common::schema::render::value_to_cli_text(value.graph(), value.root_type(), value.value()) - .unwrap_or_else(|err| format!("")) + golem_common::schema::render::value_to_cli_text_with_secret_metadata( + value.graph(), + value.root_type(), + value.value(), + ) + .unwrap_or_else(|err| format!("")) } -fn log_typed_schema_value(pad: &str, value: &TypedSchemaValue, source_language: &SourceLanguage) { - logln(render_typed_schema_value_line(pad, value, source_language)); +fn log_typed_schema_value(pad: &str, value: &TypedSchemaValue) { + logln(render_typed_schema_value_line(pad, value)); } -fn render_typed_schema_value_line( - pad: &str, - value: &TypedSchemaValue, - source_language: &SourceLanguage, -) -> String { - let rendered = crate::agent_id_display::render_typed_schema_value(value, source_language); +fn render_typed_schema_value_line(pad: &str, value: &TypedSchemaValue) -> String { + let rendered = typed_schema_value_to_string(value); format!("{pad} {rendered}") } @@ -1140,12 +1193,19 @@ mod tests { use comfy_table::{Cell, Table as ComfyTable}; use golem_common::model::component::ComponentRevision; use golem_common::model::invocation_context::TraceId; + use golem_common::model::oplog::payload::types::{SecretRevealAudit, SerializableDateTime}; use golem_common::model::oplog::{ - AgentInitializationParameters, AgentMethodInvocationParameters, LoadSnapshotParameters, - ManualUpdateParameters, ProcessOplogEntriesParameters, RawSnapshotData, + AgentInitializationParameters, AgentMethodInvocationParameters, HostRequestSecretReveal, + HostResponseSecretRevealed, LoadSnapshotParameters, ManualUpdateParameters, + ProcessOplogEntriesParameters, PublicAgentEntity, PublicAgentEntityKind, + PublicEntityInvocationContext, PublicToolInvocationOperation, RawSnapshotData, }; use golem_common::model::{Empty, IdempotencyKey}; - use golem_common::schema::{SchemaGraph, SchemaType, SchemaValue}; + use golem_common::schema::{ + IntoTypedSchemaValue, NamedFieldType, PermissionCardSpec, PermissionCardValuePayload, + QuotaTokenSpec, QuotaTokenValuePayload, SchemaGraph, SchemaType, SchemaValue, SecretSpec, + SecretValuePayload, + }; use test_r::test; fn timestamp() -> Timestamp { @@ -1316,6 +1376,198 @@ mod tests { } } + #[test] + fn oplog_attribution_renders_agent_and_nested_entity_ownership() { + let agent = + render_oplog_attribution_lines(&PublicOplogEntryAttribution::agent()).join("\n"); + assert_eq!(agent, " owner: agent"); + + let ancestor = PublicEntityInvocation { + entity: PublicAgentEntity { + kind: PublicAgentEntityKind::ToolMiddleware, + name: "authorize".to_string(), + }, + start_index: golem_common::model::OplogIndex::from_u64(7), + call_mode: PublicEntityCallMode::Synchronous, + operation: None, + }; + let invocation = PublicEntityInvocation { + entity: PublicAgentEntity { + kind: PublicAgentEntityKind::Tool, + name: "filesystem".to_string(), + }, + start_index: golem_common::model::OplogIndex::from_u64(11), + call_mode: PublicEntityCallMode::Asynchronous, + operation: Some(PublicEntityInvocationOperation::Tool( + PublicToolInvocationOperation { + command_path: vec!["files".to_string(), "lookup".to_string()], + has_stdin: true, + has_stdout: false, + declares_stdout: true, + }, + )), + }; + + let rendered = render_oplog_attribution_lines(&PublicOplogEntryAttribution::entity( + PublicEntityInvocationContext { + invocation, + ancestors: vec![ancestor], + }, + )) + .join("\n"); + + assert_contains_all( + &rendered, + &[ + "owner: entity", + "entity chain:", + "tool middleware authorize (start #7, synchronous)", + "tool filesystem (start #11, asynchronous) [current]", + "tool command: files lookup", + "stdin requested: true", + "stdout requested: false", + "stdout declared: true", + "stdout recording: none (live attachment only)", + ], + ); + assert!( + rendered.find("authorize").unwrap() < rendered.find("filesystem").unwrap(), + "ancestor must render before the current invocation:\n{rendered}" + ); + } + + #[test] + fn oplog_typed_values_show_safe_secret_metadata() { + let secret_id = uuid::Uuid::from_u128(1); + let permission_card_id = uuid::Uuid::from_u128(2); + let schema = SchemaType::record(vec![ + NamedFieldType { + name: "credential".to_string(), + body: SchemaType::secret(SecretSpec::default()), + metadata: Default::default(), + }, + NamedFieldType { + name: "quota".to_string(), + body: SchemaType::quota_token(QuotaTokenSpec::default()), + metadata: Default::default(), + }, + NamedFieldType { + name: "permission".to_string(), + body: SchemaType::permission_card(PermissionCardSpec { polymorphic: true }), + metadata: Default::default(), + }, + ]); + let value = TypedSchemaValue::new( + SchemaGraph::anonymous(schema), + SchemaValue::Record { + fields: vec![ + SchemaValue::Secret(SecretValuePayload { + secret_id, + config_key: Some(vec!["database".to_string(), "password".to_string()]), + version: 7, + resolved_at: chrono::DateTime::from_timestamp(1_700_000_000, 0).unwrap(), + category: Some("api-key".to_string()), + }), + SchemaValue::QuotaToken(QuotaTokenValuePayload { + environment_id: uuid::Uuid::nil().into(), + resource_name: "private-quota-resource".to_string(), + expected_use: 1, + last_credit: 0, + last_credit_at: chrono::DateTime::from_timestamp(1_700_000_001, 0).unwrap(), + }), + SchemaValue::PermissionCard(PermissionCardValuePayload { + card_id: permission_card_id, + parent_ids: vec![], + expires_at: None, + polymorphic: true, + }), + ], + }, + ); + + let mut invocation = agent_method_invocation(); + let PublicAgentInvocation::AgentMethodInvocation(parameters) = &mut invocation else { + unreachable!() + }; + parameters.function_input = value.clone(); + + for rendered in [ + typed_schema_value_to_string(&value), + render_for_test(AgentInvocationRenderKind::Started, &invocation), + ] { + assert_contains_all( + &rendered, + &[ + "", + "", + ], + ); + assert!(!rendered.contains("")); + assert!(!rendered.contains("secretValue")); + assert!(!rendered.contains("private-quota-resource")); + assert!(!rendered.contains(&permission_card_id.to_string())); + } + } + + #[test] + fn oplog_secret_reveal_payloads_show_audit_metadata_without_secret_values() { + let secret_id = uuid::Uuid::from_u128(3); + let request = HostRequestSecretReveal { + secret_id, + expected_type: SchemaGraph::anonymous(SchemaType::string()), + } + .into_typed_schema_value() + .expect("secret reveal request must be schema-encodable"); + let response = HostResponseSecretRevealed { + secret_id, + pinned_revision: 11, + resolved_at: SerializableDateTime { + seconds: 1_700_000_004, + nanoseconds: 0, + }, + result: Ok(()), + audit: SecretRevealAudit { + calling_agent: golem_common::model::AgentId { + component_id: golem_common::model::component::ComponentId(uuid::Uuid::nil()), + agent_id: "secret-reveal-auditor".to_string(), + }, + config_key: Some(vec!["database".to_string(), "password".to_string()]), + timestamp: SerializableDateTime { + seconds: 1_700_000_005, + nanoseconds: 0, + }, + }, + } + .into_typed_schema_value() + .expect("secret reveal response must be schema-encodable"); + + let rendered = [ + typed_schema_value_to_string(&request), + typed_schema_value_to_string(&response), + ] + .join("\n"); + assert_contains_all( + &rendered, + &[ + "low-bits: 3", + "secret-reveal-auditor", + "database", + "password", + "1700000004", + "1700000005", + "11", + ], + ); + assert!(!rendered.contains("secretValue")); + } + #[test] fn started_public_agent_invocations_render_without_debug_dump() { let cases = [ diff --git a/cli/golem-cli/src/model/cli_output/tests.rs b/cli/golem-cli/src/model/cli_output/tests.rs index 8614644ea0..4a906ca4da 100644 --- a/cli/golem-cli/src/model/cli_output/tests.rs +++ b/cli/golem-cli/src/model/cli_output/tests.rs @@ -627,6 +627,8 @@ fn cli_output_schema_focus_prunes_unrelated_definitions() { assert!(definitions.contains_key("agent.oplog")); assert!(definitions.contains_key("PublicOplogEntry")); + assert!(definitions.contains_key("PublicOplogEntryAttribution")); + assert!(definitions.contains_key("PublicEntityInvocation")); assert!(!definitions.contains_key("agent.list")); assert!(!definitions.contains_key("component.list")); @@ -1181,6 +1183,88 @@ fn cli_output_schema_validates_schema_native_secret_outputs() { } } +#[test] +fn agent_oplog_structured_output_exposes_secret_metadata_without_stdout_bytes() { + use golem_common::model::oplog::public_oplog_entry::StartParams; + use golem_common::model::oplog::{ + PublicAgentEntity, PublicAgentEntityKind, PublicDurableFunctionType, PublicEntityCallMode, + PublicEntityInvocation, PublicEntityInvocationContext, PublicEntityInvocationOperation, + PublicOplogEntry, PublicOplogEntryAttribution, PublicToolInvocationOperation, + }; + use golem_common::model::{Empty, OplogIndex, Timestamp}; + use golem_common::schema::schema_type::SecretSpec; + use golem_common::schema::{SchemaGraph, SchemaType, SchemaValue, SecretValuePayload}; + + let secret_id = uuid::Uuid::from_u128(1); + let request = golem_common::schema::TypedSchemaValue::new( + SchemaGraph::anonymous(SchemaType::secret(SecretSpec::default())), + SchemaValue::Secret(SecretValuePayload { + secret_id, + config_key: Some(vec!["database".to_string(), "password".to_string()]), + version: 7, + resolved_at: chrono::DateTime::from_timestamp(1_700_000_000, 0).unwrap(), + category: Some("api-key".to_string()), + }), + ); + let output = to_structured_output_value(crate::model::agent::oplog::AgentOplogEntryView { + index: 12, + attribution: PublicOplogEntryAttribution::entity(PublicEntityInvocationContext { + invocation: PublicEntityInvocation { + entity: PublicAgentEntity { + kind: PublicAgentEntityKind::Tool, + name: "filesystem".to_string(), + }, + start_index: OplogIndex::from_u64(11), + call_mode: PublicEntityCallMode::Synchronous, + operation: Some(PublicEntityInvocationOperation::Tool( + PublicToolInvocationOperation { + command_path: vec!["files".to_string(), "lookup".to_string()], + has_stdin: false, + has_stdout: true, + declares_stdout: true, + }, + )), + }, + ancestors: Vec::new(), + }), + entry: PublicOplogEntry::Start(StartParams { + timestamp: Timestamp::from(0), + parent_start_index: None, + function_name: "golem::entity::invoke".to_string(), + invocation_id: None, + observational_owner: None, + request: Some(request), + durable_function_type: PublicDurableFunctionType::WriteLocal(Empty {}), + }), + }) + .expect("agent.oplog should serialize"); + + let validator = jsonschema::options() + .build(&load_command_output_schema()) + .expect("command output schema must be valid"); + assert!( + validator.is_valid(&output), + "schema rejected agent.oplog secret metadata: {:?}", + validator + .iter_errors(&output) + .map(|error| error.to_string()) + .collect::>() + ); + + let metadata = &output["entry"]["request"]["value"]["value"]; + assert_eq!(metadata["secretId"], json!(secret_id)); + assert_eq!(metadata["configKey"], json!(["database", "password"])); + assert_eq!(metadata["version"], json!(7)); + assert_eq!(metadata["resolvedAt"], json!("2023-11-14T22:13:20Z")); + assert_eq!(metadata["category"], json!("api-key")); + assert!(metadata.get("secretValue").is_none()); + + let operation = &output["attribution"]["invocation"]["operation"]; + assert_eq!(operation["hasStdout"], json!(true)); + assert_eq!(operation["declaresStdout"], json!(true)); + assert!(operation.get("stdout").is_none()); +} + #[test] fn cli_output_schema_validates_schema_native_component_and_agent_outputs() { let schema = load_command_output_schema(); @@ -1211,6 +1295,7 @@ fn cli_output_schema_validates_schema_native_component_and_agent_outputs() { .expect("agent-type.list should serialize"), to_structured_output_value(crate::model::agent::oplog::AgentOplogEntryView { index: 0, + attribution: golem_common::model::oplog::PublicOplogEntryAttribution::agent(), entry: sample_public_oplog_entries() .into_iter() .next() @@ -2800,17 +2885,86 @@ fn arb_agent_oplog_result() -> OutputDocumentStrategy { serialized_output( ( arb_small_u64(), + arb_public_oplog_entry_attribution(), prop_oneof![ proptest::sample::select(sample_public_oplog_entries()), arb_typed_value_oplog_entry(), ], ) - .prop_map( - |(index, entry)| crate::model::agent::oplog::AgentOplogEntryView { index, entry }, - ), + .prop_map(|(index, attribution, entry)| { + crate::model::agent::oplog::AgentOplogEntryView { + index, + attribution, + entry, + } + }), ) } +fn arb_public_oplog_entry_attribution() +-> BoxedStrategy { + use golem_common::model::oplog::{ + PublicAgentEntity, PublicAgentEntityKind, PublicEntityCallMode, PublicEntityInvocation, + PublicEntityInvocationContext, PublicEntityInvocationOperation, + PublicOplogEntryAttribution, PublicToolInvocationOperation, + }; + + let invocation = || { + ( + prop_oneof![ + Just(PublicAgentEntityKind::Tool), + Just(PublicAgentEntityKind::ToolMiddleware), + ], + arb_small_string(), + 1u64..1000, + prop_oneof![ + Just(PublicEntityCallMode::Synchronous), + Just(PublicEntityCallMode::Asynchronous), + Just(PublicEntityCallMode::FireAndForget), + ], + proptest::option::of( + ( + proptest::collection::vec(arb_small_string(), 0..4), + any::(), + any::(), + any::(), + ) + .prop_map( + |(command_path, has_stdin, has_stdout, declares_stdout)| { + PublicEntityInvocationOperation::Tool(PublicToolInvocationOperation { + command_path, + has_stdin, + has_stdout, + declares_stdout, + }) + }, + ), + ), + ) + .prop_map(|(kind, name, start_index, call_mode, operation)| { + PublicEntityInvocation { + entity: PublicAgentEntity { kind, name }, + start_index: golem_common::model::OplogIndex::from_u64(start_index), + call_mode, + operation, + } + }) + }; + + prop_oneof![ + Just(PublicOplogEntryAttribution::agent()), + (invocation(), proptest::collection::vec(invocation(), 0..4)).prop_map( + |(invocation, ancestors)| PublicOplogEntryAttribution::entity( + PublicEntityInvocationContext { + invocation, + ancestors, + } + ) + ), + ] + .boxed() +} + /// Oplog entry carrying a structurally-comprehensive [`TypedSchemaValue`] /// (full `SchemaGraph` + `SchemaValue`). This is the non-masked path that /// exercises every schema-native value/graph case against the schema's diff --git a/docs/src/content/next/rest-api/worker.mdx b/docs/src/content/next/rest-api/worker.mdx index 0f79773d75..fb424d6068 100644 --- a/docs/src/content/next/rest-api/worker.mdx +++ b/docs/src/content/next/rest-api/worker.mdx @@ -647,6 +647,9 @@ query|string|No|- "entries": [ { "oplogIndex": 0, + "attribution": { + "type": "Agent" + }, "entry": { "type": "Create", "timestamp": "2019-08-24T14:15:22Z", diff --git a/golem-api-grpc/proto/golem/worker/public_oplog.proto b/golem-api-grpc/proto/golem/worker/public_oplog.proto index 44b5e85b2e..bcc96e0a37 100644 --- a/golem-api-grpc/proto/golem/worker/public_oplog.proto +++ b/golem-api-grpc/proto/golem/worker/public_oplog.proto @@ -632,4 +632,55 @@ message OplogProcessorCheckpointParameters { message OplogEntryWithIndex { uint64 oplog_index = 1; OplogEntry entry = 2; + PublicOplogEntryAttribution attribution = 3; +} + +enum PublicAgentEntityKind { + PUBLIC_AGENT_ENTITY_KIND_UNSPECIFIED = 0; + PUBLIC_AGENT_ENTITY_KIND_TOOL = 1; + PUBLIC_AGENT_ENTITY_KIND_TOOL_MIDDLEWARE = 2; +} + +enum PublicEntityCallMode { + PUBLIC_ENTITY_CALL_MODE_UNSPECIFIED = 0; + PUBLIC_ENTITY_CALL_MODE_SYNCHRONOUS = 1; + PUBLIC_ENTITY_CALL_MODE_ASYNCHRONOUS = 2; + PUBLIC_ENTITY_CALL_MODE_FIRE_AND_FORGET = 3; +} + +message PublicAgentEntity { + PublicAgentEntityKind kind = 1; + string name = 2; +} + +message PublicToolInvocationOperation { + repeated string command_path = 1; + bool has_stdin = 2; + bool has_stdout = 3; + bool declares_stdout = 4; +} + +message PublicEntityInvocationOperation { + oneof operation { + PublicToolInvocationOperation tool = 1; + } +} + +message PublicEntityInvocation { + PublicAgentEntity entity = 1; + optional uint64 start_index = 2; + PublicEntityCallMode call_mode = 3; + PublicEntityInvocationOperation operation = 4; +} + +message PublicEntityInvocationContext { + PublicEntityInvocation invocation = 1; + repeated PublicEntityInvocation ancestors = 2; +} + +message PublicOplogEntryAttribution { + oneof attribution { + golem.common.Empty agent = 1; + PublicEntityInvocationContext entity = 2; + } } diff --git a/golem-api-grpc/proto/golem/worker/raw_oplog.proto b/golem-api-grpc/proto/golem/worker/raw_oplog.proto index 1e51a23816..94261888ff 100644 --- a/golem-api-grpc/proto/golem/worker/raw_oplog.proto +++ b/golem-api-grpc/proto/golem/worker/raw_oplog.proto @@ -20,6 +20,7 @@ message RawOplogEntry { reserved "filesystem_storage_usage_update"; google.protobuf.Timestamp timestamp = 1; + optional uint64 entity_parent_start_index = 3; oneof entry { RawCreateParameters create = 2; RawAgentInvocationStartedParameters agent_invocation_started = 4; diff --git a/golem-api-grpc/proto/golem/workerexecutor/v1/worker_executor.proto b/golem-api-grpc/proto/golem/workerexecutor/v1/worker_executor.proto index bdec1408e0..e6f25a0b73 100644 --- a/golem-api-grpc/proto/golem/workerexecutor/v1/worker_executor.proto +++ b/golem-api-grpc/proto/golem/workerexecutor/v1/worker_executor.proto @@ -305,7 +305,7 @@ message GetOplogResponse { } message GetOplogSuccessResponse { - repeated golem.worker.OplogEntry entries = 1; + repeated golem.worker.OplogEntryWithIndex entries = 1; optional golem.worker.OplogCursor next = 2; uint64 first_index_in_chunk = 3; uint64 last_index = 5; diff --git a/golem-common/src/base_model/oplog/mod.rs b/golem-common/src/base_model/oplog/mod.rs index 26300b67f1..f5b7cae383 100644 --- a/golem-common/src/base_model/oplog/mod.rs +++ b/golem-common/src/base_model/oplog/mod.rs @@ -235,6 +235,7 @@ oplog_entry! { wit_raw_type: "raw-error-parameters" wit_public_type: "error-parameters" raw { + entity_parent_start_index: Option, error: AgentError, /// Points to the oplog index where the retry should start from. Normally this can be just the /// current oplog index (after the last persisted side-effect). When failing in an atomic region @@ -261,7 +262,9 @@ oplog_entry! { hint: false wit_raw_type: "timestamp" wit_public_type: "timestamp" - raw {} + raw { + entity_parent_start_index: Option, + } public {} }, /// The worker needs to recover up to the given target oplog index and continue running from @@ -273,6 +276,7 @@ oplog_entry! { wit_raw_type: "jump-parameters" wit_public_type: "jump-parameters" raw { + entity_parent_start_index: Option, jump: OplogRegion, } public { @@ -302,7 +306,9 @@ oplog_entry! { hint: false wit_raw_type: "timestamp" wit_public_type: "timestamp" - raw {} + raw { + entity_parent_start_index: Option, + } public {} }, /// Ends an atomic region. All oplog entries between the corresponding `BeginAtomicRegion` and this @@ -313,6 +319,7 @@ oplog_entry! { wit_raw_type: "end-atomic-region-parameters" wit_public_type: "end-atomic-region-parameters" raw { + entity_parent_start_index: Option, begin_index: OplogIndex, } public { @@ -400,6 +407,7 @@ oplog_entry! { wit_raw_type: "raw-create-resource-parameters" wit_public_type: "create-resource-parameters" raw { + entity_parent_start_index: Option, id: AgentResourceId, resource_type_id: ResourceTypeId, } @@ -415,6 +423,7 @@ oplog_entry! { wit_raw_type: "raw-drop-resource-parameters" wit_public_type: "drop-resource-parameters" raw { + entity_parent_start_index: Option, id: AgentResourceId, resource_type_id: ResourceTypeId, } @@ -652,6 +661,7 @@ oplog_entry! { wit_raw_type: "set-retry-policy-parameters" wit_public_type: "set-retry-policy-parameters" raw { + entity_parent_start_index: Option, policy: NamedRetryPolicy, } public { @@ -664,6 +674,7 @@ oplog_entry! { wit_raw_type: "remove-retry-policy-parameters" wit_public_type: "remove-retry-policy-parameters" raw { + entity_parent_start_index: Option, name: String, } public { @@ -676,6 +687,7 @@ oplog_entry! { wit_raw_type: "raw-card-event-queued-parameters" wit_public_type: "card-event-queued-parameters" raw { + entity_parent_start_index: Option, event: QueuedCardEvent, } public { @@ -689,6 +701,7 @@ oplog_entry! { wit_raw_type: "raw-card-installed-parameters" wit_public_type: "card-installed-parameters" raw { + entity_parent_start_index: Option, queued_event_index: Option, card: StoredCard, wallet_generation: Option, @@ -705,6 +718,7 @@ oplog_entry! { wit_raw_type: "card-install-failed-parameters" wit_public_type: "card-install-failed-parameters" raw { + entity_parent_start_index: Option, queued_event_index: OplogIndex, card_id: CardId, reason: CardInstallFailure, @@ -722,6 +736,7 @@ oplog_entry! { wit_raw_type: "card-revoked-parameters" wit_public_type: "card-revoked-parameters" raw { + entity_parent_start_index: Option, queued_event_index: OplogIndex, card_id: CardId, wallet_generation: Option, @@ -739,6 +754,7 @@ oplog_entry! { wit_raw_type: "card-expired-parameters" wit_public_type: "card-expired-parameters" raw { + entity_parent_start_index: Option, card_id: CardId, wallet_generation: Option, } @@ -754,6 +770,7 @@ oplog_entry! { wit_raw_type: "raw-card-derived-parameters" wit_public_type: "card-derived-parameters" raw { + entity_parent_start_index: Option, card: StoredCard, wallet_generation: Option, } @@ -782,6 +799,7 @@ oplog_entry! { wit_raw_type: "raw-card-transfer-started-parameters" wit_public_type: "card-transfer-started-parameters" raw { + entity_parent_start_index: Option, transfer_id: Uuid, card_id: CardId, source_holder: Option, @@ -812,6 +830,7 @@ oplog_entry! { wit_raw_type: "raw-card-transferred-parameters" wit_public_type: "card-transferred-parameters" raw { + entity_parent_start_index: Option, transfer_id: Uuid, source_card_id: Option, installed_card_id: CardId, @@ -844,6 +863,7 @@ oplog_entry! { wit_raw_type: "raw-card-revoked-cascade-parameters" wit_public_type: "card-revoked-cascade-parameters" raw { + entity_parent_start_index: Option, revoked_card_ids: Vec, affected_wallets: Vec, local_wallet_generation: Option, @@ -864,6 +884,7 @@ oplog_entry! { wit_raw_type: "raw-card-transfer-confirmed-parameters" wit_public_type: "card-transfer-confirmed-parameters" raw { + entity_parent_start_index: Option, transfer_id: Uuid, source_card_id: CardId, installed_card_id: CardId, @@ -907,6 +928,7 @@ oplog_entry! { wit_raw_type: "raw-durable-stream-record-parameters" wit_public_type: "durable-stream-record-parameters" raw { + entity_parent_start_index: Option, record: payload::OplogPayload, } public { @@ -919,6 +941,7 @@ oplog_entry! { wit_raw_type: "raw-durable-stream-record-parameters" wit_public_type: "durable-stream-record-parameters" raw { + entity_parent_start_index: Option, record: payload::OplogPayload, } public { @@ -931,6 +954,7 @@ oplog_entry! { wit_raw_type: "raw-durable-stream-record-parameters" wit_public_type: "durable-stream-record-parameters" raw { + entity_parent_start_index: Option, record: payload::OplogPayload, } public { @@ -943,6 +967,7 @@ oplog_entry! { wit_raw_type: "raw-durable-stream-record-parameters" wit_public_type: "durable-stream-record-parameters" raw { + entity_parent_start_index: Option, record: payload::OplogPayload, } public { @@ -956,6 +981,7 @@ oplog_entry! { wit_raw_type: "raw-durable-stream-record-parameters" wit_public_type: "durable-stream-record-parameters" raw { + entity_parent_start_index: Option, record: payload::OplogPayload, } public { diff --git a/golem-common/src/base_model/oplog/public_types.rs b/golem-common/src/base_model/oplog/public_types.rs index 38051c3b9f..a09b4fa6e8 100644 --- a/golem-common/src/base_model/oplog/public_types.rs +++ b/golem-common/src/base_model/oplog/public_types.rs @@ -20,8 +20,8 @@ use crate::base_model::oplog::PublicOplogEntry; use crate::base_model::oplog::public_oplog_entry::{Deserialize, Serialize}; use crate::base_model::retry_policy::{ApiPredicate, ApiRetryPolicy}; use crate::base_model::{Empty, IdempotencyKey, OplogIndex, Timestamp}; -use crate::declare_structs; use crate::schema::TypedSchemaValue; +use crate::{declare_structs, declare_unions}; use golem_schema_derive::{FromSchema, IntoSchema}; use std::collections::BTreeMap; use std::fmt; @@ -74,8 +74,80 @@ impl Display for OplogCursor { declare_structs! { pub struct PublicOplogEntryWithIndex { pub oplog_index: OplogIndex, + pub attribution: PublicOplogEntryAttribution, pub entry: PublicOplogEntry, } + + pub struct PublicAgentEntity { + pub kind: PublicAgentEntityKind, + pub name: String, + } + + /// One entity invocation in an owner-oplog execution chain. + pub struct PublicEntityInvocation { + pub entity: PublicAgentEntity, + pub start_index: OplogIndex, + pub call_mode: PublicEntityCallMode, + pub operation: Option, + } + + /// Attribution for an entry executed by an entity. Ancestors are ordered from the root entity + /// invocation to the immediate parent of `invocation`. + pub struct PublicEntityInvocationContext { + pub invocation: PublicEntityInvocation, + pub ancestors: Vec, + } + + pub struct PublicToolInvocationOperation { + pub command_path: Vec, + /// Whether a live stdin attachment was requested. + pub has_stdin: bool, + /// Whether a live stdout attachment was requested. Stdout bytes are not recorded in the + /// oplog. + pub has_stdout: bool, + /// Whether the tool declares stdout support. Stdout bytes are not recorded in the oplog. + pub declares_stdout: bool, + } +} + +declare_unions! { + pub enum PublicOplogEntryAttribution { + Agent(Empty), + Entity(PublicEntityInvocationContext), + } + + pub enum PublicEntityInvocationOperation { + Tool(PublicToolInvocationOperation), + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "full", derive(poem_openapi::Enum))] +#[cfg_attr(feature = "full", oai(rename_all = "camelCase"))] +#[serde(rename_all = "camelCase")] +pub enum PublicAgentEntityKind { + Tool, + ToolMiddleware, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "full", derive(poem_openapi::Enum))] +#[cfg_attr(feature = "full", oai(rename_all = "camelCase"))] +#[serde(rename_all = "camelCase")] +pub enum PublicEntityCallMode { + Synchronous, + Asynchronous, + FireAndForget, +} + +impl PublicOplogEntryAttribution { + pub fn agent() -> Self { + Self::Agent(Empty {}) + } + + pub fn entity(context: PublicEntityInvocationContext) -> Self { + Self::Entity(context) + } } #[derive(Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord, Deserialize)] diff --git a/golem-common/src/model/mod.rs b/golem-common/src/model/mod.rs index 7a3ddeab08..5e5f2e2e31 100644 --- a/golem-common/src/model/mod.rs +++ b/golem-common/src/model/mod.rs @@ -1335,6 +1335,7 @@ impl PendingInvocationRef { pub struct PendingCardEventRef { pub timestamp: Timestamp, pub oplog_index: OplogIndex, + pub entity_parent_start_index: Option, pub event: QueuedCardEvent, } diff --git a/golem-common/src/model/oplog/mod.rs b/golem-common/src/model/oplog/mod.rs index 787ebab314..e1d1e30300 100644 --- a/golem-common/src/model/oplog/mod.rs +++ b/golem-common/src/model/oplog/mod.rs @@ -32,6 +32,140 @@ pub use raw_types::*; use crate::model::component::ComponentRevision; impl OplogEntry { + pub fn entity_parent_start_index(&self) -> Option { + match self { + OplogEntry::Error { + entity_parent_start_index, + .. + } + | OplogEntry::NoOp { + entity_parent_start_index, + .. + } + | OplogEntry::Jump { + entity_parent_start_index, + .. + } + | OplogEntry::BeginAtomicRegion { + entity_parent_start_index, + .. + } + | OplogEntry::EndAtomicRegion { + entity_parent_start_index, + .. + } + | OplogEntry::CreateResource { + entity_parent_start_index, + .. + } + | OplogEntry::DropResource { + entity_parent_start_index, + .. + } + | OplogEntry::SetRetryPolicy { + entity_parent_start_index, + .. + } + | OplogEntry::RemoveRetryPolicy { + entity_parent_start_index, + .. + } + | OplogEntry::CardEventQueued { + entity_parent_start_index, + .. + } + | OplogEntry::CardInstalled { + entity_parent_start_index, + .. + } + | OplogEntry::CardInstallFailed { + entity_parent_start_index, + .. + } + | OplogEntry::CardRevoked { + entity_parent_start_index, + .. + } + | OplogEntry::CardExpired { + entity_parent_start_index, + .. + } + | OplogEntry::CardDerived { + entity_parent_start_index, + .. + } + | OplogEntry::CardTransferStarted { + entity_parent_start_index, + .. + } + | OplogEntry::CardTransferred { + entity_parent_start_index, + .. + } + | OplogEntry::CardRevokedCascade { + entity_parent_start_index, + .. + } + | OplogEntry::CardTransferConfirmed { + entity_parent_start_index, + .. + } + | OplogEntry::StreamRegistered { + entity_parent_start_index, + .. + } + | OplogEntry::StreamItems { + entity_parent_start_index, + .. + } + | OplogEntry::StreamEnd { + entity_parent_start_index, + .. + } + | OplogEntry::StreamCancel { + entity_parent_start_index, + .. + } + | OplogEntry::StreamSession { + entity_parent_start_index, + .. + } => *entity_parent_start_index, + OplogEntry::Create { .. } + | OplogEntry::Start { .. } + | OplogEntry::End { .. } + | OplogEntry::Cancelled { .. } + | OplogEntry::CompletionDiscarded { .. } + | OplogEntry::CompletionDelivered { .. } + | OplogEntry::AgentInvocationStarted { .. } + | OplogEntry::AgentInvocationFinished { .. } + | OplogEntry::Suspend { .. } + | OplogEntry::Interrupted { .. } + | OplogEntry::Exited { .. } + | OplogEntry::PendingAgentInvocation { .. } + | OplogEntry::PendingUpdate { .. } + | OplogEntry::SuccessfulUpdate { .. } + | OplogEntry::FailedUpdate { .. } + | OplogEntry::GrowMemory { .. } + | OplogEntry::Log { .. } + | OplogEntry::Restart { .. } + | OplogEntry::ActivatePlugin { .. } + | OplogEntry::DeactivatePlugin { .. } + | OplogEntry::Revert { .. } + | OplogEntry::CancelPendingInvocation { .. } + | OplogEntry::StartSpan { .. } + | OplogEntry::FinishSpan { .. } + | OplogEntry::SetSpanAttribute { .. } + | OplogEntry::BeginRemoteTransaction { .. } + | OplogEntry::PreCommitRemoteTransaction { .. } + | OplogEntry::PreRollbackRemoteTransaction { .. } + | OplogEntry::CommittedRemoteTransaction { .. } + | OplogEntry::RolledBackRemoteTransaction { .. } + | OplogEntry::Snapshot { .. } + | OplogEntry::OplogProcessorCheckpoint { .. } + | OplogEntry::HostStreamFrame { .. } => None, + } + } + pub fn is_end_atomic_region(&self, idx: OplogIndex) -> bool { matches!(self, OplogEntry::EndAtomicRegion { begin_index, .. } if *begin_index == idx) } @@ -356,7 +490,32 @@ impl OplogScopeProjection { parent_start_index: Some(parent_start_index), .. } => self.starts.contains(parent_start_index), - OplogEntry::Error { retry_from, .. } => self.starts.contains(retry_from), + OplogEntry::Error { .. } + | OplogEntry::NoOp { .. } + | OplogEntry::Jump { .. } + | OplogEntry::BeginAtomicRegion { .. } + | OplogEntry::EndAtomicRegion { .. } + | OplogEntry::CreateResource { .. } + | OplogEntry::DropResource { .. } + | OplogEntry::SetRetryPolicy { .. } + | OplogEntry::RemoveRetryPolicy { .. } + | OplogEntry::CardEventQueued { .. } + | OplogEntry::CardInstalled { .. } + | OplogEntry::CardInstallFailed { .. } + | OplogEntry::CardRevoked { .. } + | OplogEntry::CardExpired { .. } + | OplogEntry::CardDerived { .. } + | OplogEntry::CardTransferStarted { .. } + | OplogEntry::CardTransferred { .. } + | OplogEntry::CardRevokedCascade { .. } + | OplogEntry::CardTransferConfirmed { .. } + | OplogEntry::StreamRegistered { .. } + | OplogEntry::StreamItems { .. } + | OplogEntry::StreamEnd { .. } + | OplogEntry::StreamCancel { .. } + | OplogEntry::StreamSession { .. } => entry + .entity_parent_start_index() + .is_some_and(|parent| self.starts.contains(&parent)), OplogEntry::BeginRemoteTransaction { original_begin_index: Some(begin), .. @@ -380,19 +539,13 @@ impl OplogScopeProjection { | OplogEntry::AgentInvocationStarted { .. } | OplogEntry::AgentInvocationFinished { .. } | OplogEntry::Suspend { .. } - | OplogEntry::NoOp { .. } - | OplogEntry::Jump { .. } | OplogEntry::Interrupted { .. } | OplogEntry::Exited { .. } - | OplogEntry::BeginAtomicRegion { .. } - | OplogEntry::EndAtomicRegion { .. } | OplogEntry::PendingAgentInvocation { .. } | OplogEntry::PendingUpdate { .. } | OplogEntry::SuccessfulUpdate { .. } | OplogEntry::FailedUpdate { .. } | OplogEntry::GrowMemory { .. } - | OplogEntry::CreateResource { .. } - | OplogEntry::DropResource { .. } | OplogEntry::Log { parent_start_index: None, .. @@ -415,24 +568,7 @@ impl OplogScopeProjection { .. } | OplogEntry::Snapshot { .. } - | OplogEntry::OplogProcessorCheckpoint { .. } - | OplogEntry::SetRetryPolicy { .. } - | OplogEntry::RemoveRetryPolicy { .. } - | OplogEntry::CardEventQueued { .. } - | OplogEntry::CardInstalled { .. } - | OplogEntry::CardInstallFailed { .. } - | OplogEntry::CardRevoked { .. } - | OplogEntry::CardExpired { .. } - | OplogEntry::CardDerived { .. } - | OplogEntry::CardTransferStarted { .. } - | OplogEntry::CardTransferred { .. } - | OplogEntry::CardRevokedCascade { .. } - | OplogEntry::CardTransferConfirmed { .. } - | OplogEntry::StreamRegistered { .. } - | OplogEntry::StreamItems { .. } - | OplogEntry::StreamEnd { .. } - | OplogEntry::StreamCancel { .. } - | OplogEntry::StreamSession { .. } => false, + | OplogEntry::OplogProcessorCheckpoint { .. } => false, }; self.previous_index = Some(index); self.previous_included_start = included_start.then_some(index); diff --git a/golem-common/src/model/oplog/payload/tests.rs b/golem-common/src/model/oplog/payload/tests.rs index 02ed3d22f7..dc02473f6b 100644 --- a/golem-common/src/model/oplog/payload/tests.rs +++ b/golem-common/src/model/oplog/payload/tests.rs @@ -1207,7 +1207,6 @@ fn tool_invocation_host_payload_pairs_roundtrip() { let response = HostResponseGolemToolInvokeResult { result: Ok(SerializableToolInvocationResult { result: Some("match".to_string().into_typed_schema_value().unwrap()), - stdout: Some(b"line one\nline two\n".to_vec()), }), }; diff --git a/golem-common/src/model/oplog/payload/types.rs b/golem-common/src/model/oplog/payload/types.rs index 37dcabf76a..1930da1ed6 100644 --- a/golem-common/src/model/oplog/payload/types.rs +++ b/golem-common/src/model/oplog/payload/types.rs @@ -2322,7 +2322,6 @@ pub struct SerializableToolOperationTerminal { #[desert(evolution())] pub struct SerializableToolInvocationResult { pub result: Option, - pub stdout: Option>, } #[derive( diff --git a/golem-common/src/model/oplog/protobuf.rs b/golem-common/src/model/oplog/protobuf.rs index 2959e63481..5b71ed0dcc 100644 --- a/golem-common/src/model/oplog/protobuf.rs +++ b/golem-common/src/model/oplog/protobuf.rs @@ -18,13 +18,15 @@ use super::{ AgentMethodInvocationParameters, AgentResourceId, FallibleResultParameters, JsonSnapshotData, LoadSnapshotParameters, LogLevel, ManualUpdateParameters, MultipartPartData, MultipartSnapshotData, MultipartSnapshotPart, OplogCursor, PluginInstallationDescription, - ProcessOplogEntriesParameters, ProcessOplogEntriesResultParameters, PublicAgentInvocation, - PublicAgentInvocationResult, PublicAttribute, PublicAttributeValue, PublicDurableFunctionType, - PublicExternalSpanData, PublicLocalSpanData, PublicOplogEntry, PublicOplogEntryWithIndex, - PublicRetryPolicyState, PublicSnapshotData, PublicSpanData, PublicTypedAgentConfigEntry, - PublicUpdateDescription, RawSnapshotData, SaveSnapshotResultParameters, - SnapshotBasedUpdateParameters, StringAttributeValue, WriteRemoteBatchedParameters, - WriteRemoteTransactionParameters, + ProcessOplogEntriesParameters, ProcessOplogEntriesResultParameters, PublicAgentEntity, + PublicAgentEntityKind, PublicAgentInvocation, PublicAgentInvocationResult, PublicAttribute, + PublicAttributeValue, PublicDurableFunctionType, PublicEntityCallMode, PublicEntityInvocation, + PublicEntityInvocationContext, PublicEntityInvocationOperation, PublicExternalSpanData, + PublicLocalSpanData, PublicOplogEntry, PublicOplogEntryAttribution, PublicOplogEntryWithIndex, + PublicRetryPolicyState, PublicSnapshotData, PublicSpanData, PublicToolInvocationOperation, + PublicTypedAgentConfigEntry, PublicUpdateDescription, RawSnapshotData, + SaveSnapshotResultParameters, SnapshotBasedUpdateParameters, StringAttributeValue, + WriteRemoteBatchedParameters, WriteRemoteTransactionParameters, }; use crate::base_model::OplogIndex; use crate::base_model::agent::AgentMode; @@ -2759,6 +2761,10 @@ impl TryFrom ) -> Result { Ok(Self { oplog_index: OplogIndex::from_u64(value.oplog_index), + attribution: value + .attribution + .ok_or("Missing field: attribution")? + .try_into()?, entry: value.entry.ok_or("Missing field: entry")?.try_into()?, }) } @@ -2773,10 +2779,216 @@ impl TryFrom Ok(Self { oplog_index: value.oplog_index.into(), entry: Some(value.entry.try_into()?), + attribution: Some(value.attribution.into()), + }) + } +} + +impl TryFrom + for PublicOplogEntryAttribution +{ + type Error = String; + + fn try_from( + value: golem_api_grpc::proto::golem::worker::PublicOplogEntryAttribution, + ) -> Result { + use golem_api_grpc::proto::golem::worker::public_oplog_entry_attribution::Attribution; + + match value.attribution.ok_or("Missing oplog entry attribution")? { + Attribution::Agent(_) => Ok(Self::agent()), + Attribution::Entity(context) => Ok(Self::entity(context.try_into()?)), + } + } +} + +impl From + for golem_api_grpc::proto::golem::worker::PublicOplogEntryAttribution +{ + fn from(value: PublicOplogEntryAttribution) -> Self { + use golem_api_grpc::proto::golem::worker::public_oplog_entry_attribution::Attribution; + + let attribution = match value { + PublicOplogEntryAttribution::Agent(_) => { + Attribution::Agent(golem_api_grpc::proto::golem::common::Empty {}) + } + PublicOplogEntryAttribution::Entity(context) => Attribution::Entity(context.into()), + }; + Self { + attribution: Some(attribution), + } + } +} + +impl TryFrom + for PublicEntityInvocationContext +{ + type Error = String; + + fn try_from( + value: golem_api_grpc::proto::golem::worker::PublicEntityInvocationContext, + ) -> Result { + Ok(Self { + invocation: value + .invocation + .ok_or("Missing entity invocation")? + .try_into()?, + ancestors: value + .ancestors + .into_iter() + .map(TryInto::try_into) + .collect::>()?, + }) + } +} + +impl From + for golem_api_grpc::proto::golem::worker::PublicEntityInvocationContext +{ + fn from(value: PublicEntityInvocationContext) -> Self { + Self { + invocation: Some(value.invocation.into()), + ancestors: value.ancestors.into_iter().map(Into::into).collect(), + } + } +} + +impl TryFrom + for PublicEntityInvocation +{ + type Error = String; + + fn try_from( + value: golem_api_grpc::proto::golem::worker::PublicEntityInvocation, + ) -> Result { + use golem_api_grpc::proto::golem::worker::PublicEntityCallMode as ProtoCallMode; + + let call_mode = match ProtoCallMode::try_from(value.call_mode) + .map_err(|_| format!("Invalid PublicEntityCallMode: {}", value.call_mode))? + { + ProtoCallMode::Synchronous => PublicEntityCallMode::Synchronous, + ProtoCallMode::Asynchronous => PublicEntityCallMode::Asynchronous, + ProtoCallMode::FireAndForget => PublicEntityCallMode::FireAndForget, + ProtoCallMode::Unspecified => { + return Err("Unspecified PublicEntityCallMode".to_string()); + } + }; + let start_index = value.start_index.ok_or("Missing entity start index")?; + if start_index == 0 { + return Err("Invalid entity start index: 0".to_string()); + } + Ok(Self { + entity: value + .entity + .ok_or("Missing public agent entity")? + .try_into()?, + start_index: OplogIndex::from_u64(start_index), + call_mode, + operation: value.operation.map(TryInto::try_into).transpose()?, }) } } +impl From for golem_api_grpc::proto::golem::worker::PublicEntityInvocation { + fn from(value: PublicEntityInvocation) -> Self { + use golem_api_grpc::proto::golem::worker::PublicEntityCallMode as ProtoCallMode; + + let call_mode = match value.call_mode { + PublicEntityCallMode::Synchronous => ProtoCallMode::Synchronous, + PublicEntityCallMode::Asynchronous => ProtoCallMode::Asynchronous, + PublicEntityCallMode::FireAndForget => ProtoCallMode::FireAndForget, + }; + Self { + entity: Some(value.entity.into()), + start_index: Some(value.start_index.into()), + call_mode: call_mode as i32, + operation: value.operation.map(Into::into), + } + } +} + +impl TryFrom for PublicAgentEntity { + type Error = String; + + fn try_from( + value: golem_api_grpc::proto::golem::worker::PublicAgentEntity, + ) -> Result { + use golem_api_grpc::proto::golem::worker::PublicAgentEntityKind as ProtoKind; + + let kind = match ProtoKind::try_from(value.kind) + .map_err(|_| format!("Invalid PublicAgentEntityKind: {}", value.kind))? + { + ProtoKind::Tool => PublicAgentEntityKind::Tool, + ProtoKind::ToolMiddleware => PublicAgentEntityKind::ToolMiddleware, + ProtoKind::Unspecified => return Err("Unspecified PublicAgentEntityKind".to_string()), + }; + Ok(Self { + kind, + name: value.name, + }) + } +} + +impl From for golem_api_grpc::proto::golem::worker::PublicAgentEntity { + fn from(value: PublicAgentEntity) -> Self { + use golem_api_grpc::proto::golem::worker::PublicAgentEntityKind as ProtoKind; + + let kind = match value.kind { + PublicAgentEntityKind::Tool => ProtoKind::Tool, + PublicAgentEntityKind::ToolMiddleware => ProtoKind::ToolMiddleware, + }; + Self { + kind: kind as i32, + name: value.name, + } + } +} + +impl TryFrom + for PublicEntityInvocationOperation +{ + type Error = String; + + fn try_from( + value: golem_api_grpc::proto::golem::worker::PublicEntityInvocationOperation, + ) -> Result { + use golem_api_grpc::proto::golem::worker::public_entity_invocation_operation::Operation; + + match value + .operation + .ok_or("Missing entity invocation operation")? + { + Operation::Tool(tool) => Ok(Self::Tool(PublicToolInvocationOperation { + command_path: tool.command_path, + has_stdin: tool.has_stdin, + has_stdout: tool.has_stdout, + declares_stdout: tool.declares_stdout, + })), + } + } +} + +impl From + for golem_api_grpc::proto::golem::worker::PublicEntityInvocationOperation +{ + fn from(value: PublicEntityInvocationOperation) -> Self { + use golem_api_grpc::proto::golem::worker::public_entity_invocation_operation::Operation; + + let operation = match value { + PublicEntityInvocationOperation::Tool(tool) => Operation::Tool( + golem_api_grpc::proto::golem::worker::PublicToolInvocationOperation { + command_path: tool.command_path, + has_stdin: tool.has_stdin, + has_stdout: tool.has_stdout, + declares_stdout: tool.declares_stdout, + }, + ), + }; + Self { + operation: Some(operation), + } + } +} + impl From for golem_api_grpc::proto::golem::worker::RetryPolicyState { fn from(value: PublicRetryPolicyState) -> Self { use golem_api_grpc::proto::golem::worker::retry_policy_state::State; @@ -3007,6 +3219,7 @@ impl TryFrom for OplogEntry { }), PublicOplogEntry::Error(error) => Ok(OplogEntry::Error { timestamp: error.timestamp, + entity_parent_start_index: None, error: AgentError::Unknown(error.error), retry_from: error.retry_from, inside_atomic_region: error.inside_atomic_region, @@ -3014,9 +3227,11 @@ impl TryFrom for OplogEntry { }), PublicOplogEntry::NoOp(p) => Ok(OplogEntry::NoOp { timestamp: p.timestamp, + entity_parent_start_index: None, }), PublicOplogEntry::Jump(jump) => Ok(OplogEntry::Jump { timestamp: jump.timestamp, + entity_parent_start_index: None, jump: jump.jump, }), PublicOplogEntry::Interrupted(p) => Ok(OplogEntry::Interrupted { @@ -3026,11 +3241,15 @@ impl TryFrom for OplogEntry { timestamp: p.timestamp, }), PublicOplogEntry::BeginAtomicRegion(p) => { - Ok(OplogEntry::BeginAtomicRegion { timestamp: p.timestamp }) + Ok(OplogEntry::BeginAtomicRegion { + timestamp: p.timestamp, + entity_parent_start_index: None, + }) } PublicOplogEntry::EndAtomicRegion(p) => { Ok(OplogEntry::EndAtomicRegion { timestamp: p.timestamp, + entity_parent_start_index: None, begin_index: p.begin_index, }) } @@ -3066,6 +3285,7 @@ impl TryFrom for OplogEntry { }), PublicOplogEntry::CreateResource(p) => Ok(OplogEntry::CreateResource { timestamp: p.timestamp, + entity_parent_start_index: None, id: p.id, resource_type_id: ResourceTypeId { owner: p.owner, @@ -3074,6 +3294,7 @@ impl TryFrom for OplogEntry { }), PublicOplogEntry::DropResource(p) => Ok(OplogEntry::DropResource { timestamp: p.timestamp, + entity_parent_start_index: None, id: p.id, resource_type_id: ResourceTypeId { owner: p.owner, @@ -3227,14 +3448,17 @@ impl TryFrom for OplogEntry { } PublicOplogEntry::SetRetryPolicy(p) => Ok(OplogEntry::SetRetryPolicy { timestamp: p.timestamp, + entity_parent_start_index: None, policy: p.policy.into(), }), PublicOplogEntry::RemoveRetryPolicy(p) => Ok(OplogEntry::RemoveRetryPolicy { timestamp: p.timestamp, + entity_parent_start_index: None, name: p.name, }), PublicOplogEntry::CardRevoked(p) => Ok(OplogEntry::CardRevoked { timestamp: p.timestamp, + entity_parent_start_index: None, queued_event_index: p.queued_event_index, card_id: p.card_id, wallet_generation: p.wallet_generation, @@ -3262,12 +3486,14 @@ impl TryFrom for OplogEntry { } PublicOplogEntry::CardInstallFailed(p) => Ok(OplogEntry::CardInstallFailed { timestamp: p.timestamp, + entity_parent_start_index: None, queued_event_index: p.queued_event_index, card_id: p.card_id, reason: p.reason, }), PublicOplogEntry::CardExpired(p) => Ok(OplogEntry::CardExpired { timestamp: p.timestamp, + entity_parent_start_index: None, card_id: p.card_id, wallet_generation: p.wallet_generation, }), @@ -3286,6 +3512,7 @@ impl TryFrom for OplogEntry { use crate::schema::FromSchema as _; Ok(OplogEntry::StreamRegistered { timestamp: p.timestamp, + entity_parent_start_index: None, record: OplogPayload::Inline(Box::new(StreamRegisteredRecordV1::from_value( p.record.value(), ).map_err(|error| error.to_string())?)), @@ -3295,6 +3522,7 @@ impl TryFrom for OplogEntry { use crate::schema::FromSchema as _; Ok(OplogEntry::StreamItems { timestamp: p.timestamp, + entity_parent_start_index: None, record: OplogPayload::Inline(Box::new(StreamItemsRecordV1::from_value( p.record.value(), ).map_err(|error| error.to_string())?)), @@ -3304,6 +3532,7 @@ impl TryFrom for OplogEntry { use crate::schema::FromSchema as _; Ok(OplogEntry::StreamEnd { timestamp: p.timestamp, + entity_parent_start_index: None, record: OplogPayload::Inline(Box::new(StreamEndRecordV1::from_value( p.record.value(), ).map_err(|error| error.to_string())?)), @@ -3313,6 +3542,7 @@ impl TryFrom for OplogEntry { use crate::schema::FromSchema as _; Ok(OplogEntry::StreamCancel { timestamp: p.timestamp, + entity_parent_start_index: None, record: OplogPayload::Inline(Box::new(StreamCancelRecordV1::from_value( p.record.value(), ).map_err(|error| error.to_string())?)), @@ -3322,6 +3552,7 @@ impl TryFrom for OplogEntry { use crate::schema::FromSchema as _; Ok(OplogEntry::StreamSession { timestamp: p.timestamp, + entity_parent_start_index: None, record: OplogPayload::Inline(Box::new(StreamSessionRecordV1::from_value( p.record.value(), ).map_err(|error| error.to_string())?)), @@ -3707,6 +3938,9 @@ impl TryFrom for golem_api_grpc::proto::golem::worker::RawOplogEntry }; let timestamp = value.timestamp(); + let entity_parent_start_index = value + .entity_parent_start_index() + .map(|index| index.as_u64()); let proto_ts: prost_types::Timestamp = timestamp.into(); let entry = match value { @@ -4075,23 +4309,25 @@ impl TryFrom for golem_api_grpc::proto::golem::worker::RawOplogEntry queued_event_index, card_id, wallet_generation, + .. } => Entry::CardRevoked(RawCardRevokedParameters { timestamp: Some(timestamp.into()), queued_event_index: queued_event_index.into(), card_id: Some(card_id.0.into()), wallet_generation, }), - OplogEntry::CardEventQueued { timestamp, event } => { - Entry::CardEventQueued(RawCardEventQueuedParameters { - timestamp: Some(timestamp.into()), - event: Some(raw_queued_card_event_to_proto(event)?), - }) - } + OplogEntry::CardEventQueued { + timestamp, event, .. + } => Entry::CardEventQueued(RawCardEventQueuedParameters { + timestamp: Some(timestamp.into()), + event: Some(raw_queued_card_event_to_proto(event)?), + }), OplogEntry::CardInstalled { timestamp, queued_event_index, card, wallet_generation, + .. } => Entry::CardInstalled(RawCardInstalledParameters { timestamp: Some(timestamp.into()), queued_event_index: queued_event_index.map(Into::into), @@ -4103,6 +4339,7 @@ impl TryFrom for golem_api_grpc::proto::golem::worker::RawOplogEntry queued_event_index, card_id, reason, + .. } => Entry::CardInstallFailed(RawCardInstallFailedParameters { timestamp: Some(timestamp.into()), queued_event_index: queued_event_index.into(), @@ -4113,6 +4350,7 @@ impl TryFrom for golem_api_grpc::proto::golem::worker::RawOplogEntry timestamp, card, wallet_generation, + .. } => Entry::CardDerived(RawCardDerivedParameters { timestamp: Some(timestamp.into()), card: crate::serialization::serialize(&card)?, @@ -4125,6 +4363,7 @@ impl TryFrom for golem_api_grpc::proto::golem::worker::RawOplogEntry source_holder, target_holder, source_wallet_generation, + .. } => Entry::CardTransferStarted(RawCardTransferStartedParameters { timestamp: Some(timestamp.into()), transfer_id: Some(transfer_id.into()), @@ -4141,6 +4380,7 @@ impl TryFrom for golem_api_grpc::proto::golem::worker::RawOplogEntry target_holder, card, target_wallet_generation, + .. } => { if card.card_id() != installed_card_id { return Err( @@ -4162,6 +4402,7 @@ impl TryFrom for golem_api_grpc::proto::golem::worker::RawOplogEntry revoked_card_ids, affected_wallets, local_wallet_generation, + .. } => Entry::CardRevokedCascade(RawCardRevokedCascadeParameters { timestamp: Some(timestamp.into()), revoked_card_ids: revoked_card_ids.into_iter().map(|id| id.0.into()).collect(), @@ -4178,6 +4419,7 @@ impl TryFrom for golem_api_grpc::proto::golem::worker::RawOplogEntry source_card_id, installed_card_id, target_holder, + .. } => Entry::CardTransferConfirmed(RawCardTransferConfirmedParameters { timestamp: Some(timestamp.into()), transfer_id: Some(transfer_id.into()), @@ -4189,6 +4431,7 @@ impl TryFrom for golem_api_grpc::proto::golem::worker::RawOplogEntry timestamp, card_id, wallet_generation, + .. } => Entry::CardExpired(RawCardExpiredParameters { timestamp: Some(timestamp.into()), card_id: Some(card_id.0.into()), @@ -4233,6 +4476,7 @@ impl TryFrom for golem_api_grpc::proto::golem::worker::RawOplogEntry Ok(golem_api_grpc::proto::golem::worker::RawOplogEntry { timestamp: Some(proto_ts), + entity_parent_start_index, entry: Some(entry), }) } @@ -4246,6 +4490,11 @@ impl TryFrom for OplogEntry ) -> Result { use golem_api_grpc::proto::golem::worker::raw_oplog_entry::Entry; + let entity_parent_start_index = match value.entity_parent_start_index { + Some(0) => return Err("Invalid entity parent Start index: 0".to_string()), + Some(index) => Some(OplogIndex::from_u64(index)), + None => None, + }; let timestamp: crate::model::Timestamp = value .timestamp .ok_or("Missing timestamp in RawOplogEntry")? @@ -4395,17 +4644,22 @@ impl TryFrom for OplogEntry .transpose()?; Ok(OplogEntry::Error { timestamp, + entity_parent_start_index, error, retry_from, inside_atomic_region: p.inside_atomic_region, retry_policy_state, }) } - Entry::NoOp(_) => Ok(OplogEntry::NoOp { timestamp }), + Entry::NoOp(_) => Ok(OplogEntry::NoOp { + timestamp, + entity_parent_start_index, + }), Entry::Jump(p) => { let jump = p.jump.ok_or("Missing jump region")?; Ok(OplogEntry::Jump { timestamp, + entity_parent_start_index, jump: crate::model::regions::OplogRegion { start: OplogIndex::from_u64(jump.start), end: OplogIndex::from_u64(jump.end), @@ -4414,9 +4668,13 @@ impl TryFrom for OplogEntry } Entry::Interrupted(_) => Ok(OplogEntry::Interrupted { timestamp }), Entry::Exited(_) => Ok(OplogEntry::Exited { timestamp }), - Entry::BeginAtomicRegion(_) => Ok(OplogEntry::BeginAtomicRegion { timestamp }), + Entry::BeginAtomicRegion(_) => Ok(OplogEntry::BeginAtomicRegion { + timestamp, + entity_parent_start_index, + }), Entry::EndAtomicRegion(p) => Ok(OplogEntry::EndAtomicRegion { timestamp, + entity_parent_start_index, begin_index: OplogIndex::from_u64(p.begin_index), }), Entry::PendingAgentInvocation(p) => { @@ -4479,6 +4737,7 @@ impl TryFrom for OplogEntry let rt = p.resource_type_id.ok_or("Missing resource_type_id")?; Ok(OplogEntry::CreateResource { timestamp, + entity_parent_start_index, id: AgentResourceId(p.id), resource_type_id: ResourceTypeId { name: rt.name, @@ -4490,6 +4749,7 @@ impl TryFrom for OplogEntry let rt = p.resource_type_id.ok_or("Missing resource_type_id")?; Ok(OplogEntry::DropResource { timestamp, + entity_parent_start_index, id: AgentResourceId(p.id), resource_type_id: ResourceTypeId { name: rt.name, @@ -4651,30 +4911,39 @@ impl TryFrom for OplogEntry Entry::SetRetryPolicy(p) => { let policy: crate::model::retry_policy::NamedRetryPolicy = p.policy.ok_or("Missing policy")?.try_into()?; - Ok(OplogEntry::SetRetryPolicy { timestamp, policy }) + Ok(OplogEntry::SetRetryPolicy { + timestamp, + entity_parent_start_index, + policy, + }) } Entry::RemoveRetryPolicy(p) => Ok(OplogEntry::RemoveRetryPolicy { timestamp, + entity_parent_start_index, name: p.name, }), Entry::CardRevoked(p) => Ok(OplogEntry::CardRevoked { timestamp: p.timestamp.map(Into::into).unwrap_or(timestamp), + entity_parent_start_index, queued_event_index: OplogIndex::from_u64(p.queued_event_index), card_id: CardId(p.card_id.ok_or("Missing card_id")?.into()), wallet_generation: p.wallet_generation, }), Entry::CardEventQueued(p) => Ok(OplogEntry::CardEventQueued { timestamp: p.timestamp.map(Into::into).unwrap_or(timestamp), + entity_parent_start_index, event: raw_queued_card_event_from_proto(p.event.ok_or("Missing event")?)?, }), Entry::CardInstalled(p) => Ok(OplogEntry::CardInstalled { timestamp: p.timestamp.map(Into::into).unwrap_or(timestamp), + entity_parent_start_index, queued_event_index: p.queued_event_index.map(OplogIndex::from_u64), card: deserialize_stored_card(&p.card, "installed card")?, wallet_generation: p.wallet_generation, }), Entry::CardInstallFailed(p) => Ok(OplogEntry::CardInstallFailed { timestamp: p.timestamp.map(Into::into).unwrap_or(timestamp), + entity_parent_start_index, queued_event_index: OplogIndex::from_u64(p.queued_event_index), card_id: CardId(p.card_id.ok_or("Missing card_id")?.into()), reason: raw_card_install_failure_from_proto( @@ -4684,11 +4953,13 @@ impl TryFrom for OplogEntry }), Entry::CardDerived(p) => Ok(OplogEntry::CardDerived { timestamp: p.timestamp.map(Into::into).unwrap_or(timestamp), + entity_parent_start_index, card: deserialize_stored_card(&p.card, "derived card")?, wallet_generation: p.wallet_generation, }), Entry::CardTransferStarted(p) => Ok(OplogEntry::CardTransferStarted { timestamp: p.timestamp.map(Into::into).unwrap_or(timestamp), + entity_parent_start_index, transfer_id: p.transfer_id.ok_or("Missing transfer_id")?.into(), card_id: CardId(p.card_id.ok_or("Missing card_id")?.into()), source_holder: p.source_holder.map(card_holder_from_proto).transpose()?, @@ -4711,6 +4982,7 @@ impl TryFrom for OplogEntry } Ok(OplogEntry::CardTransferred { timestamp: p.timestamp.map(Into::into).unwrap_or(timestamp), + entity_parent_start_index, transfer_id: p.transfer_id.ok_or("Missing transfer_id")?.into(), source_card_id: p.source_card_id.map(|id| CardId(id.into())), installed_card_id, @@ -4723,6 +4995,7 @@ impl TryFrom for OplogEntry } Entry::CardRevokedCascade(p) => Ok(OplogEntry::CardRevokedCascade { timestamp: p.timestamp.map(Into::into).unwrap_or(timestamp), + entity_parent_start_index, revoked_card_ids: p .revoked_card_ids .into_iter() @@ -4737,6 +5010,7 @@ impl TryFrom for OplogEntry }), Entry::CardTransferConfirmed(p) => Ok(OplogEntry::CardTransferConfirmed { timestamp: p.timestamp.map(Into::into).unwrap_or(timestamp), + entity_parent_start_index, transfer_id: p.transfer_id.ok_or("Missing transfer_id")?.into(), source_card_id: CardId(p.source_card_id.ok_or("Missing source_card_id")?.into()), installed_card_id: CardId( @@ -4750,6 +5024,7 @@ impl TryFrom for OplogEntry }), Entry::CardExpired(p) => Ok(OplogEntry::CardExpired { timestamp: p.timestamp.map(Into::into).unwrap_or(timestamp), + entity_parent_start_index, card_id: CardId(p.card_id.ok_or("Missing card_id")?.into()), wallet_generation: p.wallet_generation, }), @@ -4761,22 +5036,27 @@ impl TryFrom for OplogEntry }), Entry::StreamRegistered(p) => Ok(OplogEntry::StreamRegistered { timestamp, + entity_parent_start_index, record: oplog_payload_from_proto(p.record.ok_or("Missing record")?)?, }), Entry::StreamItems(p) => Ok(OplogEntry::StreamItems { timestamp, + entity_parent_start_index, record: oplog_payload_from_proto(p.record.ok_or("Missing record")?)?, }), Entry::StreamEnd(p) => Ok(OplogEntry::StreamEnd { timestamp, + entity_parent_start_index, record: oplog_payload_from_proto(p.record.ok_or("Missing record")?)?, }), Entry::StreamCancel(p) => Ok(OplogEntry::StreamCancel { timestamp, + entity_parent_start_index, record: oplog_payload_from_proto(p.record.ok_or("Missing record")?)?, }), Entry::StreamSession(p) => Ok(OplogEntry::StreamSession { timestamp, + entity_parent_start_index, record: oplog_payload_from_proto(p.record.ok_or("Missing record")?)?, }), } diff --git a/golem-common/src/model/oplog/tests.rs b/golem-common/src/model/oplog/tests.rs index 34ecd9ef70..209812ae95 100644 --- a/golem-common/src/model/oplog/tests.rs +++ b/golem-common/src/model/oplog/tests.rs @@ -42,11 +42,14 @@ use crate::model::oplog::{ AgentInitializationParameters, AgentInvocationOutputParameters, AgentMethodInvocationParameters, AgentResourceId, AttributeMap, DurableFunctionType, JsonSnapshotData, LogLevel, MultipartPartData, MultipartSnapshotData, MultipartSnapshotPart, - OplogEntry, OplogPayload, PluginInstallationDescription, PublicAgentInvocation, - PublicAgentInvocationResult, PublicAttribute, PublicAttributeValue, PublicDurableFunctionType, - PublicLocalSpanData, PublicOplogEntry, PublicQueuedCardEvent, PublicSnapshotData, - PublicSpanData, PublicTypedAgentConfigEntry, PublicUpdateDescription, QueuedCardEvent, - RawSnapshotData, SnapshotBasedUpdateParameters, StringAttributeValue, + OplogEntry, OplogPayload, PluginInstallationDescription, PublicAgentEntity, + PublicAgentEntityKind, PublicAgentInvocation, PublicAgentInvocationResult, PublicAttribute, + PublicAttributeValue, PublicDurableFunctionType, PublicEntityCallMode, PublicEntityInvocation, + PublicEntityInvocationContext, PublicEntityInvocationOperation, PublicLocalSpanData, + PublicOplogEntry, PublicOplogEntryAttribution, PublicOplogEntryWithIndex, + PublicQueuedCardEvent, PublicSnapshotData, PublicSpanData, PublicToolInvocationOperation, + PublicTypedAgentConfigEntry, PublicUpdateDescription, QueuedCardEvent, RawSnapshotData, + SnapshotBasedUpdateParameters, StringAttributeValue, }; use crate::model::regions::OplogRegion; use crate::model::{ @@ -97,11 +100,93 @@ fn observational_start_public_protobuf_roundtrip() { assert_eq!(PublicOplogEntry::try_from(proto).unwrap(), entry); } +#[test] +fn entity_attribution_public_protobuf_and_json_roundtrip() { + let ancestor = PublicEntityInvocation { + entity: PublicAgentEntity { + kind: PublicAgentEntityKind::ToolMiddleware, + name: "audit".to_string(), + }, + start_index: OplogIndex::from_u64(7), + call_mode: PublicEntityCallMode::Synchronous, + operation: None, + }; + let invocation = PublicEntityInvocation { + entity: PublicAgentEntity { + kind: PublicAgentEntityKind::Tool, + name: "lookup".to_string(), + }, + start_index: OplogIndex::from_u64(11), + call_mode: PublicEntityCallMode::Asynchronous, + operation: Some(PublicEntityInvocationOperation::Tool( + PublicToolInvocationOperation { + command_path: vec!["bin".to_string(), "lookup".to_string()], + has_stdin: false, + has_stdout: true, + declares_stdout: true, + }, + )), + }; + let entry = PublicOplogEntryWithIndex { + oplog_index: OplogIndex::from_u64(13), + attribution: PublicOplogEntryAttribution::entity(PublicEntityInvocationContext { + invocation, + ancestors: vec![ancestor], + }), + entry: PublicOplogEntry::NoOp(NoOpParams { + timestamp: Timestamp::now_utc().rounded(), + }), + }; + + let proto: golem_api_grpc::proto::golem::worker::OplogEntryWithIndex = + entry.clone().try_into().unwrap(); + let decoded: PublicOplogEntryWithIndex = proto.try_into().unwrap(); + assert_eq!(decoded, entry); + + let json = serde_json::to_value(&entry).unwrap(); + assert_eq!(json["attribution"]["type"], "Entity"); + assert_eq!( + json["attribution"]["ancestors"][0]["entity"]["kind"], + "toolMiddleware" + ); + assert_eq!( + json["attribution"]["invocation"]["operation"]["type"], + "Tool" + ); +} + +#[test] +fn entity_attribution_public_protobuf_rejects_invalid_start_index() { + let invocation = PublicEntityInvocation { + entity: PublicAgentEntity { + kind: PublicAgentEntityKind::Tool, + name: "lookup".to_string(), + }, + start_index: OplogIndex::from_u64(11), + call_mode: PublicEntityCallMode::Synchronous, + operation: None, + }; + let mut proto: golem_api_grpc::proto::golem::worker::PublicEntityInvocation = invocation.into(); + + proto.start_index = None; + assert_eq!( + PublicEntityInvocation::try_from(proto.clone()).unwrap_err(), + "Missing entity start index" + ); + + proto.start_index = Some(0); + assert_eq!( + PublicEntityInvocation::try_from(proto).unwrap_err(), + "Invalid entity start index: 0" + ); +} + #[test] fn entity_attribution_raw_protobuf_roundtrip() { let parent_start_index = Some(OplogIndex::from_u64(17)); let span_id = SpanId::generate(); let entries = vec![ + OplogEntry::no_op(parent_start_index), OplogEntry::Log { timestamp: Timestamp::now_utc().rounded(), parent_start_index, @@ -139,6 +224,14 @@ fn entity_attribution_raw_protobuf_roundtrip() { entry.clone().try_into().unwrap(); assert_eq!(OplogEntry::try_from(proto).unwrap(), entry); } + + let mut invalid: golem_api_grpc::proto::golem::worker::RawOplogEntry = + OplogEntry::no_op(parent_start_index).try_into().unwrap(); + invalid.entity_parent_start_index = Some(0); + assert_eq!( + OplogEntry::try_from(invalid).unwrap_err(), + "Invalid entity parent Start index: 0" + ); } /// Build a single-root [`TypedSchemaValue`] fixture from an anonymous schema @@ -1301,28 +1394,33 @@ fn phase_five_raw_card_oplog_entries_protobuf_roundtrip() { let entries = vec![ OplogEntry::CardInstalled { timestamp, + entity_parent_start_index: None, queued_event_index: None, card: source_card.clone().into(), wallet_generation: Some(1), }, OplogEntry::CardDerived { timestamp, + entity_parent_start_index: None, card: source_card.clone().into(), wallet_generation: Some(3), }, OplogEntry::CardRevoked { timestamp, + entity_parent_start_index: None, queued_event_index: OplogIndex::from_u64(1), card_id: source_card_id, wallet_generation: Some(4), }, OplogEntry::CardExpired { timestamp, + entity_parent_start_index: None, card_id: installed_card_id, wallet_generation: Some(5), }, OplogEntry::CardTransferStarted { timestamp, + entity_parent_start_index: None, transfer_id, card_id: source_card_id, source_holder: Some(source_holder.clone()), @@ -1331,6 +1429,7 @@ fn phase_five_raw_card_oplog_entries_protobuf_roundtrip() { }, OplogEntry::CardTransferred { timestamp, + entity_parent_start_index: None, transfer_id, source_card_id: Some(source_card_id), installed_card_id, @@ -1340,12 +1439,14 @@ fn phase_five_raw_card_oplog_entries_protobuf_roundtrip() { }, OplogEntry::CardRevokedCascade { timestamp, + entity_parent_start_index: None, revoked_card_ids: vec![source_card_id, installed_card_id], affected_wallets: vec![source_holder.clone(), target_holder.clone()], local_wallet_generation: Some(8), }, OplogEntry::CardTransferConfirmed { timestamp, + entity_parent_start_index: None, transfer_id, source_card_id, installed_card_id, @@ -1353,10 +1454,12 @@ fn phase_five_raw_card_oplog_entries_protobuf_roundtrip() { }, OplogEntry::CardEventQueued { timestamp, + entity_parent_start_index: None, event: QueuedCardEvent::transfer_started(transfer_id, source_card, application_holder), }, OplogEntry::CardTransferStarted { timestamp, + entity_parent_start_index: None, transfer_id: Uuid::new_v4(), card_id: installed_card_id, source_holder: None, @@ -1554,11 +1657,12 @@ fn remove_retry_policy_serialization_poem_serde_equivalence() { } mod scope_scan { + use crate::model::card::CardId; use crate::model::invocation_context::SpanId; use crate::model::oplog::host_functions::HostFunctionName; use crate::model::oplog::raw_types::PayloadId; use crate::model::oplog::{ - AttributeMap, DurableFunctionType, LogLevel, OplogEntry, OplogPayload, + AgentError, AttributeMap, DurableFunctionType, LogLevel, OplogEntry, OplogPayload, OplogScopeProjection, ScopeScanState, }; use crate::model::regions::OplogRegion; @@ -1641,6 +1745,64 @@ mod scope_scan { assert_eq!(projected, vec![10, 12, 13, 15, 17, 18]); } + #[test] + fn scope_projection_uses_entity_anchor_instead_of_retry_grouping() { + let entries = [ + (10, start(None, DurableFunctionType::WriteLocal)), + ( + 11, + OplogEntry::error( + Some(idx(10)), + AgentError::TransientError("entity-owned".to_string()), + idx(99), + false, + None, + ), + ), + ( + 12, + OplogEntry::error( + None, + AgentError::TransientError("agent-owned".to_string()), + idx(10), + false, + None, + ), + ), + (13, OplogEntry::no_op(Some(idx(10)))), + (14, OplogEntry::no_op(None)), + ( + 15, + OplogEntry::CardExpired { + timestamp: Timestamp::now_utc(), + entity_parent_start_index: Some(idx(10)), + card_id: CardId::new(), + wallet_generation: None, + }, + ), + ( + 16, + OplogEntry::StreamSession { + timestamp: Timestamp::now_utc(), + entity_parent_start_index: Some(idx(10)), + record: OplogPayload::External { + payload_id: PayloadId::new(), + md5_hash: vec![0; 16], + cached: None, + }, + }, + ), + ]; + let mut projection = OplogScopeProjection::new(idx(10)); + + let projected = entries + .iter() + .filter_map(|(index, entry)| projection.includes(idx(*index), entry).then_some(*index)) + .collect::>(); + + assert_eq!(projected, vec![10, 11, 13, 15, 16]); + } + #[test] fn scope_projection_includes_attributed_logs_spans_and_transaction_markers() { let span_id = SpanId::generate(); @@ -1749,6 +1911,7 @@ mod scope_scan { 12, OplogEntry::Jump { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, jump: OplogRegion { start: idx(11), end: idx(12), @@ -1815,22 +1978,27 @@ mod scope_scan { let entries = [ OplogEntry::StreamRegistered { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, record: external_payload!(), }, OplogEntry::StreamItems { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, record: external_payload!(), }, OplogEntry::StreamEnd { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, record: external_payload!(), }, OplogEntry::StreamCancel { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, record: external_payload!(), }, OplogEntry::StreamSession { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, record: external_payload!(), }, ]; diff --git a/golem-common/src/schema/render/cli_text.rs b/golem-common/src/schema/render/cli_text.rs index f543b9d10b..8337a70e36 100644 --- a/golem-common/src/schema/render/cli_text.rs +++ b/golem-common/src/schema/render/cli_text.rs @@ -303,7 +303,26 @@ pub fn value_to_cli_text( ty: &SchemaType, value: &SchemaValue, ) -> Result { - let mut renderer = CliTextRenderer { redact: true }; + let mut renderer = CliTextRenderer { + capability_rendering: CapabilityRendering::Redacted, + }; + drive(walk(&mut renderer, graph, ty, value)) +} + +/// Render a [`SchemaValue`] while exposing secret-handle metadata. +/// +/// Secret values contain only their ID, config path, pinned revision, resolution timestamp and +/// category; plaintext secret material is never carried by [`SchemaValue`]. Other host-managed +/// capabilities remain redacted. This is intended for trusted observability surfaces such as the +/// public oplog. +pub fn value_to_cli_text_with_secret_metadata( + graph: &SchemaGraph, + ty: &SchemaType, + value: &SchemaValue, +) -> Result { + let mut renderer = CliTextRenderer { + capability_rendering: CapabilityRendering::SecretMetadata, + }; drive(walk(&mut renderer, graph, ty, value)) } @@ -318,12 +337,21 @@ pub fn value_to_cli_text_unredacted( ty: &SchemaType, value: &SchemaValue, ) -> Result { - let mut renderer = CliTextRenderer { redact: false }; + let mut renderer = CliTextRenderer { + capability_rendering: CapabilityRendering::Unredacted, + }; drive(walk(&mut renderer, graph, ty, value)) } +#[derive(Clone, Copy)] +enum CapabilityRendering { + Redacted, + SecretMetadata, + Unredacted, +} + struct CliTextRenderer { - redact: bool, + capability_rendering: CapabilityRendering, } impl SchemaWalker for CliTextRenderer { @@ -364,14 +392,24 @@ fn render_value( ty: &SchemaType, value: &SchemaValue, ) -> Result { - if r.redact - && let (Some(type_kind), Some(value_kind)) = ( - HostManagedKind::from_type(ty), - HostManagedKind::from_value(value), - ) - && type_kind == value_kind + if let (Some(type_kind), Some(value_kind)) = ( + HostManagedKind::from_type(ty), + HostManagedKind::from_value(value), + ) && type_kind == value_kind { - return Ok(type_kind.redacted_placeholder().to_string()); + match r.capability_rendering { + CapabilityRendering::Redacted => { + return Ok(type_kind.redacted_placeholder().to_string()); + } + CapabilityRendering::SecretMetadata => { + if let SchemaValue::Secret(secret) = value { + let metadata = canonical::secret::to_json(secret)?; + return Ok(format!("")); + } + return Ok(type_kind.redacted_placeholder().to_string()); + } + CapabilityRendering::Unredacted => {} + } } match (ty, value) { diff --git a/golem-common/src/schema/render/mod.rs b/golem-common/src/schema/render/mod.rs index e7b6bb19af..92b5dd7cf7 100644 --- a/golem-common/src/schema/render/mod.rs +++ b/golem-common/src/schema/render/mod.rs @@ -30,7 +30,10 @@ pub mod walker; #[cfg(test)] mod tests; -pub use cli_text::{type_to_cli_text, value_to_cli_text, value_to_cli_text_unredacted}; +pub use cli_text::{ + type_to_cli_text, value_to_cli_text, value_to_cli_text_unredacted, + value_to_cli_text_with_secret_metadata, +}; pub use docs::graph_to_markdown; pub use error::RenderError; pub use json_schema::{ diff --git a/golem-common/src/schema/render/tests/cli_text_tests.rs b/golem-common/src/schema/render/tests/cli_text_tests.rs index be851369ee..c571594acf 100644 --- a/golem-common/src/schema/render/tests/cli_text_tests.rs +++ b/golem-common/src/schema/render/tests/cli_text_tests.rs @@ -16,6 +16,7 @@ use crate::schema::canonical; use crate::schema::graph::SchemaGraph; use crate::schema::render::cli_text::{ type_to_cli_text, value_to_cli_text, value_to_cli_text_unredacted, + value_to_cli_text_with_secret_metadata, }; use crate::schema::schema_type::{ NamedFieldType, PermissionCardSpec, QuotaTokenSpec, SchemaType, SecretSpec, TextRestrictions, @@ -122,6 +123,76 @@ fn secret_value_is_redacted_by_default() { assert!(unredacted.starts_with("secret:")); } +#[test] +fn secret_metadata_rendering_keeps_other_capabilities_redacted() { + let ty = SchemaType::record(vec![ + NamedFieldType { + name: "credential".to_string(), + body: SchemaType::secret(SecretSpec::default()), + metadata: Default::default(), + }, + NamedFieldType { + name: "quota".to_string(), + body: SchemaType::quota_token(QuotaTokenSpec::default()), + metadata: Default::default(), + }, + NamedFieldType { + name: "permission".to_string(), + body: SchemaType::permission_card(PermissionCardSpec { polymorphic: true }), + metadata: Default::default(), + }, + ]); + let graph = SchemaGraph::anonymous(ty.clone()); + let secret_id = uuid::Uuid::from_u128(1); + let permission_card_id = uuid::Uuid::from_u128(2); + let value = SchemaValue::Record { + fields: vec![ + SchemaValue::Secret(SecretValuePayload { + secret_id, + config_key: Some(vec!["database".to_string(), "password".to_string()]), + version: 7, + resolved_at: Utc.timestamp_opt(1_700_000_000, 0).unwrap(), + category: Some("api-key".to_string()), + }), + SchemaValue::QuotaToken(QuotaTokenValuePayload { + environment_id: uuid::Uuid::nil().into(), + resource_name: "private-quota-resource".to_string(), + expected_use: 1, + last_credit: 0, + last_credit_at: Utc.timestamp_opt(1_700_000_001, 0).unwrap(), + }), + SchemaValue::PermissionCard(PermissionCardValuePayload { + card_id: permission_card_id, + parent_ids: vec![], + expires_at: None, + polymorphic: true, + }), + ], + }; + + let rendered = value_to_cli_text_with_secret_metadata(&graph, &ty, &value) + .expect("observability rendering must succeed"); + + for expected in [ + "", + "", + ] { + assert!( + rendered.contains(expected), + "expected {expected:?} in {rendered:?}" + ); + } + assert!(!rendered.contains("private-quota-resource")); + assert!(!rendered.contains(&permission_card_id.to_string())); +} + #[test] fn quota_token_value_is_redacted_by_default() { let ty = SchemaType::quota_token(QuotaTokenSpec { diff --git a/golem-debugging-service/src/services/debug_service.rs b/golem-debugging-service/src/services/debug_service.rs index a68c18b8ad..0970516c69 100644 --- a/golem-debugging-service/src/services/debug_service.rs +++ b/golem-debugging-service/src/services/debug_service.rs @@ -1089,6 +1089,7 @@ mod tests { fn noop_entry() -> OplogEntry { OplogEntry::NoOp { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, } } @@ -1118,6 +1119,7 @@ mod tests { fn jump_entry(start: u64, end: u64) -> OplogEntry { OplogEntry::Jump { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, jump: OplogRegion { start: OplogIndex::from_u64(start), end: OplogIndex::from_u64(end), @@ -1399,6 +1401,7 @@ mod tests { // Any other oplog entry other than export function completed OplogEntry::NoOp { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, } } } diff --git a/golem-debugging-service/tests/debug_tests.rs b/golem-debugging-service/tests/debug_tests.rs index 7be5d61e78..f7e9624c2b 100644 --- a/golem-debugging-service/tests/debug_tests.rs +++ b/golem-debugging-service/tests/debug_tests.rs @@ -561,7 +561,7 @@ async fn test_playback_with_overrides( if let Some(PublicOplogEntryWithIndex { entry: PublicOplogEntry::AgentInvocationFinished(completed), - oplog_index: _, + .. }) = entry { assert_eq!(completed.result, original_result); diff --git a/golem-worker-executor-test-utils/src/dsl_impl.rs b/golem-worker-executor-test-utils/src/dsl_impl.rs index fa5aa51f7c..27ec07baba 100644 --- a/golem-worker-executor-test-utils/src/dsl_impl.rs +++ b/golem-worker-executor-test-utils/src/dsl_impl.rs @@ -47,7 +47,7 @@ use golem_common::model::component::{ }; use golem_common::model::deployment::DeploymentRevision; use golem_common::model::environment::EnvironmentId; -use golem_common::model::oplog::{PublicOplogEntry, PublicOplogEntryWithIndex}; +use golem_common::model::oplog::PublicOplogEntryWithIndex; use golem_common::model::tool::{ToolBindingInput, ToolName}; use golem_common::model::worker::{ AgentConfigEntryDto, AgentFileSystemNode, AgentMetadataDto, RevertWorkerTarget, @@ -715,17 +715,7 @@ impl TestDsl for TestWorkerExecutor { chunk .entries .into_iter() - .enumerate() - .map(|(chunk_idx, entry)| { - PublicOplogEntry::try_from(entry).map( - |public_oplog_entry| PublicOplogEntryWithIndex { - entry: public_oplog_entry, - oplog_index: OplogIndex::from_u64( - chunk.first_index_in_chunk + chunk_idx as u64, - ), - }, - ) - }) + .map(PublicOplogEntryWithIndex::try_from) .collect::, _>>() .map_err(|err| { anyhow!("Failed to convert oplog entry: {err}") diff --git a/golem-worker-executor-test-utils/src/lib.rs b/golem-worker-executor-test-utils/src/lib.rs index 842727214e..132119f8fa 100644 --- a/golem-worker-executor-test-utils/src/lib.rs +++ b/golem-worker-executor-test-utils/src/lib.rs @@ -673,6 +673,7 @@ impl TestWorkerExecutor { .ok_or_else(|| anyhow!("worker is not loaded: {owned_agent_id}"))?; Ok(worker .add_to_oplog(OplogEntry::card_event_queued( + None, golem_common::base_model::oplog::QueuedCardEvent::revoke(card_id), )) .await) @@ -691,6 +692,7 @@ impl TestWorkerExecutor { .ok_or_else(|| anyhow!("worker is not loaded: {owned_agent_id}"))?; worker .add_and_commit_oplog(OplogEntry::card_event_queued( + None, golem_common::base_model::oplog::QueuedCardEvent::install(card), )) .await; @@ -3600,7 +3602,7 @@ impl Oplog for TestOplog { .additional_test_deps .take_no_op_oplog_read(&self.owned_agent_id.agent_id, oplog_index) { - return OplogEntry::no_op(); + return OplogEntry::no_op(None); } self.oplog.read(oplog_index).await } diff --git a/golem-worker-executor/benches/oplog_read.rs b/golem-worker-executor/benches/oplog_read.rs index b92e8675f7..0ee3e40ef7 100644 --- a/golem-worker-executor/benches/oplog_read.rs +++ b/golem-worker-executor/benches/oplog_read.rs @@ -113,6 +113,7 @@ fn execution_status() -> read_only_lock::std::ReadOnlyLock { fn entry(value: u64) -> OplogEntry { OplogEntry::Error { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, error: AgentError::Unknown(value.to_string()), retry_from: OplogIndex::NONE, inside_atomic_region: false, diff --git a/golem-worker-executor/src/durable_host/call_coordinator.rs b/golem-worker-executor/src/durable_host/call_coordinator.rs index ee5e0b16c5..49d956247d 100644 --- a/golem-worker-executor/src/durable_host/call_coordinator.rs +++ b/golem-worker-executor/src/durable_host/call_coordinator.rs @@ -428,8 +428,12 @@ where }; match &pending_event.event { QueuedCardEvent::Revoke(_) => { + let entity_parent_start_index = pending_event.entity_parent_start_index; let card_ids = pending_events .into_iter() + .filter(|pending_event| { + pending_event.entity_parent_start_index == entity_parent_start_index + }) .filter_map(|pending_event| match pending_event.event { QueuedCardEvent::Revoke(event) => Some(event.card_id), QueuedCardEvent::Install(_) @@ -437,7 +441,8 @@ where | QueuedCardEvent::TransferReceived(_) => None, }) .collect::>(); - apply_card_revocations_access(store, get_ctx, card_ids).await?; + apply_card_revocations_access(store, get_ctx, entity_parent_start_index, card_ids) + .await?; } QueuedCardEvent::Install(event) => { let Some(card) = event.card.clone() else { @@ -445,7 +450,14 @@ where "queued card install is missing card payload", )); }; - apply_card_install_access(store, get_ctx, pending_event.oplog_index, card).await?; + apply_card_install_access( + store, + get_ctx, + pending_event.entity_parent_start_index, + pending_event.oplog_index, + card, + ) + .await?; } QueuedCardEvent::TransferReceived(event) => { let Some(card) = event.card.clone() else { @@ -456,6 +468,7 @@ where apply_received_card_transfer_access( store, get_ctx, + pending_event.entity_parent_start_index, pending_event.oplog_index, event.transfer_id, event.source_card_id, @@ -477,7 +490,15 @@ where chrono::Utc::now(), ) }); - apply_card_revocations_access(store, get_ctx, expired_scope_root_ids).await + let entity_parent_start_index = + store.with(|mut access| get_ctx(access.data_mut()).entity_parent_start_index()); + apply_card_revocations_access( + store, + get_ctx, + entity_parent_start_index, + expired_scope_root_ids, + ) + .await } async fn pending_card_events_at_boundary_access( @@ -609,6 +630,7 @@ where async fn apply_card_install_access( store: &Accessor, get_ctx: fn(&mut T) -> &mut DurableWorkerCtx, + entity_parent_start_index: Option, queued_event_index: OplogIndex, card: golem_common::model::card::StoredCard, ) -> Result<(), WorkerExecutorError> @@ -621,10 +643,18 @@ where let result = admit_card_to_wallet_access(store, get_ctx, &card).await?; let worker = store.with(|mut access| get_ctx(access.data_mut()).public_state.worker().clone()); let entry = match result { - Ok(wallet_generation) => { - OplogEntry::card_installed(Some(queued_event_index), card, Some(wallet_generation)) - } - Err(reason) => OplogEntry::card_install_failed(queued_event_index, card_id, reason), + Ok(wallet_generation) => OplogEntry::card_installed( + entity_parent_start_index, + Some(queued_event_index), + card, + Some(wallet_generation), + ), + Err(reason) => OplogEntry::card_install_failed( + entity_parent_start_index, + queued_event_index, + card_id, + reason, + ), }; worker.add_and_commit_oplog(entry).await; Ok(()) @@ -633,6 +663,7 @@ where async fn apply_received_card_transfer_access( store: &Accessor, get_ctx: fn(&mut T) -> &mut DurableWorkerCtx, + entity_parent_start_index: Option, queued_event_index: OplogIndex, transfer_id: uuid::Uuid, source_card_id: Option, @@ -654,6 +685,7 @@ where }); let entry = match result { Ok(wallet_generation) => OplogEntry::card_transferred( + entity_parent_start_index, transfer_id, source_card_id, card_id, @@ -663,7 +695,12 @@ where card, Some(wallet_generation), ), - Err(reason) => OplogEntry::card_install_failed(queued_event_index, card_id, reason), + Err(reason) => OplogEntry::card_install_failed( + entity_parent_start_index, + queued_event_index, + card_id, + reason, + ), }; worker.add_and_commit_oplog(entry).await; Ok(()) @@ -678,34 +715,41 @@ where D: HasData + ?Sized, Ctx: WorkerCtx, { - let (expired_card_generations, owned_agent_id, interested_card_ids, interest_index, worker) = - store.with(|mut access| -> Result<_, WorkerExecutorError> { - let ctx = get_ctx(access.data_mut()); - let expired_card_ids = crate::durable_host::expired_wallet_card_ids_at( - &ctx.state.agent_wallet_cards, - chrono::Utc::now(), - ); - let mut expired_card_generations = Vec::with_capacity(expired_card_ids.len()); - for card_id in expired_card_ids { - if crate::durable_host::remove_wallet_card( - &mut ctx.state.agent_wallet_cards, - &mut ctx.state.wallet_generation, - card_id, - )? { - expired_card_generations.push((card_id, ctx.state.wallet_generation)); - } - } - if !expired_card_generations.is_empty() { - ctx.rederive_agent_effective_surface_from_wallet(); + let ( + expired_card_generations, + owned_agent_id, + interested_card_ids, + interest_index, + worker, + entity_parent_start_index, + ) = store.with(|mut access| -> Result<_, WorkerExecutorError> { + let ctx = get_ctx(access.data_mut()); + let expired_card_ids = crate::durable_host::expired_wallet_card_ids_at( + &ctx.state.agent_wallet_cards, + chrono::Utc::now(), + ); + let mut expired_card_generations = Vec::with_capacity(expired_card_ids.len()); + for card_id in expired_card_ids { + if crate::durable_host::remove_wallet_card( + &mut ctx.state.agent_wallet_cards, + &mut ctx.state.wallet_generation, + card_id, + )? { + expired_card_generations.push((card_id, ctx.state.wallet_generation)); } - Ok(( - expired_card_generations, - ctx.owned_agent_id.clone(), - ctx.interested_card_ids(), - ctx.state.card_interest_index.clone(), - ctx.public_state.worker().clone(), - )) - })?; + } + if !expired_card_generations.is_empty() { + ctx.rederive_agent_effective_surface_from_wallet(); + } + Ok(( + expired_card_generations, + ctx.owned_agent_id.clone(), + ctx.interested_card_ids(), + ctx.state.card_interest_index.clone(), + ctx.public_state.worker().clone(), + ctx.entity_parent_start_index(), + )) + })?; if expired_card_generations.is_empty() { return Ok(()); @@ -715,7 +759,11 @@ where .await; for (card_id, wallet_generation) in expired_card_generations { worker - .add_and_commit_oplog(OplogEntry::card_expired(card_id, Some(wallet_generation))) + .add_and_commit_oplog(OplogEntry::card_expired( + entity_parent_start_index, + card_id, + Some(wallet_generation), + )) .await; } Ok(()) @@ -1081,6 +1129,7 @@ where worker .add_and_commit_oplog(OplogEntry::card_transfer_started( + retry.entity_parent_start_index(), retry.transfer_id, retry.source_card_id, Some(golem_common::model::card::CardHolder::Agent( @@ -1167,6 +1216,7 @@ where } worker .add_and_commit_oplog(OplogEntry::card_transfer_confirmed( + retry.entity_parent_start_index(), retry.transfer_id, retry.source_card_id, retry.installed_card.card_id(), @@ -1179,6 +1229,7 @@ where async fn apply_card_revocations_access( store: &Accessor, get_ctx: fn(&mut T) -> &mut DurableWorkerCtx, + entity_parent_start_index: Option, mut card_ids: Vec, ) -> Result<(), WorkerExecutorError> where @@ -1235,6 +1286,7 @@ where worker .add_and_commit_oplog(OplogEntry::CardRevokedCascade { timestamp: Timestamp::now_utc(), + entity_parent_start_index, revoked_card_ids: card_ids, affected_wallets, local_wallet_generation: Some(wallet_generation), diff --git a/golem-worker-executor/src/durable_host/concurrent/call.rs b/golem-worker-executor/src/durable_host/concurrent/call.rs index cad1115ac2..d8d5af3996 100644 --- a/golem-worker-executor/src/durable_host/concurrent/call.rs +++ b/golem-worker-executor/src/durable_host/concurrent/call.rs @@ -198,6 +198,7 @@ impl Drop for LiveCallPermit { #[derive(Debug, Clone)] pub(super) struct BegunCallExecutionScope { + pub(super) entity_parent_start_index: Option, /// The durable scope this host-call `Start` will be nested under, if any. This is derived from /// the call's own function type / begin index, never from temporally-open sibling scopes. pub(super) parent_start_index: Option, @@ -216,6 +217,7 @@ impl BegunCallExecutionScope { atomic_lease: Option>, ) -> CallExecutionScope { CallExecutionScope { + entity_parent_start_index: self.entity_parent_start_index, retry_from: self .observational_owner .or(self.parent_start_index) @@ -229,6 +231,7 @@ impl BegunCallExecutionScope { #[derive(Debug, Clone)] pub(super) struct CallExecutionScope { + pub(super) entity_parent_start_index: Option, /// The retry point owned by this in-flight call: the enclosing durable scope `Start` if present, /// otherwise the host-call `Start` itself. pub(super) retry_from: OplogIndex, @@ -1477,6 +1480,7 @@ impl DurableCallSession { custom_invocation_scope, )?; let execution_scope = BegunCallExecutionScope { + entity_parent_start_index: ctx.entity_parent_start_index(), parent_start_index, atomic_region, observational_owner, @@ -2403,7 +2407,10 @@ impl DurableCallSession { prepared .public_state .worker() - .add_and_commit_oplog(OplogEntry::jump(deleted_region)) + .add_and_commit_oplog(OplogEntry::jump( + prepared.entity_parent_start_index, + deleted_region, + )) .await; prepared .public_state @@ -2644,6 +2651,7 @@ impl DurableCallSession { let begin_index = boundary.begin_index(); let durable_execution_state = InFunctionRetryHost::durable_execution_state(ctx); let execution_scope = BegunCallExecutionScope { + entity_parent_start_index: ctx.entity_parent_start_index(), parent_start_index: ctx.child_parent_start_index(&function_type, begin_index), atomic_region: ctx .state @@ -2963,6 +2971,7 @@ impl DurableCallSession { let mut retry_host = TaskRetryContext { retry_point, + entity_parent_start_index: self.execution_scope.entity_parent_start_index, environment_state_service, environment_id, default_retry_policy, diff --git a/golem-worker-executor/src/durable_host/concurrent/tests.rs b/golem-worker-executor/src/durable_host/concurrent/tests.rs index 6be294b041..c893f05273 100644 --- a/golem-worker-executor/src/durable_host/concurrent/tests.rs +++ b/golem-worker-executor/src/durable_host/concurrent/tests.rs @@ -193,6 +193,7 @@ fn live_unfinished_handle_with_atomic_region( retry_from: start_idx, durable_scope: None, observational_owner: None, + entity_parent_start_index: None, atomic_lease: unregistered_atomic_lease(atomic_region, true), }, retry: InFunctionRetryController::new( @@ -465,6 +466,7 @@ async fn completion_delivery_markers_preserve_handoff_order() { seed_oplog .add(OplogEntry::NoOp { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, }) .await; let seed_oplog_dyn: Arc = seed_oplog; @@ -678,6 +680,7 @@ async fn tail_gated_token_over_crash_tail( oplog .add(OplogEntry::NoOp { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, }) .await; oplog @@ -853,6 +856,7 @@ async fn completion_delivery_ordered_append_lands_before_marker() { let mut token = live_delivery_token(oplog.clone(), counter.clone(), tx).await; token.append_ordered(OplogEntry::NoOp { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, }); drop(token); } @@ -1491,6 +1495,7 @@ fn scoped_retry_host_uses_call_retry_point_not_inner_current() { retry_from: idx(42), durable_scope: Some(idx(40)), observational_owner: None, + entity_parent_start_index: None, atomic_lease: None, }; @@ -1507,6 +1512,7 @@ fn scoped_retry_host_uses_call_atomic_region_as_retry_point() { retry_from: idx(42), durable_scope: Some(idx(40)), observational_owner: None, + entity_parent_start_index: None, atomic_lease: unregistered_atomic_lease(Some(idx(7)), true), }; @@ -1523,6 +1529,7 @@ async fn scoped_retry_host_trap_retry_uses_call_retry_point() { retry_from: idx(42), durable_scope: Some(idx(40)), observational_owner: None, + entity_parent_start_index: None, atomic_lease: None, }; let mut retry_host = ScopedRetryHost::new(&mut inner, &scope); @@ -1547,12 +1554,14 @@ async fn seam2_overlapping_semantic_traps_carry_independent_retry_points() { retry_from: idx(42), durable_scope: Some(idx(40)), observational_owner: None, + entity_parent_start_index: None, atomic_lease: None, }; let scope_b = CallExecutionScope { retry_from: idx(77), durable_scope: Some(idx(70)), observational_owner: None, + entity_parent_start_index: None, atomic_lease: None, }; @@ -1594,12 +1603,14 @@ async fn seam2_overlapping_atomic_region_traps_use_initiation_membership() { retry_from: idx(42), durable_scope: Some(idx(40)), observational_owner: None, + entity_parent_start_index: None, atomic_lease: unregistered_atomic_lease(Some(idx(7)), true), }; let scope_b = CallExecutionScope { retry_from: idx(77), durable_scope: Some(idx(70)), observational_owner: None, + entity_parent_start_index: None, atomic_lease: unregistered_atomic_lease(Some(idx(8)), true), }; @@ -1661,6 +1672,7 @@ fn seam2_terminal_failure_carries_call_owned_trap_context() { retry_from: idx(42), durable_scope: Some(idx(40)), observational_owner: None, + entity_parent_start_index: None, atomic_lease: None, }; let handle = synthetic_finished_handle_with_scope::(scope); @@ -1695,12 +1707,14 @@ fn seam2_overlapping_terminal_failures_carry_independent_trap_contexts() { retry_from: idx(42), durable_scope: Some(idx(40)), observational_owner: None, + entity_parent_start_index: None, atomic_lease: unregistered_atomic_lease(Some(idx(7)), true), }; let scope_b = CallExecutionScope { retry_from: idx(77), durable_scope: Some(idx(70)), observational_owner: None, + entity_parent_start_index: None, atomic_lease: None, }; let handle_a = synthetic_finished_handle_with_scope::(scope_a); @@ -1810,6 +1824,7 @@ fn begun_execution_scope_uses_parent_scope_as_retry_from() { parent_start_index: Some(idx(10)), atomic_region: Some(idx(2)), observational_owner: None, + entity_parent_start_index: None, }; let lease = unregistered_atomic_lease(begun.atomic_region, true); @@ -1826,6 +1841,7 @@ fn begun_execution_scope_uses_call_start_as_retry_from_when_unscoped() { parent_start_index: None, atomic_region: None, observational_owner: None, + entity_parent_start_index: None, }; let scope = begun.finish(idx(12), None); @@ -1841,6 +1857,7 @@ fn begun_observational_scope_uses_custom_owner_as_retry_from() { parent_start_index: Some(idx(10)), atomic_region: Some(idx(2)), observational_owner: Some(idx(7)), + entity_parent_start_index: None, }; let lease = unregistered_atomic_lease(begun.atomic_region, true); @@ -1859,6 +1876,7 @@ fn call_execution_scope_owns_call_retry_point() { retry_from: idx(42), durable_scope: Some(idx(40)), observational_owner: None, + entity_parent_start_index: None, atomic_lease: None, }; diff --git a/golem-worker-executor/src/durable_host/durability.rs b/golem-worker-executor/src/durable_host/durability.rs index 8c063a4a77..4be4cd1689 100644 --- a/golem-worker-executor/src/durable_host/durability.rs +++ b/golem-worker-executor/src/durable_host/durability.rs @@ -1863,6 +1863,7 @@ impl InFunctionRetryHost for DurableWorkerCtx { use golem_common::model::oplog::AgentError; let entry = OplogEntry::error( + self.entity_parent_start_index(), AgentError::TransientError("in-function retry".to_string()), retry_from, inside_atomic_region, @@ -2305,6 +2306,8 @@ pub async fn count_oplog_errors_for( pub struct TaskRetryContext { /// The oplog index that error entries reference as their `retry_from` point. pub retry_point: OplogIndex, + /// Entity invocation that initiated this task, if any. + pub entity_parent_start_index: Option, /// Environment state service for lazy policy fetching pub environment_state_service: Arc, /// Environment ID for policy lookup @@ -2375,6 +2378,7 @@ impl InFunctionRetryHost for TaskRetryContext { ) { use golem_common::model::oplog::AgentError; let entry = OplogEntry::error( + self.entity_parent_start_index, AgentError::TransientError("in-function retry".to_string()), retry_from, inside_atomic_region, diff --git a/golem-worker-executor/src/durable_host/durable_session.rs b/golem-worker-executor/src/durable_host/durable_session.rs index 5978472a70..5d6c48efed 100644 --- a/golem-worker-executor/src/durable_host/durable_session.rs +++ b/golem-worker-executor/src/durable_host/durable_session.rs @@ -56,6 +56,7 @@ use golem_common::base_model::durable_stream::{ }; use golem_common::base_model::oplog::OplogEntry; use golem_common::model::Timestamp; +use golem_common::model::oplog::OplogIndex; use golem_common::model::oplog::payload::OplogPayload; use golem_schema::schema::wit::{encode_value_with_streams, wire}; use golem_schema::schema::{SchemaFingerprintV1, SchemaGraph, SchemaType, schema_fingerprint_v1}; @@ -104,6 +105,7 @@ pub(crate) struct DurableSessionStreams { session_lock: Arc>, attachment_epoch: u64, attachment_attempt_id: Option, + entity_parent_start_index: Option, } struct DurableInputSchema { @@ -324,6 +326,7 @@ impl DurableSessionStreams { session_lock, attachment_epoch: 1, attachment_attempt_id: None, + entity_parent_start_index: None, } } @@ -341,6 +344,14 @@ impl DurableSessionStreams { self } + pub(crate) fn with_entity_parent_start_index( + mut self, + entity_parent_start_index: Option, + ) -> Self { + self.entity_parent_start_index = entity_parent_start_index; + self + } + pub(crate) fn with_rpc(mut self, rpc: Arc) -> Self { self.rpc = Some(rpc); self @@ -630,7 +641,10 @@ impl DurableSessionStreams { } let result = self .producer - .append_session_record(StreamSessionRecordV1::ResumeAttempt(record)) + .append_session_record_attributed( + self.entity_parent_start_index, + StreamSessionRecordV1::ResumeAttempt(record), + ) .await .map_err(|error| error.to_string()); if result.is_ok() { @@ -674,14 +688,14 @@ impl DurableSessionStreams { pub(crate) async fn append_record(&self, record: StreamSessionRecordV1) { self.producer - .append_session_record(record) + .append_session_record_attributed(self.entity_parent_start_index, record) .await .expect("internally generated durable session record is valid"); } async fn try_append_record(&self, record: StreamSessionRecordV1) -> Result<(), String> { self.producer - .append_session_record(record) + .append_session_record_attributed(self.entity_parent_start_index, record) .await .map_err(|error| error.to_string()) } @@ -1617,6 +1631,7 @@ impl DurableSessionStreams { nested_element_types .push((nested_transport_id, element.unwrap_or_else(SchemaType::u8))); nested_requests.push(ProducerRegistrationRequestV1 { + entity_parent_start_index: self.entity_parent_start_index, coordinate, source_invocation: self.session_key.clone(), component_revision: input_schema.component_revision, @@ -2009,6 +2024,7 @@ impl DurableSessionStreams { handle } else { let request = ProducerRegistrationRequestV1 { + entity_parent_start_index: self.entity_parent_start_index, coordinate: StreamRegistrationCoordinateV1::Root { invocation_id: self.session_key.clone(), root_kind: StreamRootKindV1::MethodInput, @@ -2157,6 +2173,7 @@ impl DurableSessionStreams { .iter() .filter(|pending| pending.forwarded_handle.is_none()) .map(|pending| ProducerRegistrationRequestV1 { + entity_parent_start_index: self.entity_parent_start_index, coordinate: StreamRegistrationCoordinateV1::Root { invocation_id: self.session_key.clone(), root_kind: StreamRootKindV1::MethodResult, @@ -2226,37 +2243,43 @@ impl DurableSessionStreams { .collect::>(); let (owned_handles, _) = self .producer - .register_result_streams(requests, move |owned_handles| { - let mut owned_handles = owned_handles.into_iter(); - let handles = forwarded_handles_for_record - .into_iter() - .map(|forwarded_handle| { - forwarded_handle.unwrap_or_else(|| { - owned_handles - .next() - .expect("result registration returned too few durable handles") + .register_result_streams( + requests, + self.entity_parent_start_index, + move |owned_handles| { + let mut owned_handles = owned_handles.into_iter(); + let handles = forwarded_handles_for_record + .into_iter() + .map(|forwarded_handle| { + forwarded_handle.unwrap_or_else(|| { + owned_handles + .next() + .expect("result registration returned too few durable handles") + }) }) - }) - .collect::>(); - let stream_mappings = transport_stream_ids_for_record - .into_iter() - .zip(handles.iter().cloned()) - .map( - |(transport_stream_id, handle)| StreamSessionMappingRecordV1 { - transport_stream_id, - handle, - role: golem_common::model::durable_stream::SessionStreamRoleV1::Output, + .collect::>(); + let stream_mappings = transport_stream_ids_for_record + .into_iter() + .zip(handles.iter().cloned()) + .map( + |(transport_stream_id, handle)| StreamSessionMappingRecordV1 { + transport_stream_id, + handle, + role: golem_common::model::durable_stream::SessionStreamRoleV1::Output, + }, + ) + .collect(); + StreamSessionRecordV1::InvocationResult( + StreamSessionInvocationResultRecordV1 { + format_version: DURABLE_STREAM_FORMAT_VERSION, + session_key: result_session_key, + result: result_bytes, + output_streams: handles, + stream_mappings, }, ) - .collect(); - StreamSessionRecordV1::InvocationResult(StreamSessionInvocationResultRecordV1 { - format_version: DURABLE_STREAM_FORMAT_VERSION, - session_key: result_session_key, - result: result_bytes, - output_streams: handles, - stream_mappings, - }) - }) + }, + ) .await .map_err(|error| error.to_string())?; self.producer.notify_session_records_changed(); @@ -2693,6 +2716,7 @@ impl DurableSessionStreams { forwarded_handle, element_type: nested_element.unwrap_or_else(SchemaType::u8), registration: ProducerRegistrationRequestV1 { + entity_parent_start_index: self.entity_parent_start_index, coordinate: StreamRegistrationCoordinateV1::Nested { parent_stream_id: handle.stream_id, parent_producer_sequence: event.offset, @@ -2880,7 +2904,12 @@ impl DurableSessionStreams { self.validate_topology_complete().await?; drop(session_guard); self.producer - .finish_session(self.session_key.clone(), result, input_cancel_reason) + .finish_session( + self.session_key.clone(), + self.entity_parent_start_index, + result, + input_cancel_reason, + ) .await .map_err(|error| error.to_string()) } diff --git a/golem-worker-executor/src/durable_host/durable_stream.rs b/golem-worker-executor/src/durable_host/durable_stream.rs index f93d68d222..4b4024ab94 100644 --- a/golem-worker-executor/src/durable_host/durable_stream.rs +++ b/golem-worker-executor/src/durable_host/durable_stream.rs @@ -65,6 +65,7 @@ use tokio_util::sync::CancellationToken; pub(crate) struct ProducerRegistrationRequestV1 { pub(crate) coordinate: StreamRegistrationCoordinateV1, pub(crate) source_invocation: StreamInvocationIdV1, + pub(crate) entity_parent_start_index: Option, pub(crate) component_revision: ComponentRevision, pub(crate) element_schema_fingerprint: SchemaFingerprintV1, pub(crate) source_kind: StreamSourceKindV1, @@ -192,6 +193,7 @@ impl From for DurableStreamProducerError { #[derive(Clone, Default)] struct ProducerStreamIndex { registrations: HashMap, + entity_parent_start_indices: HashMap>, referenced_handles: HashMap)>, coordinates: HashMap, streams: HashMap, @@ -200,6 +202,7 @@ struct ProducerStreamIndex { session_stream_mappings: HashMap>, session_stream_counts: HashMap, + session_entity_parent_start_indices: HashMap>, finished_sessions: HashSet, attachments: HashMap<(AttachmentId, StreamId), IndexedStreamAttachment>, cascade_outbox: HashMap, @@ -272,7 +275,74 @@ struct IndexedProducerStream { terminal: bool, } +fn stream_session_record_key(record: &StreamSessionRecordV1) -> Option<&StreamSessionKeyV1> { + match record { + StreamSessionRecordV1::CallerAttempt(record) => Some(&record.session_key), + StreamSessionRecordV1::Prepared(record) => Some(&record.attempt.session_key), + StreamSessionRecordV1::Attached(record) => Some(&record.session_key), + StreamSessionRecordV1::ResumeAttempt(record) => Some(&record.attempt.session_key), + StreamSessionRecordV1::Detached(record) => Some(&record.session_key), + StreamSessionRecordV1::Mapping(record) => Some(&record.session_key), + StreamSessionRecordV1::AttachmentPrepared(record) => Some(&record.key.session_key), + StreamSessionRecordV1::AttachmentActivated(record) => Some(&record.key.session_key), + StreamSessionRecordV1::AttachmentRenewed(record) => Some(&record.key.session_key), + StreamSessionRecordV1::AttachmentFinalized(record) => Some(&record.key.session_key), + StreamSessionRecordV1::ProducerDeleting(_) => None, + StreamSessionRecordV1::CascadeOutbox(record) => Some(&record.key.session_key), + StreamSessionRecordV1::ConsumerDeleting(_) => None, + StreamSessionRecordV1::SourceUnavailable(record) => Some(&record.key.session_key), + StreamSessionRecordV1::TopologyPrepared(record) => Some(&record.session_key), + StreamSessionRecordV1::TopologyActivated(record) => Some(&record.session_key), + StreamSessionRecordV1::InputHighWater(record) => Some(&record.session_key), + StreamSessionRecordV1::ConsumerItemValue(record) => Some(&record.session_key), + StreamSessionRecordV1::ConsumerCancelIntent(record) => Some(&record.session_key), + StreamSessionRecordV1::ConsumerTerminal(record) => Some(&record.session_key), + StreamSessionRecordV1::InvocationResult(record) => Some(&record.session_key), + StreamSessionRecordV1::Finished(record) => Some(&record.session_key), + } +} + impl ProducerStreamIndex { + fn entity_parent_start_index( + &self, + stream_id: StreamId, + ) -> Result, DurableStreamProducerError> { + self.entity_parent_start_indices + .get(&stream_id) + .copied() + .ok_or(DurableStreamProducerError::UnknownStream(stream_id)) + } + + fn session_entity_parent_start_index( + &self, + session_key: &StreamSessionKeyV1, + ) -> Option { + self.session_entity_parent_start_indices + .get(session_key) + .copied() + .flatten() + } + + fn apply_session_attribution( + &mut self, + session_key: &StreamSessionKeyV1, + entity_parent_start_index: Option, + ) -> Result<(), DurableStreamProducerError> { + match self.session_entity_parent_start_indices.get(session_key) { + Some(existing) if *existing != entity_parent_start_index => { + Err(DurableStreamProducerError::CorruptHistory( + "durable stream session contains conflicting entity attribution".to_string(), + )) + } + Some(_) => Ok(()), + None => { + self.session_entity_parent_start_indices + .insert(session_key.clone(), entity_parent_start_index); + Ok(()) + } + } + } + fn ensure_producer_write_allowed(&self) -> Result<(), DurableStreamProducerError> { if self.deleting { Err(DurableStreamProducerError::ProducerDeleting) @@ -340,6 +410,7 @@ impl ProducerStreamIndex { fn apply_session_references( &mut self, + entity_parent_start_index: Option, record: &StreamSessionRecordV1, ) -> Result<(), DurableStreamProducerError> { if self.consumer_deleting @@ -351,6 +422,9 @@ impl ProducerStreamIndex { { return Err(DurableStreamProducerError::ConsumerDeleting); } + if let Some(session_key) = stream_session_record_key(record) { + self.apply_session_attribution(session_key, entity_parent_start_index)?; + } self.apply_consumer_journal_record(record)?; let (session_key, mappings): (&StreamSessionKeyV1, &[StreamSessionMappingRecordV1]) = match record { @@ -514,6 +588,7 @@ impl ProducerStreamIndex { fn apply_registration( &mut self, oplog_index: OplogIndex, + entity_parent_start_index: Option, record: StreamRegisteredRecordV1, environment_id: EnvironmentId, producer: &AgentId, @@ -543,7 +618,14 @@ impl ProducerStreamIndex { .registrations .get(existing_id) .expect("coordinate index points at a missing stream registration"); - return if existing == &record { + let existing_entity_parent_start_index = self + .entity_parent_start_indices + .get(existing_id) + .copied() + .flatten(); + return if existing == &record + && existing_entity_parent_start_index == entity_parent_start_index + { Ok(()) } else { Err(DurableStreamProducerError::RegistrationDivergence) @@ -559,6 +641,20 @@ impl ProducerStreamIndex { "nested stream registration references an unknown parent stream".to_string(), ) })?; + if let StreamRegistrationCoordinateV1::Nested { + parent_stream_id, .. + } = &record.coordinate + && self + .entity_parent_start_indices + .get(parent_stream_id) + .copied() + .flatten() + != entity_parent_start_index + { + return Err(DurableStreamProducerError::CorruptHistory( + "nested stream attribution differs from its parent stream".to_string(), + )); + } if self.finished_sessions.contains(&session_key) { return Err(DurableStreamProducerError::CorruptHistory( "stream registration follows its session Finished record".to_string(), @@ -609,8 +705,11 @@ impl ProducerStreamIndex { { return Err(DurableStreamProducerError::StreamLimit); } + self.apply_session_attribution(&session_key, entity_parent_start_index)?; self.coordinates .insert(record.coordinate.clone(), record.handle.stream_id); + self.entity_parent_start_indices + .insert(record.handle.stream_id, entity_parent_start_index); self.streams .insert(record.handle.stream_id, IndexedProducerStream::default()); self.stream_sessions @@ -630,7 +729,8 @@ impl ProducerStreamIndex { fn apply_item_batch( &mut self, oplog_index: OplogIndex, - pending_registrations: Vec<(OplogIndex, StreamRegisteredRecordV1)>, + entity_parent_start_index: Option, + pending_registrations: Vec<(OplogIndex, Option, StreamRegisteredRecordV1)>, record: StreamItemsRecordV1, environment_id: EnvironmentId, producer: &AgentId, @@ -650,7 +750,7 @@ impl ProducerStreamIndex { ) })?; let logical_item_count = record.payload.logical_item_count() as u64; - for (position, ((registration_index, registration), expected_stream_id)) in + for (position, ((registration_index, _, registration), expected_stream_id)) in pending_registrations .iter() .zip(&record.newly_registered_stream_ids) @@ -673,16 +773,22 @@ impl ProducerStreamIndex { } } let mut updated = self.clone(); - for (registration_index, registration) in pending_registrations { + for (registration_index, entity_parent_start_index, registration) in pending_registrations { updated.apply_registration( registration_index, + entity_parent_start_index, registration, environment_id, producer, producer_fingerprint, )?; } - let events = updated.apply_items(oplog_index, record, producer_fingerprint)?; + let events = updated.apply_items( + oplog_index, + entity_parent_start_index, + record, + producer_fingerprint, + )?; *self = updated; Ok(events) } @@ -690,6 +796,7 @@ impl ProducerStreamIndex { fn apply_items( &mut self, oplog_index: OplogIndex, + entity_parent_start_index: Option, record: StreamItemsRecordV1, producer_fingerprint: AgentFingerprint, ) -> Result, DurableStreamProducerError> { @@ -697,6 +804,11 @@ impl ProducerStreamIndex { if record.producer_fingerprint != producer_fingerprint { return Err(DurableStreamProducerError::InvalidHandle); } + if self.entity_parent_start_index(record.stream_id)? != entity_parent_start_index { + return Err(DurableStreamProducerError::CorruptHistory( + "stream item attribution differs from its registration".to_string(), + )); + } validate_items_payload(&record.payload)?; let session_key = self .stream_sessions @@ -856,6 +968,7 @@ impl ProducerStreamIndex { fn apply_end( &mut self, oplog_index: OplogIndex, + entity_parent_start_index: Option, record: StreamEndRecordV1, producer_fingerprint: AgentFingerprint, ) -> Result { @@ -865,6 +978,11 @@ impl ProducerStreamIndex { { return Err(DurableStreamProducerError::InvalidHandle); } + if self.entity_parent_start_index(record.stream_id)? != entity_parent_start_index { + return Err(DurableStreamProducerError::CorruptHistory( + "stream end attribution differs from its registration".to_string(), + )); + } let stream = self .streams .get_mut(&record.stream_id) @@ -910,6 +1028,7 @@ impl ProducerStreamIndex { fn apply_cancel( &mut self, oplog_index: OplogIndex, + entity_parent_start_index: Option, record: StreamCancelRecordV1, producer_fingerprint: AgentFingerprint, ) -> Result { @@ -919,6 +1038,11 @@ impl ProducerStreamIndex { { return Err(DurableStreamProducerError::InvalidHandle); } + if self.entity_parent_start_index(record.stream_id)? != entity_parent_start_index { + return Err(DurableStreamProducerError::CorruptHistory( + "stream cancellation attribution differs from its registration".to_string(), + )); + } let stream = self .streams .get_mut(&record.stream_id) @@ -1408,7 +1532,11 @@ impl DurableStreamProducer { .await; for (oplog_index, entry) in entries { match entry { - OplogEntry::StreamRegistered { record, .. } => { + OplogEntry::StreamRegistered { + entity_parent_start_index, + record, + .. + } => { let record = oplog .download_payload(record) .await @@ -1417,7 +1545,11 @@ impl DurableStreamProducer { &record.coordinate, StreamRegistrationCoordinateV1::Nested { .. } ) { - pending_nested_registrations.push((oplog_index, record)); + pending_nested_registrations.push(( + oplog_index, + entity_parent_start_index, + record, + )); } else { if !pending_nested_registrations.is_empty() { return Err(DurableStreamProducerError::CorruptHistory( @@ -1427,6 +1559,7 @@ impl DurableStreamProducer { } index.apply_registration( oplog_index, + entity_parent_start_index, record, environment_id, &producer, @@ -1434,13 +1567,18 @@ impl DurableStreamProducer { )?; } } - OplogEntry::StreamItems { record, .. } => { + OplogEntry::StreamItems { + entity_parent_start_index, + record, + .. + } => { let record = oplog .download_payload(record) .await .map_err(DurableStreamProducerError::Oplog)?; index.apply_item_batch( oplog_index, + entity_parent_start_index, std::mem::take(&mut pending_nested_registrations), record, environment_id, @@ -1448,7 +1586,11 @@ impl DurableStreamProducer { producer_fingerprint, )?; } - OplogEntry::StreamEnd { record, .. } => { + OplogEntry::StreamEnd { + entity_parent_start_index, + record, + .. + } => { if !pending_nested_registrations.is_empty() { return Err(DurableStreamProducerError::CorruptHistory( "nested registration batch is missing its enclosing item" @@ -1459,9 +1601,18 @@ impl DurableStreamProducer { .download_payload(record) .await .map_err(DurableStreamProducerError::Oplog)?; - index.apply_end(oplog_index, record, producer_fingerprint)?; + index.apply_end( + oplog_index, + entity_parent_start_index, + record, + producer_fingerprint, + )?; } - OplogEntry::StreamCancel { record, .. } => { + OplogEntry::StreamCancel { + entity_parent_start_index, + record, + .. + } => { if !pending_nested_registrations.is_empty() { return Err(DurableStreamProducerError::CorruptHistory( "nested registration batch is missing its enclosing item" @@ -1472,9 +1623,18 @@ impl DurableStreamProducer { .download_payload(record) .await .map_err(DurableStreamProducerError::Oplog)?; - index.apply_cancel(oplog_index, record, producer_fingerprint)?; + index.apply_cancel( + oplog_index, + entity_parent_start_index, + record, + producer_fingerprint, + )?; } - OplogEntry::StreamSession { record, .. } => { + OplogEntry::StreamSession { + entity_parent_start_index, + record, + .. + } => { if !pending_nested_registrations.is_empty() { return Err(DurableStreamProducerError::CorruptHistory( "nested registration batch is missing its enclosing item" @@ -1485,7 +1645,7 @@ impl DurableStreamProducer { .download_payload(record) .await .map_err(DurableStreamProducerError::Oplog)?; - index.apply_session_references(&record)?; + index.apply_session_references(entity_parent_start_index, &record)?; index.apply_deletion_record( &record, environment_id, @@ -1590,6 +1750,14 @@ impl DurableStreamProducer { pub(crate) async fn append_session_record( &self, record: StreamSessionRecordV1, + ) -> Result<(), DurableStreamProducerError> { + self.append_session_record_attributed(None, record).await + } + + pub(crate) async fn append_session_record_attributed( + &self, + entity_parent_start_index: Option, + record: StreamSessionRecordV1, ) -> Result<(), DurableStreamProducerError> { if !record.has_supported_format() { return Err(DurableStreamProducerError::CorruptHistory( @@ -1617,7 +1785,7 @@ impl DurableStreamProducer { { return Err(DurableStreamProducerError::ConsumerDeleting); } - index.apply_session_references(&record)?; + index.apply_session_references(entity_parent_start_index, &record)?; index.apply_deletion_record( &record, self.environment_id, @@ -1625,9 +1793,10 @@ impl DurableStreamProducer { self.producer_fingerprint, )?; self.oplog - .add(OplogEntry::stream_session(OplogPayload::Inline(Box::new( - record, - )))) + .add(OplogEntry::stream_session( + entity_parent_start_index, + OplogPayload::Inline(Box::new(record)), + )) .await; self.commit().await; drop(index); @@ -1740,11 +1909,16 @@ impl DurableStreamProducer { source_offset, consumer_read_ordinal, }); - index.apply_consumer_journal_record(&record)?; + let entity_parent_start_index = index.session_entity_parent_start_index( + stream_session_record_key(&record) + .expect("source-unavailable record always identifies a session"), + ); + index.apply_session_references(entity_parent_start_index, &record)?; self.oplog - .add(OplogEntry::stream_session(OplogPayload::Inline(Box::new( - record, - )))) + .add(OplogEntry::stream_session( + entity_parent_start_index, + OplogPayload::Inline(Box::new(record)), + )) .await; self.commit().await; self.notify_session_records_changed(); @@ -1761,7 +1935,16 @@ impl DurableStreamProducer { )); } let mut index = self.index.lock().await; + let stream_id = match &record { + StreamSessionRecordV1::AttachmentPrepared(record) => record.key.stream_id, + StreamSessionRecordV1::AttachmentActivated(record) => record.key.stream_id, + StreamSessionRecordV1::AttachmentRenewed(record) => record.key.stream_id, + StreamSessionRecordV1::AttachmentFinalized(record) => record.key.stream_id, + _ => unreachable!("attachment persistence received a non-attachment record"), + }; + let entity_parent_start_index = index.entity_parent_start_index(stream_id)?; let mut updated = index.clone(); + updated.apply_session_references(entity_parent_start_index, &record)?; let outcome = updated.apply_attachment_record( &record, self.environment_id, @@ -1770,9 +1953,10 @@ impl DurableStreamProducer { )?; if outcome == AttachmentApplyOutcome::Changed { self.oplog - .add(OplogEntry::stream_session(OplogPayload::Inline(Box::new( - record, - )))) + .add(OplogEntry::stream_session( + entity_parent_start_index, + OplogPayload::Inline(Box::new(record)), + )) .await; self.commit().await; *index = updated; @@ -1850,7 +2034,9 @@ impl DurableStreamProducer { .registrations .get(stream_id) .expect("coordinate index points at a missing registration"); - if registration_matches(existing, &request) { + if registration_matches(existing, &request) + && index.entity_parent_start_index(*stream_id)? == request.entity_parent_start_index + { crate::metrics::durable_stream::record_producer_operation("register", true); tracing::debug!( stream_id = %existing.handle.stream_id, @@ -1906,17 +2092,21 @@ impl DurableStreamProducer { let environment_id = self.environment_id; let producer = self.producer.clone(); let producer_fingerprint = self.producer_fingerprint; + let entity_parent_start_index = request.entity_parent_start_index; let request_for_entry = request.clone(); let mut entries = self .oplog .add_durable_stream_batch(Box::new(move |oplog_index| { - vec![DurableStreamOplogRecord::Registered(registration_record( - oplog_index, - environment_id, - producer, - producer_fingerprint, - request_for_entry, - ))] + vec![DurableStreamOplogRecord::Registered( + entity_parent_start_index, + registration_record( + oplog_index, + environment_id, + producer, + producer_fingerprint, + request_for_entry, + ), + )] })) .await .map_err(DurableStreamProducerError::Oplog)?; @@ -1934,6 +2124,7 @@ impl DurableStreamProducer { .map_err(DurableStreamProducerError::Oplog)?; index.apply_registration( oplog_index, + entity_parent_start_index, record.clone(), self.environment_id, &self.producer, @@ -1976,7 +2167,13 @@ impl DurableStreamProducer { crate::metrics::durable_stream::record_limit_violation("streams_per_session"); return Err(DurableStreamProducerError::StreamLimit); } + let entity_parent_start_index = requests + .first() + .and_then(|(_, request)| request.entity_parent_start_index); for (_, request) in &requests { + if request.entity_parent_start_index != entity_parent_start_index { + return Err(DurableStreamProducerError::RegistrationDivergence); + } if registration_coordinate_depth(&request.coordinate) > MAX_STREAM_VALUE_TRAVERSAL_DEPTH { crate::metrics::durable_stream::record_limit_violation("traversal_depth"); @@ -2020,7 +2217,10 @@ impl DurableStreamProducer { request, ); handles.push((transport_stream_id, record.handle.clone())); - result.push(DurableStreamOplogRecord::Registered(record)); + result.push(DurableStreamOplogRecord::Registered( + entity_parent_start_index, + record, + )); } let prepared = make_prepared(handles); let prepared_record = StreamSessionRecordV1::Prepared(prepared); @@ -2043,11 +2243,15 @@ impl DurableStreamProducer { epoch: 1, pending_invocation_oplog_index, }; - result.push(DurableStreamOplogRecord::Session(Box::new(prepared_record))); + result.push(DurableStreamOplogRecord::Session( + entity_parent_start_index, + Box::new(prepared_record), + )); result.push(DurableStreamOplogRecord::InlineEntry(pending_invocation)); - result.push(DurableStreamOplogRecord::Session(Box::new( - StreamSessionRecordV1::Attached(attached), - ))); + result.push(DurableStreamOplogRecord::Session( + entity_parent_start_index, + Box::new(StreamSessionRecordV1::Attached(attached)), + )); result })) .await @@ -2057,13 +2261,17 @@ impl DurableStreamProducer { let mut registrations = Vec::with_capacity(requests.len()); for (oplog_index, entry) in entries { match entry { - OplogEntry::StreamRegistered { record, .. } => { + OplogEntry::StreamRegistered { + entity_parent_start_index, + record, + .. + } => { let record = self .oplog .download_payload(record) .await .map_err(DurableStreamProducerError::Oplog)?; - registrations.push((oplog_index, record)); + registrations.push((oplog_index, entity_parent_start_index, record)); } OplogEntry::StreamSession { record, .. } => { let record = self @@ -2097,9 +2305,10 @@ impl DurableStreamProducer { })?; let mut updated_index = index.clone(); let mut buses = Vec::with_capacity(registrations.len()); - for (oplog_index, record) in registrations { + for (oplog_index, entity_parent_start_index, record) in registrations { updated_index.apply_registration( oplog_index, + entity_parent_start_index, record.clone(), self.environment_id, &self.producer, @@ -2132,6 +2341,7 @@ impl DurableStreamProducer { pub(crate) async fn register_result_streams( &self, requests: Vec, + entity_parent_start_index: Option, make_result: impl FnOnce(Vec) -> StreamSessionRecordV1 + Send + 'static, ) -> Result<(Vec, StreamSessionRecordV1), DurableStreamProducerError> { @@ -2141,6 +2351,12 @@ impl DurableStreamProducer { crate::metrics::durable_stream::record_limit_violation("streams_per_value"); return Err(DurableStreamProducerError::ValueStreamLimit); } + if requests + .iter() + .any(|request| request.entity_parent_start_index != entity_parent_start_index) + { + return Err(DurableStreamProducerError::RegistrationDivergence); + } if requests.is_empty() { let expected = make_result(Vec::new()); let StreamSessionRecordV1::InvocationResult(expected_result) = &expected else { @@ -2192,9 +2408,10 @@ impl DurableStreamProducer { let mut entries = self .oplog .add_durable_stream_batch(Box::new(move |_| { - vec![DurableStreamOplogRecord::Session(Box::new( - expected_for_entry, - ))] + vec![DurableStreamOplogRecord::Session( + entity_parent_start_index, + Box::new(expected_for_entry), + )] })) .await .map_err(DurableStreamProducerError::Oplog)?; @@ -2214,7 +2431,10 @@ impl DurableStreamProducer { .download_payload(record) .await .map_err(DurableStreamProducerError::Oplog)?; - self.index.lock().await.apply_session_references(&record)?; + self.index + .lock() + .await + .apply_session_references(entity_parent_start_index, &record)?; return Ok((Vec::new(), record)); } let existing_handles = requests @@ -2321,13 +2541,19 @@ impl DurableStreamProducer { request, ); handles.push(record.handle.clone()); - result.push(DurableStreamOplogRecord::Registered(record)); + result.push(DurableStreamOplogRecord::Registered( + entity_parent_start_index, + record, + )); } let session_record = make_result(handles); if !session_record.has_supported_format() { return Vec::new(); } - result.push(DurableStreamOplogRecord::Session(Box::new(session_record))); + result.push(DurableStreamOplogRecord::Session( + entity_parent_start_index, + Box::new(session_record), + )); result })) .await @@ -2338,7 +2564,11 @@ impl DurableStreamProducer { let mut session_record = None; for (oplog_index, entry) in entries { match entry { - OplogEntry::StreamRegistered { record, .. } => { + OplogEntry::StreamRegistered { + entity_parent_start_index, + record, + .. + } => { let record = self .oplog .download_payload(record) @@ -2347,6 +2577,7 @@ impl DurableStreamProducer { handles.push(record.handle.clone()); index.apply_registration( oplog_index, + entity_parent_start_index, record.clone(), self.environment_id, &self.producer, @@ -2360,13 +2591,17 @@ impl DurableStreamProducer { Arc::new(DurableLiveStreamBus::new(self.live_join_capacity)?), ); } - OplogEntry::StreamSession { record, .. } => { + OplogEntry::StreamSession { + entity_parent_start_index, + record, + .. + } => { let record = self .oplog .download_payload(record) .await .map_err(DurableStreamProducerError::Oplog)?; - index.apply_session_references(&record)?; + index.apply_session_references(entity_parent_start_index, &record)?; session_record = Some(record); } _ => { @@ -2588,6 +2823,13 @@ impl DurableStreamProducer { .collect::>(); let mut index = self.index.lock().await; + let entity_parent_start_index = index.entity_parent_start_index(stream_id)?; + if nested + .iter() + .any(|request| request.entity_parent_start_index != entity_parent_start_index) + { + return Err(DurableStreamProducerError::RegistrationDivergence); + } let session_key = index .stream_sessions .get(&stream_id) @@ -2823,7 +3065,10 @@ impl DurableStreamProducer { registration.coordinate.clone(), registration.handle.stream_id, ); - records.push(DurableStreamOplogRecord::Registered(registration)); + records.push(DurableStreamOplogRecord::Registered( + entity_parent_start_index, + registration, + )); } let nested_stream_ids = nested_for_entry .iter() @@ -2844,16 +3089,19 @@ impl DurableStreamProducer { .map(|sub_index| StreamOffsetV1::new(item_index, sub_index as u32)) .collect::>(); let payload_for_high_water = payload_for_entry.clone(); - records.push(DurableStreamOplogRecord::Items(StreamItemsRecordV1 { - format_version: DURABLE_STREAM_FORMAT_VERSION, - stream_id, - producer_fingerprint, - first_sequence, - nested_stream_ids, - newly_registered_stream_ids, - payload: payload_for_entry, - offsets, - })); + records.push(DurableStreamOplogRecord::Items( + entity_parent_start_index, + StreamItemsRecordV1 { + format_version: DURABLE_STREAM_FORMAT_VERSION, + stream_id, + producer_fingerprint, + first_sequence, + nested_stream_ids, + newly_registered_stream_ids, + payload: payload_for_entry, + offsets, + }, + )); if let Some(session_key) = ingress_session_key { let logical_item_count = payload_for_high_water.logical_item_count() as u64; let resulting_offset = StreamOffsetV1::new( @@ -2861,8 +3109,9 @@ impl DurableStreamProducer { u32::try_from(logical_item_count - 1) .expect("validated stream batch length fits in u32"), ); - records.push(DurableStreamOplogRecord::Session(Box::new( - StreamSessionRecordV1::InputHighWater( + records.push(DurableStreamOplogRecord::Session( + entity_parent_start_index, + Box::new(StreamSessionRecordV1::InputHighWater( StreamSessionInputHighWaterRecordV1 { format_version: DURABLE_STREAM_FORMAT_VERSION, session_key, @@ -2878,8 +3127,8 @@ impl DurableStreamProducer { terminal: false, }, }, - ), - ))); + )), + )); } records })) @@ -2891,33 +3140,42 @@ impl DurableStreamProducer { let mut committed_item = None; for (oplog_index, entry) in entries { match entry { - OplogEntry::StreamRegistered { record, .. } => { + OplogEntry::StreamRegistered { + entity_parent_start_index, + record, + .. + } => { let record = self .oplog .download_payload(record) .await .map_err(DurableStreamProducerError::Oplog)?; - pending_registrations.push((oplog_index, record)); + pending_registrations.push((oplog_index, entity_parent_start_index, record)); } - OplogEntry::StreamItems { record, .. } => { + OplogEntry::StreamItems { + entity_parent_start_index, + record, + .. + } => { let record = self .oplog .download_payload(record) .await .map_err(DurableStreamProducerError::Oplog)?; - committed_item = Some((oplog_index, record)); + committed_item = Some((oplog_index, entity_parent_start_index, record)); } OplogEntry::StreamSession { .. } => {} _ => unreachable!("stream item batch builder returned a different entry"), } } - let (item_index, item_record) = + let (item_index, entity_parent_start_index, item_record) = committed_item.expect("stream item batch returned no item entry"); let item_offsets = item_record.offsets.clone(); let newly_registered_stream_ids = item_record.newly_registered_stream_ids.clone(); let newly_registered_stream_count = newly_registered_stream_ids.len(); let item_events = index.apply_item_batch( item_index, + entity_parent_start_index, pending_registrations, item_record, self.environment_id, @@ -2968,19 +3226,23 @@ impl DurableStreamProducer { sequence: u64, ) -> Result<(), DurableStreamProducerError> { let result = StreamEndResultV1::ErrorContext(resource_exhausted_error_context()?); + let entity_parent_start_index = index.entity_parent_start_index(stream_id)?; let producer_fingerprint = self.producer_fingerprint; let mut entries = self .oplog .add_durable_stream_batch(Box::new(move |oplog_index| { - vec![DurableStreamOplogRecord::End(StreamEndRecordV1 { - format_version: DURABLE_STREAM_FORMAT_VERSION, - stream_id, - producer_fingerprint, - sequence, - offset: StreamOffsetV1::new(oplog_index, 0), - authored_by: StreamTerminalAuthorV1::Protocol, - result, - })] + vec![DurableStreamOplogRecord::End( + entity_parent_start_index, + StreamEndRecordV1 { + format_version: DURABLE_STREAM_FORMAT_VERSION, + stream_id, + producer_fingerprint, + sequence, + offset: StreamOffsetV1::new(oplog_index, 0), + authored_by: StreamTerminalAuthorV1::Protocol, + result, + }, + )] })) .await .map_err(DurableStreamProducerError::Oplog)?; @@ -2988,7 +3250,12 @@ impl DurableStreamProducer { let (oplog_index, entry) = entries .pop() .expect("resource exhaustion terminal batch returned no oplog entry"); - let OplogEntry::StreamEnd { record, .. } = entry else { + let OplogEntry::StreamEnd { + entity_parent_start_index, + record, + .. + } = entry + else { unreachable!("resource exhaustion terminal builder returned a different entry") }; let record = self @@ -2996,7 +3263,12 @@ impl DurableStreamProducer { .download_payload(record) .await .map_err(DurableStreamProducerError::Oplog)?; - let event = index.apply_end(oplog_index, record, self.producer_fingerprint)?; + let event = index.apply_end( + oplog_index, + entity_parent_start_index, + record, + self.producer_fingerprint, + )?; drop(index); self.bus(stream_id)? .publish_committed(DurableLiveStreamEvent { @@ -3093,19 +3365,23 @@ impl DurableStreamProducer { return Err(error); } validate_new_terminal(&index, stream_id, sequence)?; + let entity_parent_start_index = index.entity_parent_start_index(stream_id)?; let producer_fingerprint = self.producer_fingerprint; let mut entries = self .oplog .add_durable_stream_batch(Box::new(move |oplog_index| { - vec![DurableStreamOplogRecord::End(StreamEndRecordV1 { - format_version: DURABLE_STREAM_FORMAT_VERSION, - stream_id, - producer_fingerprint, - sequence, - offset: StreamOffsetV1::new(oplog_index, 0), - authored_by, - result, - })] + vec![DurableStreamOplogRecord::End( + entity_parent_start_index, + StreamEndRecordV1 { + format_version: DURABLE_STREAM_FORMAT_VERSION, + stream_id, + producer_fingerprint, + sequence, + offset: StreamOffsetV1::new(oplog_index, 0), + authored_by, + result, + }, + )] })) .await .map_err(DurableStreamProducerError::Oplog)?; @@ -3113,7 +3389,12 @@ impl DurableStreamProducer { let (oplog_index, entry) = entries .pop() .expect("stream end batch returned no oplog entry"); - let OplogEntry::StreamEnd { record, .. } = entry else { + let OplogEntry::StreamEnd { + entity_parent_start_index, + record, + .. + } = entry + else { unreachable!("stream end builder returned a different entry") }; let record = self @@ -3121,7 +3402,12 @@ impl DurableStreamProducer { .download_payload(record) .await .map_err(DurableStreamProducerError::Oplog)?; - let event = index.apply_end(oplog_index, record, self.producer_fingerprint)?; + let event = index.apply_end( + oplog_index, + entity_parent_start_index, + record, + self.producer_fingerprint, + )?; let offset = event.offset; drop(index); self.bus(stream_id)? @@ -3202,21 +3488,25 @@ impl DurableStreamProducer { } index.ensure_producer_write_allowed()?; validate_new_terminal(&index, stream_id, sequence)?; + let entity_parent_start_index = index.entity_parent_start_index(stream_id)?; let producer_fingerprint = self.producer_fingerprint; let mut entries = self .oplog .add_durable_stream_batch(Box::new(move |oplog_index| { - vec![DurableStreamOplogRecord::Cancel(StreamCancelRecordV1 { - format_version: DURABLE_STREAM_FORMAT_VERSION, - stream_id, - producer_fingerprint, - sequence, - offset: StreamOffsetV1::new(oplog_index, 0), - authored_by: StreamTerminalAuthorV1::Protocol, - role, - reason, - details, - })] + vec![DurableStreamOplogRecord::Cancel( + entity_parent_start_index, + StreamCancelRecordV1 { + format_version: DURABLE_STREAM_FORMAT_VERSION, + stream_id, + producer_fingerprint, + sequence, + offset: StreamOffsetV1::new(oplog_index, 0), + authored_by: StreamTerminalAuthorV1::Protocol, + role, + reason, + details, + }, + )] })) .await .map_err(DurableStreamProducerError::Oplog)?; @@ -3224,7 +3514,12 @@ impl DurableStreamProducer { let (oplog_index, entry) = entries .pop() .expect("stream cancellation batch returned no oplog entry"); - let OplogEntry::StreamCancel { record, .. } = entry else { + let OplogEntry::StreamCancel { + entity_parent_start_index, + record, + .. + } = entry + else { unreachable!("stream cancellation builder returned a different entry") }; let record = self @@ -3232,7 +3527,12 @@ impl DurableStreamProducer { .download_payload(record) .await .map_err(DurableStreamProducerError::Oplog)?; - let event = index.apply_cancel(oplog_index, record, self.producer_fingerprint)?; + let event = index.apply_cancel( + oplog_index, + entity_parent_start_index, + record, + self.producer_fingerprint, + )?; let offset = event.offset; drop(index); self.cancel_source(stream_id); @@ -3374,6 +3674,7 @@ impl DurableStreamProducer { pub(crate) async fn finish_session( &self, session_key: StreamSessionKeyV1, + entity_parent_start_index: Option, result: Result<(), Vec>, input_cancel_reason: StreamCancelReasonV1, ) -> Result<(), DurableStreamProducerError> { @@ -3401,11 +3702,23 @@ impl DurableStreamProducer { .get(stream_id) .expect("session stream index points at a missing role"), stream.next_sequence, + *index + .entity_parent_start_indices + .get(stream_id) + .expect("session stream index points at missing attribution"), ) }) }) .collect::>(); - open_streams.sort_by_key(|(stream_id, _, _)| *stream_id); + open_streams.sort_by_key(|(stream_id, _, _, _)| *stream_id); + if open_streams + .iter() + .any(|(_, _, _, attribution)| *attribution != entity_parent_start_index) + { + return Err(DurableStreamProducerError::CorruptHistory( + "session stream attribution differs from its session".to_string(), + )); + } let producer_fingerprint = self.producer_fingerprint; let result_for_batch = result.clone(); @@ -3414,50 +3727,60 @@ impl DurableStreamProducer { .oplog .add_durable_stream_batch(Box::new(move |first_index| { let mut records = Vec::with_capacity(open_streams.len() + 1); - for (position, (stream_id, role, sequence)) in open_streams.into_iter().enumerate() + for (position, (stream_id, role, sequence, stream_attribution)) in + open_streams.into_iter().enumerate() { let oplog_index = OplogIndex::from_u64(first_index.as_u64() + position as u64); match role { SessionStreamRoleV1::Input => { - records.push(DurableStreamOplogRecord::Cancel(StreamCancelRecordV1 { - format_version: DURABLE_STREAM_FORMAT_VERSION, - stream_id, - producer_fingerprint, - sequence, - offset: StreamOffsetV1::new(oplog_index, 0), - authored_by: StreamTerminalAuthorV1::Protocol, - role: StreamCancelRoleV1::InputConsumer, - reason: input_cancel_reason, - details: Some( - "invocation finished before consuming the complete input" - .to_string(), - ), - })); + records.push(DurableStreamOplogRecord::Cancel( + stream_attribution, + StreamCancelRecordV1 { + format_version: DURABLE_STREAM_FORMAT_VERSION, + stream_id, + producer_fingerprint, + sequence, + offset: StreamOffsetV1::new(oplog_index, 0), + authored_by: StreamTerminalAuthorV1::Protocol, + role: StreamCancelRoleV1::InputConsumer, + reason: input_cancel_reason, + details: Some( + "invocation finished before consuming the complete input" + .to_string(), + ), + }, + )); } SessionStreamRoleV1::Output => { let details = match &result_for_batch { Ok(()) => b"output stream ended without a terminal".to_vec(), Err(details) => details.clone(), }; - records.push(DurableStreamOplogRecord::End(StreamEndRecordV1 { - format_version: DURABLE_STREAM_FORMAT_VERSION, - stream_id, - producer_fingerprint, - sequence, - offset: StreamOffsetV1::new(oplog_index, 0), - authored_by: StreamTerminalAuthorV1::Protocol, - result: StreamEndResultV1::ErrorContext(details), - })); + records.push(DurableStreamOplogRecord::End( + stream_attribution, + StreamEndRecordV1 { + format_version: DURABLE_STREAM_FORMAT_VERSION, + stream_id, + producer_fingerprint, + sequence, + offset: StreamOffsetV1::new(oplog_index, 0), + authored_by: StreamTerminalAuthorV1::Protocol, + result: StreamEndResultV1::ErrorContext(details), + }, + )); } } } - records.push(DurableStreamOplogRecord::Session(Box::new( - StreamSessionRecordV1::Finished(StreamSessionFinishedRecordV1 { - format_version: DURABLE_STREAM_FORMAT_VERSION, - session_key: session_key_for_batch, - result: result_for_batch, - }), - ))); + records.push(DurableStreamOplogRecord::Session( + entity_parent_start_index, + Box::new(StreamSessionRecordV1::Finished( + StreamSessionFinishedRecordV1 { + format_version: DURABLE_STREAM_FORMAT_VERSION, + session_key: session_key_for_batch, + result: result_for_batch, + }, + )), + )); records })) .await @@ -3467,7 +3790,11 @@ impl DurableStreamProducer { let mut terminal_events = Vec::new(); for (oplog_index, entry) in entries { match entry { - OplogEntry::StreamEnd { record, .. } => { + OplogEntry::StreamEnd { + entity_parent_start_index, + record, + .. + } => { let record = self .oplog .download_payload(record) @@ -3475,11 +3802,16 @@ impl DurableStreamProducer { .map_err(DurableStreamProducerError::Oplog)?; terminal_events.push(index.apply_end( oplog_index, + entity_parent_start_index, record, self.producer_fingerprint, )?); } - OplogEntry::StreamCancel { record, .. } => { + OplogEntry::StreamCancel { + entity_parent_start_index, + record, + .. + } => { let record = self .oplog .download_payload(record) @@ -3487,6 +3819,7 @@ impl DurableStreamProducer { .map_err(DurableStreamProducerError::Oplog)?; terminal_events.push(index.apply_cancel( oplog_index, + entity_parent_start_index, record, self.producer_fingerprint, )?); @@ -4824,10 +5157,17 @@ impl DurableStreamProducer { .streams .iter() .filter_map(|(stream_id, stream)| { - (!stream.terminal).then_some((*stream_id, stream.next_sequence)) + (!stream.terminal).then_some(( + *stream_id, + stream.next_sequence, + *index + .entity_parent_start_indices + .get(stream_id) + .expect("stream index points at missing attribution"), + )) }) .collect::>(); - open_streams.sort_by_key(|(stream_id, _)| *stream_id); + open_streams.sort_by_key(|(stream_id, _, _)| *stream_id); let environment_id = self.environment_id; let producer = self.producer.clone(); let producer_fingerprint = self.producer_fingerprint; @@ -4835,29 +5175,37 @@ impl DurableStreamProducer { .oplog .add_durable_stream_batch(Box::new(move |first_index| { let mut records = Vec::with_capacity(open_streams.len() + 1); - for (position, (stream_id, sequence)) in open_streams.into_iter().enumerate() { + for (position, (stream_id, sequence, entity_parent_start_index)) in + open_streams.into_iter().enumerate() + { let oplog_index = OplogIndex::from_u64(first_index.as_u64() + position as u64); - records.push(DurableStreamOplogRecord::Cancel(StreamCancelRecordV1 { - format_version: DURABLE_STREAM_FORMAT_VERSION, - stream_id, - producer_fingerprint, - sequence, - offset: StreamOffsetV1::new(oplog_index, 0), - authored_by: StreamTerminalAuthorV1::Protocol, - role: StreamCancelRoleV1::System, - reason: StreamCancelReasonV1::ProducerDeleting, - details: None, - })); + records.push(DurableStreamOplogRecord::Cancel( + entity_parent_start_index, + StreamCancelRecordV1 { + format_version: DURABLE_STREAM_FORMAT_VERSION, + stream_id, + producer_fingerprint, + sequence, + offset: StreamOffsetV1::new(oplog_index, 0), + authored_by: StreamTerminalAuthorV1::Protocol, + role: StreamCancelRoleV1::System, + reason: StreamCancelReasonV1::ProducerDeleting, + details: None, + }, + )); } - records.push(DurableStreamOplogRecord::Session(Box::new( - StreamSessionRecordV1::ProducerDeleting(StreamProducerDeletingRecordV1 { - format_version: DURABLE_STREAM_FORMAT_VERSION, - producer_environment_id: environment_id, - producer, - producer_fingerprint, - deleting_at_millis: now_millis, - }), - ))); + records.push(DurableStreamOplogRecord::Session( + None, + Box::new(StreamSessionRecordV1::ProducerDeleting( + StreamProducerDeletingRecordV1 { + format_version: DURABLE_STREAM_FORMAT_VERSION, + producer_environment_id: environment_id, + producer, + producer_fingerprint, + deleting_at_millis: now_millis, + }, + )), + )); records })) .await @@ -4866,7 +5214,11 @@ impl DurableStreamProducer { let mut terminal_events = Vec::new(); for (oplog_index, entry) in entries { match entry { - OplogEntry::StreamCancel { record, .. } => { + OplogEntry::StreamCancel { + entity_parent_start_index, + record, + .. + } => { let record = self .oplog .download_payload(record) @@ -4874,6 +5226,7 @@ impl DurableStreamProducer { .map_err(DurableStreamProducerError::Oplog)?; terminal_events.push(index.apply_cancel( oplog_index, + entity_parent_start_index, record, self.producer_fingerprint, )?); @@ -4947,6 +5300,7 @@ impl DurableStreamProducer { let attachment_id = key.attachment_id; let stream_id = key.stream_id; let epoch = key.epoch; + let entity_parent_start_index = index.entity_parent_start_index(stream_id)?; let record = StreamSessionRecordV1::CascadeOutbox(StreamCascadeOutboxRecordV1 { format_version: DURABLE_STREAM_FORMAT_VERSION, key, @@ -4954,11 +5308,13 @@ impl DurableStreamProducer { result, }); self.oplog - .add(OplogEntry::stream_session(OplogPayload::Inline(Box::new( - record.clone(), - )))) + .add(OplogEntry::stream_session( + entity_parent_start_index, + OplogPayload::Inline(Box::new(record.clone())), + )) .await; self.commit().await; + index.apply_session_references(entity_parent_start_index, &record)?; index.apply_deletion_record( &record, self.environment_id, @@ -5742,6 +6098,7 @@ pub(crate) mod tests { source_kind: StreamSourceKindV1, ) -> ProducerRegistrationRequestV1 { ProducerRegistrationRequestV1 { + entity_parent_start_index: None, coordinate, source_invocation: identity.invocation.clone(), component_revision: ComponentRevision::INITIAL, @@ -5991,6 +6348,50 @@ pub(crate) mod tests { .await } + #[test] + async fn delayed_stream_records_retain_registration_entity_attribution() { + let identity = identity(); + let oplog = Arc::new(TestOplog::default()); + let live = producer(oplog.clone(), &identity, None).await; + let entity_parent_start_index = Some(OplogIndex::from_u64(42)); + let mut request = root_registration(&identity); + request.entity_parent_start_index = entity_parent_start_index; + let handle = live.register(request).await.unwrap().value; + + oplog.add(OplogEntry::no_op(None)).await; + live.write_items(handle.stream_id, 0, StreamItemsPayloadV1::PackedU8(vec![1])) + .await + .unwrap(); + live.prepare_attachment(attachment_key(&identity, handle.stream_id), 100) + .await + .unwrap(); + live.end(handle.stream_id, 1, StreamEndResultV1::Ok) + .await + .unwrap(); + + let attributed = oplog + .entries() + .into_iter() + .filter(|entry| { + matches!( + entry, + OplogEntry::StreamRegistered { .. } + | OplogEntry::StreamItems { .. } + | OplogEntry::StreamEnd { .. } + | OplogEntry::StreamSession { .. } + ) + }) + .collect::>(); + assert_eq!(attributed.len(), 4); + assert!( + attributed + .iter() + .all(|entry| { entry.entity_parent_start_index() == entity_parent_start_index }) + ); + + producer(oplog, &identity, None).await; + } + #[test] async fn attachment_lifecycle_is_idempotent_fenced_and_rebuildable() { let identity = identity(); @@ -6248,6 +6649,7 @@ pub(crate) mod tests { element_schema_fingerprint: SchemaFingerprintV1([7; 32]), source_kind: StreamSourceKindV1::InvocationOutput, session_mapping: None, + entity_parent_start_index: None, }) .await, Err(DurableStreamProducerError::ProducerDeleting) @@ -6799,6 +7201,7 @@ pub(crate) mod tests { element_schema_fingerprint: SchemaFingerprintV1([7; 32]), source_kind: StreamSourceKindV1::InvocationOutput, session_mapping: None, + entity_parent_start_index: None, }) .await .unwrap() @@ -7383,7 +7786,7 @@ pub(crate) mod tests { }; producer - .register_result_streams(Vec::new(), { + .register_result_streams(Vec::new(), None, { let record = result(vec![1]); move |_| record }) @@ -7392,7 +7795,7 @@ pub(crate) mod tests { let committed = oplog.committed_length(); producer - .register_result_streams(Vec::new(), { + .register_result_streams(Vec::new(), None, { let record = result(vec![1]); move |_| record }) @@ -7402,7 +7805,7 @@ pub(crate) mod tests { assert_eq!( producer - .register_result_streams(Vec::new(), { + .register_result_streams(Vec::new(), None, { let record = result(vec![2]); move |_| record }) @@ -7604,16 +8007,19 @@ pub(crate) mod tests { let producer_fingerprint = identity.fingerprint; oplog .add_durable_stream_batch(Box::new(move |item_index| { - vec![DurableStreamOplogRecord::Items(StreamItemsRecordV1 { - format_version: 1, - stream_id, - producer_fingerprint, - first_sequence: 1, - nested_stream_ids: Vec::new(), - newly_registered_stream_ids: Vec::new(), - payload: StreamItemsPayloadV1::Values(vec![vec![1]]), - offsets: vec![StreamOffsetV1::new(item_index, 0)], - })] + vec![DurableStreamOplogRecord::Items( + None, + StreamItemsRecordV1 { + format_version: 1, + stream_id, + producer_fingerprint, + first_sequence: 1, + nested_stream_ids: Vec::new(), + newly_registered_stream_ids: Vec::new(), + payload: StreamItemsPayloadV1::Values(vec![vec![1]]), + offsets: vec![StreamOffsetV1::new(item_index, 0)], + }, + )] })) .await .unwrap(); @@ -7652,6 +8058,7 @@ pub(crate) mod tests { index .apply_registration( root_index, + None, root, identity.environment_id, &identity.agent_id, @@ -7680,7 +8087,8 @@ pub(crate) mod tests { let error = index .apply_item_batch( item_index, - vec![(nested_index, nested)], + None, + vec![(nested_index, None, nested)], StreamItemsRecordV1 { format_version: DURABLE_STREAM_FORMAT_VERSION, stream_id: parent_stream_id, @@ -7743,17 +8151,20 @@ pub(crate) mod tests { let nested_stream_id = nested_record.handle.stream_id; let item_index = OplogIndex::from_u64(registration_index.as_u64() + 1); vec![ - DurableStreamOplogRecord::Registered(nested_record), - DurableStreamOplogRecord::Items(StreamItemsRecordV1 { - format_version: DURABLE_STREAM_FORMAT_VERSION, - stream_id: parent_stream_id, - producer_fingerprint, - first_sequence: 0, - nested_stream_ids: vec![nested_stream_id, nested_stream_id], - newly_registered_stream_ids: vec![nested_stream_id], - payload: StreamItemsPayloadV1::Values(vec![vec![1]]), - offsets: vec![StreamOffsetV1::new(item_index, 0)], - }), + DurableStreamOplogRecord::Registered(None, nested_record), + DurableStreamOplogRecord::Items( + None, + StreamItemsRecordV1 { + format_version: DURABLE_STREAM_FORMAT_VERSION, + stream_id: parent_stream_id, + producer_fingerprint, + first_sequence: 0, + nested_stream_ids: vec![nested_stream_id, nested_stream_id], + newly_registered_stream_ids: vec![nested_stream_id], + payload: StreamItemsPayloadV1::Values(vec![vec![1]]), + offsets: vec![StreamOffsetV1::new(item_index, 0)], + }, + ), ] })) .await @@ -7798,13 +8209,16 @@ pub(crate) mod tests { ); oplog .add_durable_stream_batch(Box::new(move |registration_index| { - vec![DurableStreamOplogRecord::Registered(registration_record( - registration_index, - environment_id, - agent_id, - producer_fingerprint, - nested, - ))] + vec![DurableStreamOplogRecord::Registered( + None, + registration_record( + registration_index, + environment_id, + agent_id, + producer_fingerprint, + nested, + ), + )] })) .await .unwrap(); @@ -7938,6 +8352,7 @@ pub(crate) mod tests { element_schema_fingerprint: SchemaFingerprintV1([7; 32]), source_kind: StreamSourceKindV1::InvocationOutput, session_mapping: None, + entity_parent_start_index: None, }; producer.register(request).await.expect( @@ -7976,7 +8391,7 @@ pub(crate) mod tests { let mut index = ProducerStreamIndex::default(); for position in 0..MAX_DURABLE_STREAMS_PER_SESSION { index - .apply_session_references(&record(mapping(position))) + .apply_session_references(None, &record(mapping(position))) .unwrap(); } assert_eq!( @@ -7984,13 +8399,15 @@ pub(crate) mod tests { MAX_DURABLE_STREAMS_PER_SESSION ); - index.apply_session_references(&record(mapping(0))).unwrap(); + index + .apply_session_references(None, &record(mapping(0))) + .unwrap(); assert_eq!( index.session_stream_counts[&identity.invocation], MAX_DURABLE_STREAMS_PER_SESSION ); assert_eq!( - index.apply_session_references(&record(mapping(MAX_DURABLE_STREAMS_PER_SESSION))), + index.apply_session_references(None, &record(mapping(MAX_DURABLE_STREAMS_PER_SESSION))), Err(DurableStreamProducerError::StreamLimit) ); } @@ -8615,6 +9032,7 @@ pub(crate) mod tests { element_schema_fingerprint: SchemaFingerprintV1([7; 32]), source_kind: StreamSourceKindV1::InvocationOutput, session_mapping: None, + entity_parent_start_index: None, }) .await .unwrap(); @@ -8681,6 +9099,7 @@ pub(crate) mod tests { producer .finish_session( session_key, + None, Err(b"failed".to_vec()), golem_common::base_model::durable_stream::StreamCancelReasonV1::InvocationFailed, ) diff --git a/golem-worker-executor/src/durable_host/golem/retry_api.rs b/golem-worker-executor/src/durable_host/golem/retry_api.rs index b3648f8618..b6495686b4 100644 --- a/golem-worker-executor/src/durable_host/golem/retry_api.rs +++ b/golem-worker-executor/src/durable_host/golem/retry_api.rs @@ -134,7 +134,10 @@ impl Host for DurableWorkerCtx { } else if self.state.is_live() { self.public_state .worker() - .add_and_commit_oplog(OplogEntry::set_retry_policy(named_policy.clone())) + .add_and_commit_oplog(OplogEntry::set_retry_policy( + self.entity_parent_start_index(), + named_policy.clone(), + )) .await; } else { let (_, _) = get_oplog_entry!(self.state.replay_state, OplogEntry::SetRetryPolicy)?; @@ -152,7 +155,10 @@ impl Host for DurableWorkerCtx { } else if self.state.is_live() { self.public_state .worker() - .add_and_commit_oplog(OplogEntry::remove_retry_policy(name.clone())) + .add_and_commit_oplog(OplogEntry::remove_retry_policy( + self.entity_parent_start_index(), + name.clone(), + )) .await; } else { let (_, _) = get_oplog_entry!(self.state.replay_state, OplogEntry::RemoveRetryPolicy)?; diff --git a/golem-worker-executor/src/durable_host/golem/v1x.rs b/golem-worker-executor/src/durable_host/golem/v1x.rs index c811ebfce8..be3f38bcfc 100644 --- a/golem-worker-executor/src/durable_host/golem/v1x.rs +++ b/golem-worker-executor/src/durable_host/golem/v1x.rs @@ -619,7 +619,12 @@ impl Host for DurableWorkerCtx { // tip would nondeterministically point past the `NoOp` entry. Debugging sessions // discard writes and return `NONE` from `add`; fall back to the session's replay // target there so the guest never observes an invalid index. - let marker = match self.state.oplog.add(OplogEntry::no_op()).await { + let marker = match self + .state + .oplog + .add(OplogEntry::no_op(self.entity_parent_start_index())) + .await + { OplogIndex::NONE => self.state.current_oplog_index().await, index => index, }; @@ -700,7 +705,7 @@ impl Host for DurableWorkerCtx { // Write an oplog entry with the new jump and then restart the worker self.public_state .worker() - .add_and_commit_oplog(OplogEntry::jump(jump)) + .add_and_commit_oplog(OplogEntry::jump(self.entity_parent_start_index(), jump)) .await; debug!("Interrupting live execution for jumping from {jump_source} to {jump_target}",); @@ -755,7 +760,9 @@ impl Host for DurableWorkerCtx { let begin_index = match self .state .oplog - .add(OplogEntry::begin_atomic_region()) + .add(OplogEntry::begin_atomic_region( + self.entity_parent_start_index(), + )) .await { OplogIndex::NONE => self.state.current_oplog_index().await, @@ -812,7 +819,10 @@ impl Host for DurableWorkerCtx { self.public_state .worker() - .add_and_commit_oplog(OplogEntry::jump(deleted_region)) + .add_and_commit_oplog(OplogEntry::jump( + self.entity_parent_start_index(), + deleted_region, + )) .await; // TODO: this recomputation should not be necessary. @@ -873,7 +883,10 @@ impl Host for DurableWorkerCtx { // append leaves the region uncommitted and replay retries it as a whole. self.state .oplog - .add(OplogEntry::end_atomic_region(begin_index)) + .add(OplogEntry::end_atomic_region( + self.entity_parent_start_index(), + begin_index, + )) .await; } else { let (_, _) = get_oplog_entry!(self.state.replay_state, OplogEntry::EndAtomicRegion)?; @@ -1710,8 +1723,13 @@ impl HostGetOplog for DurableWorkerCtx { let result = get_oplog_chunk(self, &entry).await; let response = match result { Ok(chunk) if chunk.next_oplog_index != entry.next_oplog_index => { + let entries = chunk + .entries + .into_iter() + .map(|entry| entry.entry) + .collect::>(); HostResponseGolemApiOplogChunk { - result: serde_json::to_vec(&chunk.entries) + result: serde_json::to_vec(&entries) .map(Some) .map_err(|error| error.to_string()), next_oplog_index: chunk.next_oplog_index, @@ -2066,8 +2084,13 @@ impl HostSearchOplog for DurableWorkerCtx { let result = get_search_oplog_chunk(self, &entry).await; let response = match result { Ok(chunk) if chunk.next_oplog_index != entry.next_oplog_index => { + let entries = chunk + .entries + .into_iter() + .map(|entry| (entry.oplog_index, entry.entry)) + .collect::>(); HostResponseGolemApiOplogChunk { - result: serde_json::to_vec(&chunk.entries) + result: serde_json::to_vec(&entries) .map(Some) .map_err(|error| error.to_string()), next_oplog_index: chunk.next_oplog_index, diff --git a/golem-worker-executor/src/durable_host/http/inline_retry.rs b/golem-worker-executor/src/durable_host/http/inline_retry.rs index c6eab58776..573a989764 100644 --- a/golem-worker-executor/src/durable_host/http/inline_retry.rs +++ b/golem-worker-executor/src/durable_host/http/inline_retry.rs @@ -696,6 +696,7 @@ pub(crate) fn spawn_http_status_retry_after_body_finish, max_delay: Duration, begin_index: OplogIndex, + entity_parent_start_index: Option, ) -> FutureIncomingResponseHandle { // No span: this task waits for the guest to finish its outgoing body, so its // duration is decided by guest code rather than by an operation the executor @@ -741,6 +742,7 @@ pub(crate) fn spawn_http_status_retry_after_body_finish( retry_properties: RetryProperties, max_delay: Duration, begin_index: OplogIndex, + entity_parent_start_index: Option, execution_status: Arc>, ) -> FutureIncomingResponseHandle { // Capture config fields individually since OutgoingRequestConfig is not Clone @@ -990,6 +993,7 @@ pub fn spawn_http_request_with_retry( .cloned(); let mut task_ctx = crate::durable_host::durability::TaskRetryContext { retry_point: begin_index, + entity_parent_start_index, environment_state_service, environment_id, default_retry_policy, @@ -1272,6 +1276,7 @@ pub async fn try_output_stream_inline_retry( retry_properties, exec_state.max_in_function_retry_delay, request_state.begin_index(), + ctx.entity_parent_start_index(), ctx.execution_status.clone(), ); HostFutureIncomingResponse::pending(retry_handle) diff --git a/golem-worker-executor/src/durable_host/http/outgoing_http.rs b/golem-worker-executor/src/durable_host/http/outgoing_http.rs index 5a96e93945..5322ad7336 100644 --- a/golem-worker-executor/src/durable_host/http/outgoing_http.rs +++ b/golem-worker-executor/src/durable_host/http/outgoing_http.rs @@ -100,6 +100,7 @@ pub(crate) async fn maybe_enable_http_background_retry( retry_properties, durable_state.max_in_function_retry_delay, state.begin_index(), + ctx.entity_parent_start_index(), ctx.execution_status.clone(), ); HostFutureIncomingResponse::pending(retry_handle) @@ -185,6 +186,7 @@ pub(crate) async fn maybe_enable_http_pending_status_retry( agent_type, ctx.state.config.max_in_function_retry_delay, state.begin_index(), + ctx.entity_parent_start_index(), )) } else { old diff --git a/golem-worker-executor/src/durable_host/http/types.rs b/golem-worker-executor/src/durable_host/http/types.rs index 3ab1e703c7..7ce9ee8a9e 100644 --- a/golem-worker-executor/src/durable_host/http/types.rs +++ b/golem-worker-executor/src/durable_host/http/types.rs @@ -1325,6 +1325,7 @@ impl DurableWorkerCtx { retry_properties, exec_state.max_in_function_retry_delay, request_state.begin_index(), + self.entity_parent_start_index(), self.execution_status.clone(), ); wasmtime_wasi_http::p2::types::HostFutureIncomingResponse::pending(retry_handle) diff --git a/golem-worker-executor/src/durable_host/mod.rs b/golem-worker-executor/src/durable_host/mod.rs index 0fda3e8ab3..0f2a77cf9b 100644 --- a/golem-worker-executor/src/durable_host/mod.rs +++ b/golem-worker-executor/src/durable_host/mod.rs @@ -1847,8 +1847,12 @@ impl DurableWorkerCtx { }; match &pending_event.event { QueuedCardEvent::Revoke(_) => { + let entity_parent_start_index = pending_event.entity_parent_start_index; let card_ids = pending_events .into_iter() + .filter(|pending_event| { + pending_event.entity_parent_start_index == entity_parent_start_index + }) .filter_map(|pending_event| match pending_event.event { QueuedCardEvent::Revoke(event) => Some(event.card_id), QueuedCardEvent::Install(_) @@ -1856,7 +1860,8 @@ impl DurableWorkerCtx { | QueuedCardEvent::TransferReceived(_) => None, }) .collect::>(); - self.apply_card_revoked_cascade(&card_ids, true).await?; + self.apply_card_revoked_cascade(entity_parent_start_index, &card_ids, true) + .await?; } QueuedCardEvent::Install(event) => { let Some(card) = event.card.clone() else { @@ -1865,7 +1870,11 @@ impl DurableWorkerCtx { )); }; let _ = self - .apply_card_install(Some(pending_event.oplog_index), card) + .apply_card_install( + pending_event.entity_parent_start_index, + Some(pending_event.oplog_index), + card, + ) .await?; } QueuedCardEvent::TransferReceived(event) => { @@ -1876,6 +1885,7 @@ impl DurableWorkerCtx { }; let _ = self .apply_received_card_transfer( + pending_event.entity_parent_start_index, pending_event.oplog_index, event.transfer_id, event.source_card_id, @@ -1898,8 +1908,12 @@ impl DurableWorkerCtx { let expired_root_ids = expired_wallet_card_ids_at(&self.state.invocation_scope_root_cards, Utc::now()); if !expired_root_ids.is_empty() { - self.apply_card_revoked_cascade(&expired_root_ids, true) - .await?; + self.apply_card_revoked_cascade( + self.entity_parent_start_index(), + &expired_root_ids, + true, + ) + .await?; } Ok(()) } @@ -1997,6 +2011,7 @@ impl DurableWorkerCtx { pub(crate) async fn apply_card_install( &mut self, + entity_parent_start_index: Option, queued_event_index: Option, card: StoredCard, ) -> Result, WorkerExecutorError> { @@ -2006,6 +2021,7 @@ impl DurableWorkerCtx { self.public_state .worker() .add_and_commit_oplog(OplogEntry::card_install_failed( + entity_parent_start_index, queued_event_index, card_id, reason, @@ -2017,6 +2033,7 @@ impl DurableWorkerCtx { self.public_state .worker() .add_and_commit_oplog(OplogEntry::card_installed( + entity_parent_start_index, queued_event_index, card, Some(self.state.wallet_generation), @@ -2028,6 +2045,7 @@ impl DurableWorkerCtx { async fn apply_received_card_transfer( &mut self, + entity_parent_start_index: Option, queued_event_index: OplogIndex, transfer_id: uuid::Uuid, source_card_id: Option, @@ -2038,6 +2056,7 @@ impl DurableWorkerCtx { self.public_state .worker() .add_and_commit_oplog(OplogEntry::card_install_failed( + entity_parent_start_index, queued_event_index, card_id, reason, @@ -2049,6 +2068,7 @@ impl DurableWorkerCtx { self.public_state .worker() .add_and_commit_oplog(OplogEntry::card_transferred( + entity_parent_start_index, transfer_id, source_card_id, card_id, @@ -2085,6 +2105,7 @@ impl DurableWorkerCtx { self.public_state .worker() .add_and_commit_oplog(OplogEntry::card_revoked( + self.entity_parent_start_index(), queued_event_index, card_id, Some(self.state.wallet_generation), @@ -2097,6 +2118,7 @@ impl DurableWorkerCtx { pub(crate) async fn apply_card_revoked_cascade( &mut self, + entity_parent_start_index: Option, card_ids: &[CardId], commit_immediately: bool, ) -> Result<(), WorkerExecutorError> { @@ -2128,6 +2150,7 @@ impl DurableWorkerCtx { }; let entry = OplogEntry::CardRevokedCascade { timestamp: Timestamp::now_utc(), + entity_parent_start_index, revoked_card_ids: card_ids, affected_wallets, local_wallet_generation: Some(self.state.wallet_generation), @@ -2169,7 +2192,11 @@ impl DurableWorkerCtx { for (card_id, wallet_generation) in expired_card_generations { self.public_state .worker() - .add_and_commit_oplog(OplogEntry::card_expired(card_id, Some(wallet_generation))) + .add_and_commit_oplog(OplogEntry::card_expired( + self.entity_parent_start_index(), + card_id, + Some(wallet_generation), + )) .await; } Ok(()) @@ -2839,7 +2866,10 @@ impl DurableWorkerCtx { self.public_state .worker() - .add_and_commit_oplog(OplogEntry::jump(deleted_region)) + .add_and_commit_oplog(OplogEntry::jump( + self.entity_parent_start_index(), + deleted_region, + )) .await; // TODO: this recomputation should not be necessary. @@ -3239,7 +3269,10 @@ impl DurableWorkerCtx { self.public_state .worker() - .add_and_commit_oplog(OplogEntry::jump(deleted_region)) + .add_and_commit_oplog(OplogEntry::jump( + self.entity_parent_start_index(), + deleted_region, + )) .await; // TODO: this recomputation should not be necessary. @@ -4811,6 +4844,7 @@ impl InvocationHooks for DurableWorkerCtx { ) .await; + let entity_parent_start_index = self.entity_parent_start_index(); let permission_denial_persisted = if let ( Some(idempotency_key), TrapType::Error { @@ -4832,6 +4866,7 @@ impl InvocationHooks for DurableWorkerCtx { let inside_atomic_region = *atomic_region_had_side_effects; move |_| { OplogEntry::error( + entity_parent_start_index, AgentError::PermissionDenied(error), retry_from, inside_atomic_region, @@ -4869,6 +4904,7 @@ impl InvocationHooks for DurableWorkerCtx { atomic_region_had_side_effects, .. } => Some(OplogEntry::error( + entity_parent_start_index, error.clone(), *retry_from, *atomic_region_had_side_effects, @@ -5126,7 +5162,11 @@ impl ResourceStore for DurableWorkerCtx { let id = self.state.add(resource, name.clone()).await; let resource_id = AgentResourceId(id); if self.state.is_live() { - let entry = OplogEntry::create_resource(resource_id, name.clone()); + let entry = OplogEntry::create_resource( + self.entity_parent_start_index(), + resource_id, + name.clone(), + ); self.public_state.worker().add_to_oplog(entry).await; } id @@ -5137,7 +5177,11 @@ impl ResourceStore for DurableWorkerCtx { if let Some((resource_type_id, _)) = &result { let id = AgentResourceId(resource_id); if self.state.is_live() { - let entry = OplogEntry::drop_resource(id, resource_type_id.clone()); + let entry = OplogEntry::drop_resource( + self.entity_parent_start_index(), + id, + resource_type_id.clone(), + ); self.public_state.worker().add_to_oplog(entry).await; } } @@ -8023,6 +8067,7 @@ mod tests { let pending_transfer = PendingCardEventRef { timestamp: Timestamp::now_utc(), oplog_index: OplogIndex::from_u64(1), + entity_parent_start_index: None, event: QueuedCardEvent::transfer_started(Uuid::new_v4(), transfer_card, target_holder), }; assert!(next_drainable_card_events(vec![pending_transfer.clone()]).is_empty()); @@ -8032,6 +8077,7 @@ mod tests { PendingCardEventRef { timestamp: Timestamp::now_utc(), oplog_index: OplogIndex::from_u64(2), + entity_parent_start_index: None, event: QueuedCardEvent::revoke(revoked_card_id), }, ]; @@ -8052,6 +8098,7 @@ mod tests { let receipt = PendingCardEventRef { timestamp: Timestamp::now_utc(), oplog_index: OplogIndex::from_u64(1), + entity_parent_start_index: None, event: QueuedCardEvent::transfer_received(Uuid::new_v4(), source_card_id, card.clone()), }; @@ -8077,21 +8124,25 @@ mod tests { PendingCardEventRef { timestamp: Timestamp::now_utc(), oplog_index: OplogIndex::from_u64(1), + entity_parent_start_index: None, event: QueuedCardEvent::revoke(first), }, PendingCardEventRef { timestamp: Timestamp::now_utc(), oplog_index: OplogIndex::from_u64(2), + entity_parent_start_index: None, event: QueuedCardEvent::revoke(second), }, PendingCardEventRef { timestamp: Timestamp::now_utc(), oplog_index: OplogIndex::from_u64(3), + entity_parent_start_index: None, event: QueuedCardEvent::install(install), }, PendingCardEventRef { timestamp: Timestamp::now_utc(), oplog_index: OplogIndex::from_u64(4), + entity_parent_start_index: None, event: QueuedCardEvent::revoke(after_install), }, ]; @@ -8121,8 +8172,8 @@ mod tests { assert_eq!(count, 2); let entries = BTreeMap::from([ - (start, OplogEntry::no_op()), - (current_idx, OplogEntry::no_op()), + (start, OplogEntry::no_op(None)), + (current_idx, OplogEntry::no_op(None)), ]); scanned_entries += entries.len(); scan.fold_through(current_idx, &entries); @@ -8144,7 +8195,7 @@ mod tests { queued_idx, &BTreeMap::from([( queued_idx, - OplogEntry::card_event_queued(QueuedCardEvent::revoke(card_id)), + OplogEntry::card_event_queued(None, QueuedCardEvent::revoke(card_id)), )]), ); @@ -8155,7 +8206,7 @@ mod tests { terminal_idx, &BTreeMap::from([( terminal_idx, - OplogEntry::card_revoked(queued_idx, card_id, None), + OplogEntry::card_revoked(None, queued_idx, card_id, None), )]), ); @@ -8170,11 +8221,13 @@ mod tests { let cached_pending = PendingCardEventRef { timestamp: Timestamp::now_utc(), oplog_index: OplogIndex::from_u64(9), + entity_parent_start_index: None, event: QueuedCardEvent::revoke(cached_card_id), }; let status_pending = PendingCardEventRef { timestamp: Timestamp::now_utc(), oplog_index: OplogIndex::from_u64(10), + entity_parent_start_index: None, event: QueuedCardEvent::revoke(status_card_id), }; let mut scan = @@ -8204,11 +8257,13 @@ mod tests { let cached_pending = PendingCardEventRef { timestamp: Timestamp::now_utc(), oplog_index: OplogIndex::from_u64(20), + entity_parent_start_index: None, event: QueuedCardEvent::revoke(CardId::new()), }; let status_pending = PendingCardEventRef { timestamp: Timestamp::now_utc(), oplog_index: OplogIndex::from_u64(5), + entity_parent_start_index: None, event: QueuedCardEvent::revoke(CardId::new()), }; let mut scan = CardEventBoundaryScan::new(OplogIndex::from_u64(20), vec![cached_pending]); diff --git a/golem-worker-executor/src/durable_host/p3/http/send.rs b/golem-worker-executor/src/durable_host/p3/http/send.rs index adbd9c458f..64872fb3b2 100644 --- a/golem-worker-executor/src/durable_host/p3/http/send.rs +++ b/golem-worker-executor/src/durable_host/p3/http/send.rs @@ -1155,6 +1155,7 @@ where agent_config_retry_policies, runtime_retry_policy_mutations, max_in_function_retry_delay, + entity_parent_start_index, worker, ) = store.with(|mut access| { let ctx = durable_worker_ctx::(access.data_mut()); @@ -1166,6 +1167,7 @@ where ctx.state.agent_config_retry_policies(), ctx.state.runtime_retry_policy_mutations.clone(), ctx.state.config.max_in_function_retry_delay, + ctx.entity_parent_start_index(), ctx.public_state.worker(), ) }); @@ -1178,6 +1180,7 @@ where TaskRetryContext { retry_point, + entity_parent_start_index, environment_state_service, environment_id, default_retry_policy, diff --git a/golem-worker-executor/src/durable_host/permissions/mod.rs b/golem-worker-executor/src/durable_host/permissions/mod.rs index 8cd0aa252b..3e989a6528 100644 --- a/golem-worker-executor/src/durable_host/permissions/mod.rs +++ b/golem-worker-executor/src/durable_host/permissions/mod.rs @@ -979,6 +979,7 @@ where .worker() .add_and_commit_oplog(OplogEntry::CardDerived { timestamp: Timestamp::now_utc(), + entity_parent_start_index: ctx.entity_parent_start_index(), card: created.clone(), wallet_generation, }) @@ -1646,6 +1647,7 @@ async fn load_card_transfer_request( async fn complete_source_card_transfer( ctx: &mut DurableWorkerCtx, + entity_parent_start_index: Option, transfer_id: Uuid, source_card_id: CardId, installed_card: &StoredCard, @@ -1659,6 +1661,7 @@ async fn complete_source_card_transfer( ensure_source_card_transfer_started( ctx, + entity_parent_start_index, transfer_id, source_card_id, &target_holder, @@ -1685,6 +1688,7 @@ async fn complete_source_card_transfer( ctx.public_state .worker() .add_and_commit_oplog(OplogEntry::card_transfer_confirmed( + entity_parent_start_index, transfer_id, source_card_id, installed_card.card_id(), @@ -1697,6 +1701,7 @@ async fn complete_source_card_transfer( async fn ensure_source_card_transfer_started( ctx: &mut DurableWorkerCtx, + entity_parent_start_index: Option, transfer_id: Uuid, source_card_id: CardId, target_holder: &CardHolder, @@ -1721,6 +1726,7 @@ async fn ensure_source_card_transfer_started( ctx.public_state .worker() .add_and_commit_oplog(OplogEntry::card_transfer_started( + entity_parent_start_index, transfer_id, source_card_id, Some(CardHolder::Agent(AgentCardHolder { @@ -1777,6 +1783,7 @@ async fn execute_source_card_transfer( .worker() .add_and_commit_oplog(OplogEntry::CardDerived { timestamp: Timestamp::now_utc(), + entity_parent_start_index: ctx.entity_parent_start_index(), card: installed_card.clone(), wallet_generation: Some(ctx.state.wallet_generation), }) @@ -1786,6 +1793,7 @@ async fn execute_source_card_transfer( ctx.public_state .worker() .add_and_commit_oplog(OplogEntry::card_event_queued( + ctx.entity_parent_start_index(), QueuedCardEvent::transfer_started_with_source( transfer.transfer_id, transfer.source_card.card_id(), @@ -1798,6 +1806,7 @@ async fn execute_source_card_transfer( complete_source_card_transfer( ctx, + ctx.entity_parent_start_index(), transfer.transfer_id, transfer.source_card.card_id(), &installed_card, @@ -1950,6 +1959,10 @@ pub(super) struct PendingSourceCardTransferRetry { } impl PendingSourceCardTransferRetry { + pub(super) fn entity_parent_start_index(&self) -> Option { + self.pending.entity_parent_start_index + } + pub(super) async fn is_confirmed( &self, oplog: &dyn Oplog, @@ -2057,6 +2070,7 @@ pub(super) async fn prepare_pending_source_card_transfers( }); ensure_source_card_transfer_started( ctx, + pending.entity_parent_start_index, retry.transfer_id, retry.source_card_id, &target_holder, @@ -2103,6 +2117,7 @@ pub(super) async fn complete_pending_source_card_transfers( ctx.public_state .worker() .add_and_commit_oplog(OplogEntry::card_transfer_confirmed( + retry.entity_parent_start_index(), retry.transfer_id, retry.source_card_id, retry.installed_card.card_id(), @@ -2230,8 +2245,9 @@ async fn complete_permission_card_revoke( | Err(PermissionCardRevokeError::CardRevoked(_)) => vec![card_id], Err(PermissionCardRevokeError::NotPermitted(_)) => Vec::new(), }; + let entity_parent_start_index = ctx.entity_parent_start_index(); if let Err(error) = ctx - .apply_card_revoked_cascade(&locally_revoked_card_ids, false) + .apply_card_revoked_cascade(entity_parent_start_index, &locally_revoked_card_ids, false) .await { handle.abandon_for_trap(); diff --git a/golem-worker-executor/src/durable_host/replay_state/tests.rs b/golem-worker-executor/src/durable_host/replay_state/tests.rs index 0cde1c1751..58abf8a960 100644 --- a/golem-worker-executor/src/durable_host/replay_state/tests.rs +++ b/golem-worker-executor/src/durable_host/replay_state/tests.rs @@ -210,6 +210,7 @@ async fn test_replay_state( fn noop() -> OplogEntry { OplogEntry::NoOp { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, } } @@ -1007,6 +1008,7 @@ async fn custom_claim_id_can_be_reused_after_replay_restart() { fn begin_atomic_region() -> OplogEntry { OplogEntry::BeginAtomicRegion { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, } } @@ -1523,6 +1525,7 @@ async fn permission_events_replay_after_invocation_wallet_pin() { invocation_started(wallet_pin.clone()), OplogEntry::CardDerived { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, card: derived_card.clone(), wallet_generation: Some(0), }, @@ -1576,6 +1579,7 @@ async fn recorded_success_replays_without_live_expiry_or_authority_inputs() { noop(), OplogEntry::CardInstalled { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, queued_event_index: None, card: card.clone(), wallet_generation: Some(7), @@ -1644,6 +1648,7 @@ async fn permission_events_are_recovered_from_skipped_regions() { noop(), OplogEntry::CardTransferred { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, transfer_id, source_card_id: Some(source_card_id), installed_card_id: card.card_id(), @@ -1682,6 +1687,7 @@ async fn snapshot_prefix_suppresses_replayed_permission_events() { noop(), OplogEntry::CardInstalled { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, queued_event_index: None, card, wallet_generation: Some(1), @@ -2123,6 +2129,7 @@ async fn error_hint_between_start_and_end_resolves() { noop(), start_now(), OplogEntry::error( + None, AgentError::TransientError("boom".to_string()), OplogIndex::from_u64(2), false, @@ -4415,6 +4422,7 @@ fn cancelled_with_partial_for(start_index: u64, nanos: u64) -> OplogEntry { fn end_atomic_region(begin_index: u64) -> OplogEntry { OplogEntry::EndAtomicRegion { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, begin_index: OplogIndex::from_u64(begin_index), } } diff --git a/golem-worker-executor/src/durable_host/tool/mod.rs b/golem-worker-executor/src/durable_host/tool/mod.rs index 7bdd610043..9abcf09f7d 100644 --- a/golem-worker-executor/src/durable_host/tool/mod.rs +++ b/golem-worker-executor/src/durable_host/tool/mod.rs @@ -1183,17 +1183,16 @@ fn project_tool_rpc_error( fn project_tool_response_value( response: ToolInvokeResponse, ctx: &mut DurableWorkerCtx, -) -> Result<(Option, Option>), RpcError> { +) -> Result, RpcError> { response .map_err(|error| project_tool_rpc_error(error, ctx)) .and_then(|response| { - let result = response + response .result .as_ref() .map(|value| encode_typed_tool_value(value, ctx)) .transpose() - .map_err(RpcError::ProtocolError)?; - Ok((result, response.stdout)) + .map_err(RpcError::ProtocolError) }) } @@ -1206,12 +1205,11 @@ where Ctx: WorkerCtx, { accessor.with(|mut access| { - let (result, stdout) = project_tool_response_value(response, access.get())?; - let stdout = stdout - .map(|bytes| StreamReader::new(&mut access, bytes)) - .transpose() - .map_err(|error| RpcError::RemoteInternalError(error.to_string()))?; - Ok(InvocationResult { result, stdout }) + let result = project_tool_response_value(response, access.get())?; + Ok(InvocationResult { + result, + stdout: None, + }) }) } @@ -1694,10 +1692,7 @@ fn decode_tool_terminal( "invalid durable tool result payload: {error}" )) })?; - Ok(Ok(SerializableToolInvocationResult { - result, - stdout: None, - })) + Ok(Ok(SerializableToolInvocationResult { result })) } Err(error) => Ok(Err(error)), } diff --git a/golem-worker-executor/src/durable_host/wasm_rpc/mod.rs b/golem-worker-executor/src/durable_host/wasm_rpc/mod.rs index 91c5f43e9c..e86ede9255 100644 --- a/golem-worker-executor/src/durable_host/wasm_rpc/mod.rs +++ b/golem-worker-executor/src/durable_host/wasm_rpc/mod.rs @@ -2300,6 +2300,7 @@ async fn caller_durable_rpc_streams( [], ) .with_consumer_invocation(consumer_invocation) + .with_entity_parent_start_index(ctx.entity_parent_start_index()) .with_rpc(ctx.rpc()) .with_consumer_journal(worker.durable_stream_consumer_journal()) .with_auth_ctx(auth_ctx) @@ -3779,6 +3780,7 @@ struct TaskRetryParams { max_in_function_retry_delay: Duration, worker: Arc>, retry_point: OplogIndex, + entity_parent_start_index: Option, execution_status: Arc>, } @@ -3939,6 +3941,7 @@ fn spawn_rpc_task_with_retry( .cloned(); let task_ctx = crate::durable_host::durability::TaskRetryContext { retry_point: retry_params.retry_point, + entity_parent_start_index: retry_params.entity_parent_start_index, environment_state_service: retry_params.environment_state_service, environment_id: retry_params.environment_id, default_retry_policy: retry_params.default_retry_policy, @@ -4005,6 +4008,7 @@ fn spawn_invoke_and_await_task( max_in_function_retry_delay: ctx.durable_execution_state().max_in_function_retry_delay, worker: ctx.public_state.worker(), retry_point, + entity_parent_start_index: ctx.entity_parent_start_index(), execution_status: ctx.execution_status.clone(), }) }; diff --git a/golem-worker-executor/src/grpc/invocation_session.rs b/golem-worker-executor/src/grpc/invocation_session.rs index 42b917c7e9..a199d46965 100644 --- a/golem-worker-executor/src/grpc/invocation_session.rs +++ b/golem-worker-executor/src/grpc/invocation_session.rs @@ -1955,6 +1955,7 @@ pub(crate) fn build_durable_streaming_request( registrations.push(( transport_stream_id, ProducerRegistrationRequestV1 { + entity_parent_start_index: None, coordinate: StreamRegistrationCoordinateV1::Root { invocation_id: session_key.clone(), root_kind: StreamRootKindV1::MethodInput, diff --git a/golem-worker-executor/src/grpc/mod.rs b/golem-worker-executor/src/grpc/mod.rs index 3c85d16fc1..b6b91f19e0 100644 --- a/golem-worker-executor/src/grpc/mod.rs +++ b/golem-worker-executor/src/grpc/mod.rs @@ -1405,14 +1405,7 @@ impl + UsesAllDeps + Send + Sync + entries: chunk .entries .into_iter() - .map(|(idx, entry)| { - entry.try_into().map(|entry: golem::worker::OplogEntry| { - golem::worker::OplogEntryWithIndex { - oplog_index: idx.into(), - entry: Some(entry), - } - }) - }) + .map(|entry| entry.try_into()) .collect::, _>>() .map_err(WorkerExecutorError::unknown)?, next, diff --git a/golem-worker-executor/src/model/public_oplog/mod.rs b/golem-worker-executor/src/model/public_oplog/mod.rs index 8b544b642a..ea97e3676f 100644 --- a/golem-worker-executor/src/model/public_oplog/mod.rs +++ b/golem-worker-executor/src/model/public_oplog/mod.rs @@ -23,9 +23,13 @@ use crate::services::oplog::OplogServiceOps; use async_trait::async_trait; use golem_common::model::agent::{AgentMode, AgentTypeName, ParsedAgentId}; use golem_common::model::component::{ComponentRevision, InstalledPlugin}; -use golem_common::model::entity::EntityInvocationId; +use golem_common::model::entity::{ + AgentEntity, EntityCallMode, EntityInvocationDescriptor, EntityInvocationId, + EntityInvocationRequest, +}; use golem_common::model::invocation_context::InvocationContextStack; use golem_common::model::lucene::Query; +use golem_common::model::oplog::host_functions::HostFunctionName; use golem_common::model::oplog::public_oplog_entry::{ ActivatePluginParams, AgentInvocationFinishedParams, AgentInvocationStartedParams, BeginAtomicRegionParams, BeginRemoteTransactionParams, CancelPendingInvocationParams, @@ -51,10 +55,13 @@ use golem_common::model::oplog::{ HostResponseEntityInvocation, JsonSnapshotData, LoadSnapshotParameters, ManualUpdateParameters, MultipartPartData, MultipartSnapshotData, MultipartSnapshotPart, OplogEntry, OplogIndex, OplogScopeProjection, PluginInstallationDescription, ProcessOplogEntriesParameters, - ProcessOplogEntriesResultParameters, PublicAgentInvocation, PublicAgentInvocationResult, - PublicAttribute, PublicOplogEntry, PublicSnapshotData, PublicTypedAgentConfigEntry, - PublicUpdateDescription, RawSnapshotData, SaveSnapshotResultParameters, - SnapshotBasedUpdateParameters, UpdateDescription, + ProcessOplogEntriesResultParameters, PublicAgentEntity, PublicAgentEntityKind, + PublicAgentInvocation, PublicAgentInvocationResult, PublicAttribute, PublicEntityCallMode, + PublicEntityInvocation, PublicEntityInvocationContext, PublicEntityInvocationOperation, + PublicOplogEntry, PublicOplogEntryAttribution, PublicOplogEntryWithIndex, PublicSnapshotData, + PublicToolInvocationOperation, PublicTypedAgentConfigEntry, PublicUpdateDescription, + RawSnapshotData, SaveSnapshotResultParameters, SnapshotBasedUpdateParameters, + UpdateDescription, }; use golem_common::model::{ AgentId, AgentInvocation, AgentInvocationPayload, AgentInvocationResult, Empty, OwnedAgentId, @@ -65,16 +72,451 @@ use golem_common::schema::{ SchemaValue, TypedSchemaValue, }; use golem_service_base::error::worker_executor::WorkerExecutorError; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; pub struct PublicOplogChunk { - pub entries: Vec, + pub entries: Vec, pub next_oplog_index: OplogIndex, pub current_component_revision: ComponentRevision, pub first_index_in_chunk: OplogIndex, pub last_index: OplogIndex, } +#[derive(Clone)] +struct OplogStartAttributionSource { + parent_start_index: Option, + observational_owner: Option, + function_name: HostFunctionName, + request: Option>, +} + +impl OplogStartAttributionSource { + fn from_entry(entry: &OplogEntry) -> Option { + match entry { + OplogEntry::Start { + parent_start_index, + observational_owner, + function_name, + request, + .. + } => Some(Self { + parent_start_index: *parent_start_index, + observational_owner: *observational_owner, + function_name: function_name.clone(), + request: request.clone(), + }), + _ => None, + } + } + + fn parent(&self) -> Option { + self.observational_owner.or(self.parent_start_index) + } +} + +enum AttributionPathNode { + Inherit(OplogIndex), + Entity(OplogIndex, PublicEntityInvocation), + Agent(OplogIndex), +} + +struct PublicOplogAttributionResolver<'a> { + oplog_service: Arc, + owned_agent_id: &'a OwnedAgentId, + agent_mode: AgentMode, + starts: HashMap>, + resolved: HashMap>, +} + +impl<'a> PublicOplogAttributionResolver<'a> { + fn new( + oplog_service: Arc, + owned_agent_id: &'a OwnedAgentId, + agent_mode: AgentMode, + ) -> Self { + Self { + oplog_service, + owned_agent_id, + agent_mode, + starts: HashMap::new(), + resolved: HashMap::new(), + } + } + + fn cache_entry(&mut self, index: OplogIndex, entry: &OplogEntry) { + self.starts + .insert(index, OplogStartAttributionSource::from_entry(entry)); + } + + async fn attribution_for_entry( + &mut self, + index: OplogIndex, + entry: &OplogEntry, + ) -> Result { + if let Some(start_index) = entry.entity_parent_start_index() { + if start_index >= index { + return Err(format!( + "oplog entry {index} has non-causal entity parent Start index {start_index}" + )); + } + let source = self.load_start(start_index).await?.ok_or_else(|| { + format!( + "oplog entry {index} entity parent index {start_index} does not reference a Start" + ) + })?; + if source.function_name != HostFunctionName::GolemEntityInvoke { + return Err(format!( + "oplog entry {index} entity parent Start {start_index} is not an entity invocation" + )); + } + return self + .entity_context_for_start(start_index) + .await? + .map(PublicOplogEntryAttribution::entity) + .ok_or_else(|| { + format!( + "oplog entry {index} entity parent Start {start_index} has no entity attribution" + ) + }); + } + + let owner_start_index = match entry { + OplogEntry::Start { .. } => Some(index), + OplogEntry::End { start_index, .. } + | OplogEntry::Cancelled { start_index, .. } + | OplogEntry::CompletionDiscarded { start_index, .. } + | OplogEntry::CompletionDelivered { start_index, .. } => Some(*start_index), + OplogEntry::HostStreamFrame { + parent_start_index, .. + } + | OplogEntry::Log { + parent_start_index: Some(parent_start_index), + .. + } + | OplogEntry::StartSpan { + parent_start_index: Some(parent_start_index), + .. + } + | OplogEntry::FinishSpan { + parent_start_index: Some(parent_start_index), + .. + } + | OplogEntry::SetSpanAttribute { + parent_start_index: Some(parent_start_index), + .. + } => Some(*parent_start_index), + OplogEntry::Error { + entity_parent_start_index, + .. + } + | OplogEntry::NoOp { + entity_parent_start_index, + .. + } + | OplogEntry::Jump { + entity_parent_start_index, + .. + } + | OplogEntry::BeginAtomicRegion { + entity_parent_start_index, + .. + } + | OplogEntry::EndAtomicRegion { + entity_parent_start_index, + .. + } + | OplogEntry::CreateResource { + entity_parent_start_index, + .. + } + | OplogEntry::DropResource { + entity_parent_start_index, + .. + } + | OplogEntry::SetRetryPolicy { + entity_parent_start_index, + .. + } + | OplogEntry::RemoveRetryPolicy { + entity_parent_start_index, + .. + } + | OplogEntry::CardEventQueued { + entity_parent_start_index, + .. + } + | OplogEntry::CardInstalled { + entity_parent_start_index, + .. + } + | OplogEntry::CardInstallFailed { + entity_parent_start_index, + .. + } + | OplogEntry::CardRevoked { + entity_parent_start_index, + .. + } + | OplogEntry::CardExpired { + entity_parent_start_index, + .. + } + | OplogEntry::CardDerived { + entity_parent_start_index, + .. + } + | OplogEntry::CardTransferStarted { + entity_parent_start_index, + .. + } + | OplogEntry::CardTransferred { + entity_parent_start_index, + .. + } + | OplogEntry::CardRevokedCascade { + entity_parent_start_index, + .. + } + | OplogEntry::CardTransferConfirmed { + entity_parent_start_index, + .. + } + | OplogEntry::StreamRegistered { + entity_parent_start_index, + .. + } + | OplogEntry::StreamItems { + entity_parent_start_index, + .. + } + | OplogEntry::StreamEnd { + entity_parent_start_index, + .. + } + | OplogEntry::StreamCancel { + entity_parent_start_index, + .. + } + | OplogEntry::StreamSession { + entity_parent_start_index, + .. + } => *entity_parent_start_index, + OplogEntry::BeginRemoteTransaction { + original_begin_index: Some(begin_index), + .. + } => Some(*begin_index), + OplogEntry::BeginRemoteTransaction { + original_begin_index: None, + .. + } => Some(index.previous()), + OplogEntry::PreCommitRemoteTransaction { begin_index, .. } + | OplogEntry::PreRollbackRemoteTransaction { begin_index, .. } + | OplogEntry::CommittedRemoteTransaction { begin_index, .. } + | OplogEntry::RolledBackRemoteTransaction { begin_index, .. } => Some(*begin_index), + OplogEntry::Create { .. } + | OplogEntry::AgentInvocationStarted { .. } + | OplogEntry::AgentInvocationFinished { .. } + | OplogEntry::Suspend { .. } + | OplogEntry::Interrupted { .. } + | OplogEntry::Exited { .. } + | OplogEntry::PendingAgentInvocation { .. } + | OplogEntry::PendingUpdate { .. } + | OplogEntry::SuccessfulUpdate { .. } + | OplogEntry::FailedUpdate { .. } + | OplogEntry::GrowMemory { .. } + | OplogEntry::Log { + parent_start_index: None, + .. + } + | OplogEntry::Restart { .. } + | OplogEntry::ActivatePlugin { .. } + | OplogEntry::DeactivatePlugin { .. } + | OplogEntry::Revert { .. } + | OplogEntry::CancelPendingInvocation { .. } + | OplogEntry::StartSpan { + parent_start_index: None, + .. + } + | OplogEntry::FinishSpan { + parent_start_index: None, + .. + } + | OplogEntry::SetSpanAttribute { + parent_start_index: None, + .. + } + | OplogEntry::Snapshot { .. } + | OplogEntry::OplogProcessorCheckpoint { .. } => None, + }; + + match owner_start_index { + Some(start_index) => self + .entity_context_for_start(start_index) + .await + .map(|context| { + context.map_or_else( + PublicOplogEntryAttribution::agent, + PublicOplogEntryAttribution::entity, + ) + }), + None => Ok(PublicOplogEntryAttribution::agent()), + } + } + + async fn entity_context_for_start( + &mut self, + start_index: OplogIndex, + ) -> Result, String> { + let mut path = Vec::new(); + let mut visited = HashSet::new(); + let mut current = start_index; + let mut context = loop { + if let Some(resolved) = self.resolved.get(¤t) { + break resolved.clone(); + } + if !visited.insert(current) { + return Err(format!("cyclic oplog Start attribution at index {current}")); + } + + let Some(source) = self.load_start(current).await? else { + break None; + }; + if source.function_name == HostFunctionName::GolemToolInvocationRejected { + path.push(AttributionPathNode::Agent(current)); + break None; + } + if source.function_name == HostFunctionName::GolemEntityInvoke { + let invocation = self.load_public_entity_invocation(current, &source).await?; + let parent = source.parent(); + path.push(AttributionPathNode::Entity(current, invocation)); + match parent { + Some(parent) => current = parent, + None => break None, + } + } else { + let parent = source.parent(); + path.push(AttributionPathNode::Inherit(current)); + match parent { + Some(parent) => current = parent, + None => break None, + } + } + }; + + for node in path.into_iter().rev() { + let index = match node { + AttributionPathNode::Inherit(index) => index, + AttributionPathNode::Agent(index) => { + context = None; + index + } + AttributionPathNode::Entity(index, invocation) => { + let ancestors = context + .as_ref() + .map(|parent| { + let mut ancestors = parent.ancestors.clone(); + ancestors.push(parent.invocation.clone()); + ancestors + }) + .unwrap_or_default(); + context = Some(PublicEntityInvocationContext { + invocation, + ancestors, + }); + index + } + }; + self.resolved.insert(index, context.clone()); + } + + Ok(context) + } + + async fn load_start( + &mut self, + index: OplogIndex, + ) -> Result, String> { + if let Some(start) = self.starts.get(&index) { + return Ok(start.clone()); + } + + let entry = self + .oplog_service + .read_exact(self.owned_agent_id, self.agent_mode, index, 1) + .await + .remove(&index); + let start = entry + .as_ref() + .and_then(OplogStartAttributionSource::from_entry); + self.starts.insert(index, start.clone()); + Ok(start) + } + + async fn load_public_entity_invocation( + &self, + start_index: OplogIndex, + source: &OplogStartAttributionSource, + ) -> Result { + let request_payload = source + .request + .clone() + .ok_or_else(|| format!("entity invocation Start {start_index} has no request"))?; + let request: HostRequest = self + .oplog_service + .download_payload(self.owned_agent_id, self.agent_mode, request_payload) + .await?; + let request = match request { + HostRequest::EntityInvocation(request) => request, + actual => { + return Err(format!( + "entity invocation Start {start_index} has unexpected request {actual:?}" + )); + } + }; + let metadata = desert_rust::deserialize::(&request.metadata) + .map_err(|error| { + format!("failed to decode entity invocation Start {start_index}: {error}") + })?; + + Ok(public_entity_invocation(start_index, metadata)) + } +} + +fn public_entity_invocation( + start_index: OplogIndex, + request: EntityInvocationRequest, +) -> PublicEntityInvocation { + let entity = PublicAgentEntity { + kind: match &request.entity { + AgentEntity::Tool(_) => PublicAgentEntityKind::Tool, + AgentEntity::ToolMiddleware(_) => PublicAgentEntityKind::ToolMiddleware, + }, + name: request.entity.name().to_string(), + }; + let call_mode = match request.call_mode { + EntityCallMode::Synchronous => PublicEntityCallMode::Synchronous, + EntityCallMode::Asynchronous => PublicEntityCallMode::Asynchronous, + EntityCallMode::FireAndForget => PublicEntityCallMode::FireAndForget, + }; + let operation = request.operation.map(|operation| match operation { + EntityInvocationDescriptor::Tool(tool) => { + PublicEntityInvocationOperation::Tool(PublicToolInvocationOperation { + command_path: tool.command_path, + has_stdin: tool.has_stdin, + has_stdout: tool.has_stdout, + declares_stdout: tool.declares_stdout, + }) + } + }); + PublicEntityInvocation { + entity, + start_index, + call_mode, + operation, + } +} + /// Projects one entity invocation's transitive durable-call tree from its owner's raw oplog. /// Entity histories remain owner records; this is a filtered view, not a child oplog or status. pub fn project_entity_oplog_entries( @@ -121,6 +563,12 @@ pub async fn get_public_oplog_chunk( let mut next_oplog_index = initial_oplog_index; let mut first_index_in_chunk = None; + let mut attribution_resolver = + PublicOplogAttributionResolver::new(oplog_service.clone(), owned_agent_id, agent_mode); + for (index, raw_entry) in &raw_entries { + attribution_resolver.cache_entry(*index, raw_entry); + } + for (index, raw_entry) in raw_entries { if first_index_in_chunk.is_none() { first_index_in_chunk = Some(index); @@ -129,6 +577,9 @@ pub async fn get_public_oplog_chunk( current_component_revision = revision; } + let attribution = attribution_resolver + .attribution_for_entry(index, &raw_entry) + .await?; let entry = PublicOplogEntry::from_oplog_entry( index, raw_entry, @@ -140,7 +591,11 @@ pub async fn get_public_oplog_chunk( current_component_revision, ) .await?; - entries.push(entry); + entries.push(PublicOplogEntryWithIndex { + oplog_index: index, + attribution, + entry, + }); next_oplog_index = index.next(); } @@ -154,7 +609,7 @@ pub async fn get_public_oplog_chunk( } pub struct PublicOplogSearchResult { - pub entries: Vec<(OplogIndex, PublicOplogEntry)>, + pub entries: Vec, pub next_oplog_index: OplogIndex, pub current_component_revision: ComponentRevision, pub last_index: OplogIndex, @@ -191,12 +646,9 @@ pub async fn search_public_oplog( ) .await?; - for (idx, entry) in chunk.entries.into_iter().enumerate() { - if entry.matches(&query) { - results.push(( - OplogIndex::from_u64(u64::from(current_index) + idx as u64), - entry, - )); + for entry in chunk.entries { + if entry.entry.matches(&query) { + results.push(entry); } } @@ -204,7 +656,7 @@ pub async fn search_public_oplog( current_index = chunk.next_oplog_index; current_component_revision = chunk.current_component_revision; - if current_index >= last_index || results.len() >= count { + if current_index > last_index || results.len() >= count { break; } } @@ -358,23 +810,25 @@ impl PublicOplogEntryOps for PublicOplogEntry { .download_payload(owned_agent_id, agent_mode, request_payload) .await?; - // Enriching data - let host_request = match host_request { + let request_value = match host_request { + HostRequest::EntityInvocation(request) => request.input, HostRequest::GolemRpcInvoke(inner) => HostRequest::GolemRpcInvoke( enrich_golem_rpc_invoke(components, inner).await, - ), + ) + .into_typed_schema_value() + .map_err(|error| error.to_string())?, HostRequest::GolemRpcScheduledInvocation(inner) => { HostRequest::GolemRpcScheduledInvocation( enrich_golem_rpc_scheduled_invocation(components, inner).await, ) + .into_typed_schema_value() + .map_err(|error| error.to_string())? } - other => other, - }; - Some( - host_request + other => other .into_typed_schema_value() - .map_err(|e| e.to_string())?, - ) + .map_err(|error| error.to_string())?, + }; + Some(request_value) } else { None }; @@ -532,6 +986,7 @@ impl PublicOplogEntryOps for PublicOplogEntry { retry_from, inside_atomic_region, retry_policy_state, + .. } => Ok(PublicOplogEntry::Error(ErrorParams { timestamp, error: error.to_string(""), @@ -539,10 +994,12 @@ impl PublicOplogEntryOps for PublicOplogEntry { inside_atomic_region, retry_policy_state: retry_policy_state.map(Into::into), })), - OplogEntry::NoOp { timestamp } => Ok(PublicOplogEntry::NoOp(NoOpParams { timestamp })), - OplogEntry::Jump { timestamp, jump } => { - Ok(PublicOplogEntry::Jump(JumpParams { timestamp, jump })) + OplogEntry::NoOp { timestamp, .. } => { + Ok(PublicOplogEntry::NoOp(NoOpParams { timestamp })) } + OplogEntry::Jump { + timestamp, jump, .. + } => Ok(PublicOplogEntry::Jump(JumpParams { timestamp, jump })), OplogEntry::Interrupted { timestamp } => { Ok(PublicOplogEntry::Interrupted(InterruptedParams { timestamp, @@ -551,12 +1008,13 @@ impl PublicOplogEntryOps for PublicOplogEntry { OplogEntry::Exited { timestamp } => { Ok(PublicOplogEntry::Exited(ExitedParams { timestamp })) } - OplogEntry::BeginAtomicRegion { timestamp } => Ok(PublicOplogEntry::BeginAtomicRegion( - BeginAtomicRegionParams { timestamp }, - )), + OplogEntry::BeginAtomicRegion { timestamp, .. } => Ok( + PublicOplogEntry::BeginAtomicRegion(BeginAtomicRegionParams { timestamp }), + ), OplogEntry::EndAtomicRegion { timestamp, begin_index, + .. } => Ok(PublicOplogEntry::EndAtomicRegion(EndAtomicRegionParams { timestamp, begin_index, @@ -672,6 +1130,7 @@ impl PublicOplogEntryOps for PublicOplogEntry { timestamp, id, resource_type_id, + .. } => Ok(PublicOplogEntry::CreateResource(CreateResourceParams { timestamp, id, @@ -682,6 +1141,7 @@ impl PublicOplogEntryOps for PublicOplogEntry { timestamp, id, resource_type_id, + .. } => Ok(PublicOplogEntry::DropResource(DropResourceParams { timestamp, id, @@ -919,20 +1379,23 @@ impl PublicOplogEntryOps for PublicOplogEntry { }, )) } - OplogEntry::SetRetryPolicy { timestamp, policy } => { - Ok(PublicOplogEntry::SetRetryPolicy(SetRetryPolicyParams { - timestamp, - policy: policy.into(), - })) - } - OplogEntry::RemoveRetryPolicy { timestamp, name } => Ok( - PublicOplogEntry::RemoveRetryPolicy(RemoveRetryPolicyParams { timestamp, name }), - ), + OplogEntry::SetRetryPolicy { + timestamp, policy, .. + } => Ok(PublicOplogEntry::SetRetryPolicy(SetRetryPolicyParams { + timestamp, + policy: policy.into(), + })), + OplogEntry::RemoveRetryPolicy { + timestamp, name, .. + } => Ok(PublicOplogEntry::RemoveRetryPolicy( + RemoveRetryPolicyParams { timestamp, name }, + )), OplogEntry::CardRevoked { timestamp, queued_event_index, card_id, wallet_generation, + .. } => Ok(PublicOplogEntry::CardRevoked(CardRevokedParams { timestamp, queued_event_index, @@ -943,6 +1406,7 @@ impl PublicOplogEntryOps for PublicOplogEntry { timestamp, card_id, wallet_generation, + .. } => Ok(PublicOplogEntry::CardExpired(CardExpiredParams { timestamp, card_id, @@ -967,7 +1431,9 @@ impl PublicOplogEntryOps for PublicOplogEntry { .map_err(|e| e.to_string())?, })) } - OplogEntry::StreamRegistered { timestamp, record } => { + OplogEntry::StreamRegistered { + timestamp, record, .. + } => { let record = oplog_service .download_payload(owned_agent_id, agent_mode, record) .await?; @@ -978,7 +1444,9 @@ impl PublicOplogEntryOps for PublicOplogEntry { .map_err(|e| e.to_string())?, })) } - OplogEntry::StreamItems { timestamp, record } => { + OplogEntry::StreamItems { + timestamp, record, .. + } => { let record = oplog_service .download_payload(owned_agent_id, agent_mode, record) .await?; @@ -989,7 +1457,9 @@ impl PublicOplogEntryOps for PublicOplogEntry { .map_err(|e| e.to_string())?, })) } - OplogEntry::StreamEnd { timestamp, record } => { + OplogEntry::StreamEnd { + timestamp, record, .. + } => { let record = oplog_service .download_payload(owned_agent_id, agent_mode, record) .await?; @@ -1000,7 +1470,9 @@ impl PublicOplogEntryOps for PublicOplogEntry { .map_err(|e| e.to_string())?, })) } - OplogEntry::StreamCancel { timestamp, record } => { + OplogEntry::StreamCancel { + timestamp, record, .. + } => { let record = oplog_service .download_payload(owned_agent_id, agent_mode, record) .await?; @@ -1011,7 +1483,9 @@ impl PublicOplogEntryOps for PublicOplogEntry { .map_err(|e| e.to_string())?, })) } - OplogEntry::StreamSession { timestamp, record } => { + OplogEntry::StreamSession { + timestamp, record, .. + } => { let record = oplog_service .download_payload(owned_agent_id, agent_mode, record) .await?; @@ -1022,17 +1496,18 @@ impl PublicOplogEntryOps for PublicOplogEntry { .map_err(|e| e.to_string())?, })) } - OplogEntry::CardEventQueued { timestamp, event } => { - Ok(PublicOplogEntry::CardEventQueued(CardEventQueuedParams { - timestamp, - event: event.into(), - })) - } + OplogEntry::CardEventQueued { + timestamp, event, .. + } => Ok(PublicOplogEntry::CardEventQueued(CardEventQueuedParams { + timestamp, + event: event.into(), + })), OplogEntry::CardInstalled { timestamp, queued_event_index, card, wallet_generation, + .. } => Ok(PublicOplogEntry::CardInstalled(CardInstalledParams { timestamp, queued_event_index, @@ -1044,6 +1519,7 @@ impl PublicOplogEntryOps for PublicOplogEntry { queued_event_index, card_id, reason, + .. } => Ok(PublicOplogEntry::CardInstallFailed( CardInstallFailedParams { timestamp, @@ -1056,6 +1532,7 @@ impl PublicOplogEntryOps for PublicOplogEntry { timestamp, card, wallet_generation, + .. } => Ok(PublicOplogEntry::CardDerived(CardDerivedParams { timestamp, card_id: card.card_id(), @@ -1112,6 +1589,7 @@ impl PublicOplogEntryOps for PublicOplogEntry { source_card_id, installed_card_id, target_holder, + .. } => Ok(PublicOplogEntry::CardTransferConfirmed( CardTransferConfirmedParams { timestamp, diff --git a/golem-worker-executor/src/model/public_oplog/tests.rs b/golem-worker-executor/src/model/public_oplog/tests.rs index 8f7f1d51bb..b2ebd08b99 100644 --- a/golem-worker-executor/src/model/public_oplog/tests.rs +++ b/golem-worker-executor/src/model/public_oplog/tests.rs @@ -16,31 +16,46 @@ use super::*; use crate::services::oplog::{CommitLevel, OplogOps, PrimaryOplogService}; use crate::storage::indexed::memory::InMemoryIndexedStorage; use golem_common::model::account::{AccountEmail, AccountId}; +use golem_common::model::agent::{AgentPrincipal, AgentTypeName, Principal}; +use golem_common::model::component::ComponentName; +use golem_common::model::deployment::DeploymentRevision; +use golem_common::model::entity::{ + EntityActivation, EntityActivationPolicy, ExecutableTarget, FilesystemCapability, + ToolInvocationDescriptor, ToolMiddlewareName, +}; use golem_common::model::environment::EnvironmentId; +use golem_common::model::invocation_context::SpanId; +use golem_common::model::json::NormalizedJsonValue; use golem_common::model::oplog::payload::host_functions::HostFunctionName; use golem_common::model::oplog::payload::types::{ - SerializableEntityBodyExecution, SerializableHttpErrorCode, SerializableHttpMethod, - SerializableIpAddress, SerializableP3HttpBodyChunk, SerializableP3HttpClientSend, - SerializableP3HttpClientSendResult, SerializableP3HttpConsumeBodyResult, - SerializableP3HttpRequestOptions, SerializableP3HttpScheme, SerializableP3IpSocketAddress, - SerializableP3SocketErrorCode, SerializableP3TcpChunk, SerializableP3UdpDatagram, - SerializableResponseHeaders, SerializableToolOperationTerminal, - SerializableToolStructuredResult, + SecretRevealAudit, SerializableDateTime, SerializableEntityBodyExecution, + SerializableHttpErrorCode, SerializableHttpMethod, SerializableIpAddress, + SerializableP3HttpBodyChunk, SerializableP3HttpClientSend, SerializableP3HttpClientSendResult, + SerializableP3HttpConsumeBodyResult, SerializableP3HttpRequestOptions, + SerializableP3HttpScheme, SerializableP3IpSocketAddress, SerializableP3SocketErrorCode, + SerializableP3TcpChunk, SerializableP3UdpDatagram, SerializableResponseHeaders, + SerializableToolOperationTerminal, SerializableToolRpcError, SerializableToolStructuredResult, }; use golem_common::model::oplog::{ - DurableFunctionType, HostRequestEntityInvocation, HostRequestNoInput, - HostRequestP3HttpClientSend, HostRequestP3SocketsUdpSend, HostResponseEntityInvocation, + AttributeMap, DurableFunctionType, HostRequestEntityInvocation, + HostRequestGolemToolInvocationRejected, HostRequestNoInput, HostRequestP3HttpClientSend, + HostRequestP3SocketsUdpSend, HostRequestSecretReveal, HostResponseEntityInvocation, HostResponseP3BlobstoreIncomingValueStream, HostResponseP3HttpClientConsumeBodyChunk, HostResponseP3HttpClientConsumeBodyResult, HostResponseP3HttpClientSendResult, HostResponseP3KeyvalueIncomingValueStream, HostResponseP3SocketsTcpAcquire, HostResponseP3SocketsTcpReceiveChunk, HostResponseP3SocketsUdpReceive, - HostResponseP3SocketsUdpSend, + HostResponseP3SocketsUdpSend, HostResponseSecretRevealed, HostStreamKind, LogLevel, + OplogPayload, +}; +use golem_common::model::tool::{ + CompiledToolBinding, SecretKeyScope, ToolFilesystemAccess, ToolName, ToolProvisionConfig, + ToolSource, }; use golem_common::model::{ - AgentFingerprint, AgentMetadata, AgentStatusRecord, RetryConfig, Timestamp, + AgentFingerprint, AgentMetadata, AgentStatusRecord, RetryConfig, Timestamp, TransactionId, }; use golem_common::read_only_lock; -use golem_common::schema::IntoTypedSchemaValue; +use golem_common::schema::{IntoTypedSchemaValue, SecretValuePayload}; use golem_service_base::model::component::Component; use golem_service_base::storage::blob::memory::InMemoryBlobStorage; use prost::Message; @@ -131,6 +146,76 @@ fn header_map(key: &str, value: &[u8]) -> HashMap>> { HashMap::from_iter(vec![(key.to_string(), vec![value.to_vec()])]) } +fn test_entity_activation(entity: &AgentEntity) -> EntityActivation { + let component_id = golem_common::model::component::ComponentId::new(); + let component_revision = ComponentRevision::new(1).unwrap(); + let deployment_revision = DeploymentRevision::try_from(1_u64).unwrap(); + let executable = ExecutableTarget::new(component_id, component_revision); + let policy = match entity { + AgentEntity::Tool(tool_name) => EntityActivationPolicy::Tool { + provision: ToolProvisionConfig::default(), + binding: Box::new(CompiledToolBinding { + deployment_revision, + release_id: None, + agent_type_name: AgentTypeName("Agent".to_string()), + tool_name: tool_name.clone(), + version: "1".to_string(), + metadata_version: "1".to_string(), + metadata_digest: Default::default(), + account_id: AccountId::new(), + account_email: AccountEmail::new("owner@example.com"), + parameters: NormalizedJsonValue::new(serde_json::json!({})), + secret_keys_readable: SecretKeyScope::All, + secret_keys_revealable: SecretKeyScope::All, + filesystem_access: ToolFilesystemAccess::Unset, + source: ToolSource::Component { + component_id, + component_revision, + component_name: ComponentName("tools".to_string()), + }, + }), + }, + AgentEntity::ToolMiddleware(middleware_name) => EntityActivationPolicy::ToolMiddleware { + middleware_name: middleware_name.clone(), + provision: ToolProvisionConfig::default(), + secret_keys_readable: SecretKeyScope::All, + secret_keys_revealable: SecretKeyScope::All, + filesystem_access: ToolFilesystemAccess::Unset, + }, + }; + EntityActivation::new( + executable, + deployment_revision, + policy, + FilesystemCapability::Incapable, + ) + .unwrap() +} + +fn test_entity_request( + owner: &OwnedAgentId, + entity: AgentEntity, + call_mode: EntityCallMode, + operation: Option, + input: TypedSchemaValue, +) -> HostRequest { + let metadata = EntityInvocationRequest { + activation: test_entity_activation(&entity), + entity, + calling_principal: Principal::Agent(AgentPrincipal { + agent_id: owner.agent_id.clone(), + }), + call_mode, + operation, + principal: None, + }; + HostRequestEntityInvocation { + metadata: desert_rust::serialize_to_byte_vec(&metadata).unwrap(), + input, + } + .into() +} + #[test] async fn public_oplog_zero_start_reads_from_initial_index() { let indexed_storage = Arc::new(InMemoryIndexedStorage::new()); @@ -165,7 +250,12 @@ async fn public_oplog_zero_start_reads_from_initial_index() { .await; let timestamp = Timestamp::now_utc(); assert_eq!( - oplog.add(OplogEntry::NoOp { timestamp }).await, + oplog + .add(OplogEntry::NoOp { + timestamp, + entity_parent_start_index: None, + }) + .await, OplogIndex::INITIAL ); oplog.commit(CommitLevel::Always).await; @@ -186,7 +276,667 @@ async fn public_oplog_zero_start_reads_from_initial_index() { assert_eq!(chunk.first_index_in_chunk, OplogIndex::INITIAL); assert_eq!(chunk.next_oplog_index, OplogIndex::from_u64(2)); assert_eq!(chunk.entries.len(), 1); - assert!(matches!(chunk.entries[0], PublicOplogEntry::NoOp(_))); + assert!(matches!(chunk.entries[0].entry, PublicOplogEntry::NoOp(_))); +} + +#[test] +async fn entity_attribution_is_nested_page_independent_and_order_preserving() { + let indexed_storage = Arc::new(InMemoryIndexedStorage::new()); + let blob_storage = Arc::new(InMemoryBlobStorage::new()); + let oplog_service = Arc::new( + PrimaryOplogService::new( + indexed_storage, + blob_storage, + 1, + 1, + 100, + RetryConfig::default(), + ) + .await, + ); + let account_id = AccountId::new(); + let environment_id = EnvironmentId::new(); + let agent_id = AgentId { + component_id: golem_common::model::component::ComponentId::new(), + agent_id: "Agent(\"entity-attribution\")".to_string(), + }; + let owned_agent_id = OwnedAgentId::new(environment_id, &agent_id); + let oplog = oplog_service + .open( + &owned_agent_id, + AgentMode::Durable, + None, + make_agent_metadata(agent_id, account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + ) + .await; + + let agent_entry = oplog.add(OplogEntry::no_op(None)).await; + let observational_owner = oplog + .add(OplogEntry::Start { + timestamp: Timestamp::now_utc(), + parent_start_index: None, + function_name: HostFunctionName::Custom("agent-custom-owner".to_string()), + invocation_id: None, + observational_owner: None, + request: None, + durable_function_type: DurableFunctionType::WriteLocal, + }) + .await; + let middleware_entity = + AgentEntity::ToolMiddleware(ToolMiddlewareName::try_from("audit").unwrap()); + let middleware_input = "middleware-input" + .to_string() + .into_typed_schema_value() + .unwrap(); + let middleware_request = test_entity_request( + &owned_agent_id, + middleware_entity, + EntityCallMode::Synchronous, + None, + middleware_input.clone(), + ); + let middleware_start = oplog + .add(OplogEntry::Start { + timestamp: Timestamp::now_utc(), + parent_start_index: None, + function_name: HostFunctionName::GolemEntityInvoke, + invocation_id: None, + observational_owner: None, + request: Some(OplogPayload::Inline(Box::new(middleware_request))), + durable_function_type: DurableFunctionType::WriteLocal, + }) + .await; + + let interleaved_agent_entry = oplog.add(OplogEntry::no_op(None)).await; + let child_start = oplog + .add(OplogEntry::Start { + timestamp: Timestamp::now_utc(), + parent_start_index: Some(middleware_start), + function_name: HostFunctionName::Custom("entity-child".to_string()), + invocation_id: None, + observational_owner: None, + request: Some(OplogPayload::Inline(Box::new(HostRequestNoInput {}.into()))), + durable_function_type: DurableFunctionType::ReadLocal, + }) + .await; + + let tool_entity = AgentEntity::Tool(ToolName::try_from("lookup").unwrap()); + let secret_id = Uuid::from_u128(1); + let tool_input = TypedSchemaValue::new( + SchemaGraph::anonymous(SchemaType::secret( + golem_common::schema::schema_type::SecretSpec::default(), + )), + SchemaValue::Secret(SecretValuePayload { + secret_id, + config_key: Some(vec!["database".to_string(), "password".to_string()]), + version: 7, + resolved_at: chrono::DateTime::from_timestamp(1_700_000_000, 0).unwrap(), + category: Some("api-key".to_string()), + }), + ); + let tool_request = test_entity_request( + &owned_agent_id, + tool_entity, + EntityCallMode::Asynchronous, + Some(EntityInvocationDescriptor::Tool(ToolInvocationDescriptor { + attempt_ordinal: 0, + command_path: vec!["files".to_string(), "lookup".to_string()], + args: vec!["configured-secret-rendering".to_string()], + has_stdin: true, + has_stdout: true, + declares_stdout: true, + })), + tool_input.clone(), + ); + let tool_start = oplog + .add(OplogEntry::Start { + timestamp: Timestamp::now_utc(), + parent_start_index: Some(child_start), + function_name: HostFunctionName::GolemEntityInvoke, + invocation_id: None, + observational_owner: None, + request: Some(OplogPayload::Inline(Box::new(tool_request))), + durable_function_type: DurableFunctionType::WriteLocal, + }) + .await; + let entity_retry_error = oplog + .add(OplogEntry::error( + Some(tool_start), + golem_common::model::oplog::AgentError::TransientError("entity retry".to_string()), + agent_entry, + false, + None, + )) + .await; + let entity_marker = oplog.add(OplogEntry::no_op(Some(tool_start))).await; + let log_index = oplog + .add(OplogEntry::Log { + timestamp: Timestamp::now_utc(), + parent_start_index: Some(tool_start), + level: LogLevel::Info, + context: "tool".to_string(), + message: "entity-attribution-needle".to_string(), + }) + .await; + let span_id = SpanId::generate(); + let span_index = oplog + .add(OplogEntry::StartSpan { + timestamp: Timestamp::now_utc(), + parent_start_index: Some(tool_start), + span_id, + parent: None, + linked_context_id: None, + attributes: AttributeMap(HashMap::new()), + }) + .await; + let stream_frame_index = oplog + .add(OplogEntry::HostStreamFrame { + timestamp: Timestamp::now_utc(), + parent_start_index: tool_start, + kind: HostStreamKind::P3HttpRequestBody, + payload: OplogPayload::Inline(Box::new(HostRequestNoInput {}.into())), + }) + .await; + + let reveal_secret_id = Uuid::from_u128(2); + let reveal_request = HostRequestSecretReveal { + secret_id: reveal_secret_id, + expected_type: SchemaGraph::anonymous(SchemaType::string()), + }; + let reveal_response = HostResponseSecretRevealed { + secret_id: reveal_secret_id, + pinned_revision: 9, + resolved_at: SerializableDateTime { + seconds: 1_700_000_002, + nanoseconds: 0, + }, + result: Ok(()), + audit: SecretRevealAudit { + calling_agent: AgentId { + component_id: golem_common::model::component::ComponentId(Uuid::nil()), + agent_id: "secret-reveal-auditor".to_string(), + }, + config_key: Some(vec!["database".to_string(), "password".to_string()]), + timestamp: SerializableDateTime { + seconds: 1_700_000_003, + nanoseconds: 0, + }, + }, + }; + let reveal_request_payload: HostRequest = reveal_request.clone().into(); + let reveal_response_payload: HostResponse = reveal_response.clone().into(); + let (reveal_start, reveal_end) = oplog + .add_completed_host_call( + HostFunctionName::GolemSecretsReveal, + &reveal_request_payload, + &reveal_response_payload, + DurableFunctionType::ReadRemote, + Some(tool_start), + ) + .await + .unwrap(); + + let observational_start = oplog + .add(OplogEntry::Start { + timestamp: Timestamp::now_utc(), + parent_start_index: Some(tool_start), + function_name: HostFunctionName::Custom("observational-call".to_string()), + invocation_id: None, + observational_owner: Some(observational_owner), + request: None, + durable_function_type: DurableFunctionType::ReadLocal, + }) + .await; + let observational_log = oplog + .add(OplogEntry::Log { + timestamp: Timestamp::now_utc(), + parent_start_index: Some(observational_start), + level: LogLevel::Info, + context: "custom".to_string(), + message: "agent-owned observation".to_string(), + }) + .await; + let observational_end = oplog + .add(OplogEntry::end(observational_start, None, false)) + .await; + + let transaction_start = oplog + .add(OplogEntry::Start { + timestamp: Timestamp::now_utc(), + parent_start_index: Some(tool_start), + function_name: HostFunctionName::Custom("transaction".to_string()), + invocation_id: None, + observational_owner: None, + request: None, + durable_function_type: DurableFunctionType::WriteRemoteTransaction(None), + }) + .await; + let transaction_begin = oplog + .add(OplogEntry::BeginRemoteTransaction { + timestamp: Timestamp::now_utc(), + transaction_id: TransactionId::new("entity-transaction".to_string()), + original_begin_index: None, + }) + .await; + let transaction_commit = oplog + .add(OplogEntry::CommittedRemoteTransaction { + timestamp: Timestamp::now_utc(), + begin_index: transaction_start, + }) + .await; + let transaction_end = oplog + .add(OplogEntry::end(transaction_start, None, false)) + .await; + let child_end = oplog.add(OplogEntry::end(child_start, None, false)).await; + let tool_terminal = SerializableToolOperationTerminal { + body_execution: SerializableEntityBodyExecution::Executed, + result: Ok(SerializableToolStructuredResult { result: None }), + } + .into_typed_schema_value() + .unwrap(); + let tool_response: HostResponse = HostResponseEntityInvocation { + result: Ok(tool_terminal.clone()), + } + .into(); + let tool_end = oplog + .add(OplogEntry::end( + tool_start, + Some(OplogPayload::Inline(Box::new(tool_response))), + false, + )) + .await; + let completion = oplog + .add(OplogEntry::completion_delivered(tool_start)) + .await; + + let rejected_request: HostRequest = HostRequestGolemToolInvocationRejected { + attempt_ordinal: 0, + tool_name: "rejected".to_string(), + command_path: vec!["reject".to_string()], + input: None, + input_decode_failure: None, + has_stdin: false, + has_stdout: false, + call_mode: EntityCallMode::Synchronous, + error: SerializableToolRpcError::Denied("not allowed".to_string()), + } + .into(); + let rejected_start = oplog + .add(OplogEntry::Start { + timestamp: Timestamp::now_utc(), + parent_start_index: Some(tool_start), + function_name: HostFunctionName::GolemToolInvocationRejected, + invocation_id: None, + observational_owner: None, + request: Some(OplogPayload::Inline(Box::new(rejected_request))), + durable_function_type: DurableFunctionType::WriteLocal, + }) + .await; + let rejected_end = oplog + .add(OplogEntry::end(rejected_start, None, false)) + .await; + let middleware_end = oplog + .add(OplogEntry::end(middleware_start, None, false)) + .await; + let final_log = oplog + .add(OplogEntry::Log { + timestamp: Timestamp::now_utc(), + parent_start_index: Some(tool_start), + level: LogLevel::Info, + context: "tool".to_string(), + message: "last-entity-attribution-needle".to_string(), + }) + .await; + oplog.commit(CommitLevel::Always).await; + + let components: Arc = Arc::new(PanicComponentService); + let chunk = get_public_oplog_chunk( + components.clone(), + oplog_service.clone(), + &owned_agent_id, + AgentMode::Durable, + None, + ComponentRevision::INITIAL, + OplogIndex::INITIAL, + final_log.as_u64() as usize, + ) + .await + .unwrap(); + let expected_order = (agent_entry.as_u64()..=final_log.as_u64()) + .map(OplogIndex::from_u64) + .collect::>(); + assert_eq!( + chunk + .entries + .iter() + .map(|entry| entry.oplog_index) + .collect::>(), + expected_order + ); + assert!(matches!( + chunk.entries[agent_entry.as_u64() as usize - 1].attribution, + PublicOplogEntryAttribution::Agent(_) + )); + assert!(matches!( + chunk.entries[interleaved_agent_entry.as_u64() as usize - 1].attribution, + PublicOplogEntryAttribution::Agent(_) + )); + for index in [ + observational_owner, + observational_start, + observational_log, + observational_end, + ] { + assert!(matches!( + chunk.entries[index.as_u64() as usize - 1].attribution, + PublicOplogEntryAttribution::Agent(_) + )); + } + + let middleware = &chunk.entries[middleware_start.as_u64() as usize - 1]; + let PublicOplogEntryAttribution::Entity(middleware_context) = &middleware.attribution else { + panic!("middleware Start must be entity-attributed"); + }; + assert_eq!(middleware_context.invocation.entity.name, "audit"); + assert_eq!(middleware_context.invocation.start_index, middleware_start); + assert!(middleware_context.ancestors.is_empty()); + let PublicOplogEntry::Start(middleware_params) = &middleware.entry else { + panic!("expected middleware Start"); + }; + assert_eq!(middleware_params.request.as_ref(), Some(&middleware_input)); + + for index in [ + tool_start, + entity_retry_error, + entity_marker, + log_index, + span_index, + stream_frame_index, + reveal_start, + reveal_end, + transaction_start, + transaction_begin, + transaction_commit, + transaction_end, + tool_end, + completion, + final_log, + ] { + let entry = &chunk.entries[index.as_u64() as usize - 1]; + let PublicOplogEntryAttribution::Entity(context) = &entry.attribution else { + panic!("entry {index} must be attributed to the nested tool"); + }; + assert_eq!(context.invocation.entity.name, "lookup"); + assert_eq!(context.invocation.start_index, tool_start); + assert_eq!(context.ancestors.len(), 1); + assert_eq!(context.ancestors[0].entity.name, "audit"); + assert_eq!(context.ancestors[0].start_index, middleware_start); + } + for index in [child_start, child_end, middleware_end] { + let entry = &chunk.entries[index.as_u64() as usize - 1]; + let PublicOplogEntryAttribution::Entity(context) = &entry.attribution else { + panic!("entry {index} must be attributed to the middleware"); + }; + assert_eq!(context.invocation.start_index, middleware_start); + assert!(context.ancestors.is_empty()); + } + for index in [rejected_start, rejected_end] { + assert!(matches!( + chunk.entries[index.as_u64() as usize - 1].attribution, + PublicOplogEntryAttribution::Agent(_) + )); + } + + let tool = &chunk.entries[tool_start.as_u64() as usize - 1]; + let PublicOplogEntry::Start(tool_params) = &tool.entry else { + panic!("expected tool Start"); + }; + assert_eq!(tool_params.request.as_ref(), Some(&tool_input)); + let tool_json = serde_json::to_string(tool).unwrap(); + assert!(tool_json.contains(&secret_id.to_string())); + assert!(tool_json.contains("database")); + assert!(tool_json.contains("password")); + assert!(tool_json.contains("api-key")); + assert!(tool_json.contains("2023-11-14T22:13:20Z")); + assert!(!tool_json.contains("secretValue")); + assert!(!tool_json.contains("configured-secret-rendering")); + assert!(!tool_json.contains("owner@example.com")); + assert!(!tool_json.contains("metadata")); + + let PublicOplogEntry::Start(reveal_start_params) = + &chunk.entries[reveal_start.as_u64() as usize - 1].entry + else { + panic!("expected secret reveal Start"); + }; + assert_eq!( + reveal_start_params.request.as_ref(), + Some( + &reveal_request + .into_typed_schema_value() + .expect("secret reveal request must be schema-encodable") + ) + ); + let PublicOplogEntry::End(reveal_end_params) = + &chunk.entries[reveal_end.as_u64() as usize - 1].entry + else { + panic!("expected secret reveal End"); + }; + assert_eq!( + reveal_end_params.response.as_ref(), + Some( + &reveal_response + .into_typed_schema_value() + .expect("secret reveal response must be schema-encodable") + ) + ); + let reveal_json = serde_json::to_string(&[ + &chunk.entries[reveal_start.as_u64() as usize - 1], + &chunk.entries[reveal_end.as_u64() as usize - 1], + ]) + .unwrap(); + for safe_metadata in [ + "secret-reveal-auditor".to_string(), + "database".to_string(), + "password".to_string(), + "1700000002".to_string(), + "1700000003".to_string(), + ] { + assert!( + reveal_json.contains(&safe_metadata), + "expected {safe_metadata:?} in {reveal_json}" + ); + } + assert!(!reveal_json.contains("secretValue")); + + let tool_terminal_entry = &chunk.entries[tool_end.as_u64() as usize - 1]; + let PublicOplogEntry::End(tool_terminal_params) = &tool_terminal_entry.entry else { + panic!("expected tool End"); + }; + assert_eq!(tool_terminal_params.response.as_ref(), Some(&tool_terminal)); + assert!( + !serde_json::to_string(tool_terminal_entry) + .unwrap() + .contains("stdout") + ); + + let page = get_public_oplog_chunk( + components.clone(), + oplog_service.clone(), + &owned_agent_id, + AgentMode::Durable, + None, + ComponentRevision::INITIAL, + log_index, + 1, + ) + .await + .unwrap(); + assert_eq!(page.entries.len(), 1); + let PublicOplogEntryAttribution::Entity(page_context) = &page.entries[0].attribution else { + panic!("page beginning inside an entity must resolve historical attribution"); + }; + assert_eq!(page_context.invocation.start_index, tool_start); + assert_eq!(page_context.ancestors[0].start_index, middleware_start); + + let search = search_public_oplog( + components, + oplog_service, + &owned_agent_id, + AgentMode::Durable, + None, + ComponentRevision::INITIAL, + OplogIndex::INITIAL, + 1, + "last-entity-attribution-needle", + ) + .await + .unwrap(); + assert_eq!(search.entries.len(), 1); + assert_eq!(search.entries[0].oplog_index, final_log); + let PublicOplogEntryAttribution::Entity(search_context) = &search.entries[0].attribution else { + panic!("search result without its Start must retain entity attribution"); + }; + assert_eq!(search_context.invocation.start_index, tool_start); + assert_eq!(search_context.ancestors[0].start_index, middleware_start); +} + +#[test] +async fn explicit_entity_attribution_rejects_non_causal_and_non_entity_anchors() { + let indexed_storage = Arc::new(InMemoryIndexedStorage::new()); + let blob_storage = Arc::new(InMemoryBlobStorage::new()); + let oplog_service = Arc::new( + PrimaryOplogService::new( + indexed_storage, + blob_storage, + 1, + 1, + 100, + RetryConfig::default(), + ) + .await, + ); + let account_id = AccountId::new(); + let environment_id = EnvironmentId::new(); + let agent_id = AgentId { + component_id: golem_common::model::component::ComponentId::new(), + agent_id: "Agent(\"invalid-entity-attribution\")".to_string(), + }; + let owned_agent_id = OwnedAgentId::new(environment_id, &agent_id); + let oplog = oplog_service + .open( + &owned_agent_id, + AgentMode::Durable, + None, + make_agent_metadata(agent_id, account_id, environment_id), + default_last_known_status(), + default_execution_status(AgentMode::Durable), + ) + .await; + + let non_start = oplog.add(OplogEntry::no_op(None)).await; + let non_entity_start = oplog + .add(OplogEntry::Start { + timestamp: Timestamp::now_utc(), + parent_start_index: None, + function_name: HostFunctionName::Custom("not-an-entity".to_string()), + invocation_id: None, + observational_owner: None, + request: None, + durable_function_type: DurableFunctionType::WriteLocal, + }) + .await; + let entity_request = test_entity_request( + &owned_agent_id, + AgentEntity::Tool(ToolName::try_from("valid").unwrap()), + EntityCallMode::Synchronous, + None, + "input".to_string().into_typed_schema_value().unwrap(), + ); + let entity_start = oplog + .add(OplogEntry::Start { + timestamp: Timestamp::now_utc(), + parent_start_index: None, + function_name: HostFunctionName::GolemEntityInvoke, + invocation_id: None, + observational_owner: None, + request: Some(OplogPayload::Inline(Box::new(entity_request))), + durable_function_type: DurableFunctionType::WriteLocal, + }) + .await; + let valid = oplog.add(OplogEntry::no_op(Some(entity_start))).await; + let invalid_non_start = oplog.add(OplogEntry::no_op(Some(non_start))).await; + let invalid_non_entity = oplog.add(OplogEntry::no_op(Some(non_entity_start))).await; + let invalid_forward_index = invalid_non_entity.next(); + let future_entity_start = invalid_forward_index.next(); + assert_eq!( + oplog + .add(OplogEntry::no_op(Some(future_entity_start))) + .await, + invalid_forward_index + ); + assert_eq!( + oplog + .add(OplogEntry::Start { + timestamp: Timestamp::now_utc(), + parent_start_index: None, + function_name: HostFunctionName::GolemEntityInvoke, + invocation_id: None, + observational_owner: None, + request: None, + durable_function_type: DurableFunctionType::WriteLocal, + }) + .await, + future_entity_start + ); + oplog.commit(CommitLevel::Always).await; + + let components: Arc = Arc::new(PanicComponentService); + let valid_chunk = get_public_oplog_chunk( + components.clone(), + oplog_service.clone(), + &owned_agent_id, + AgentMode::Durable, + None, + ComponentRevision::INITIAL, + valid, + 1, + ) + .await + .unwrap(); + assert!(matches!( + valid_chunk.entries[0].attribution, + PublicOplogEntryAttribution::Entity(_) + )); + + for (index, expected_error) in [ + (invalid_non_start, "does not reference a Start"), + (invalid_non_entity, "is not an entity invocation"), + ( + invalid_forward_index, + "has non-causal entity parent Start index", + ), + ] { + let result = get_public_oplog_chunk( + components.clone(), + oplog_service.clone(), + &owned_agent_id, + AgentMode::Durable, + None, + ComponentRevision::INITIAL, + index, + 1, + ) + .await; + let Err(error) = result else { + panic!("expected invalid entity attribution at {index} to fail"); + }; + assert!( + error.to_string().contains(expected_error), + "expected {expected_error:?}, got {error}" + ); + } } /// Renders P3 host call oplog entries (`P3HttpClientSend`, @@ -400,14 +1150,13 @@ async fn p3_payloads_render_through_public_oplog_api_and_wit() { .unwrap(); let entity_request: HostRequest = HostRequestEntityInvocation { metadata: vec![1, 2, 3], - input: entity_input, + input: entity_input.clone(), } .into(); let entity_response: HostResponse = HostResponseEntityInvocation { result: Ok(entity_terminal.clone()), } .into(); - let expected_request = entity_request.clone().into_typed_schema_value().unwrap(); let (entity_start_idx, entity_end_idx) = oplog .add_completed_host_call( HostFunctionName::GolemEntityInvoke, @@ -422,7 +1171,7 @@ async fn p3_payloads_render_through_public_oplog_api_and_wit() { entity_start_idx, ( HostFunctionName::GolemEntityInvoke.to_string(), - expected_request, + entity_input, ), ); expected_ends.insert(entity_end_idx, entity_terminal); diff --git a/golem-worker-executor/src/model/public_oplog/wit.rs b/golem-worker-executor/src/model/public_oplog/wit.rs index 4ef445f1fb..57b6dc6285 100644 --- a/golem-worker-executor/src/model/public_oplog/wit.rs +++ b/golem-worker-executor/src/model/public_oplog/wit.rs @@ -1234,6 +1234,7 @@ impl TryFrom for golem_common::model::oplog::OplogEntry { }), oplog::OplogEntry::Error(params) => Ok(Self::Error { timestamp: timestamp_from_datetime(params.timestamp), + entity_parent_start_index: None, error: params.error.into(), retry_from: golem_common::model::OplogIndex::from_u64(params.retry_from), inside_atomic_region: params.inside_atomic_region, @@ -1244,9 +1245,11 @@ impl TryFrom for golem_common::model::oplog::OplogEntry { }), oplog::OplogEntry::NoOp(ts) => Ok(Self::NoOp { timestamp: timestamp_from_datetime(ts.timestamp), + entity_parent_start_index: None, }), oplog::OplogEntry::Jump(params) => Ok(Self::Jump { timestamp: timestamp_from_datetime(params.timestamp), + entity_parent_start_index: None, jump: golem_common::model::regions::OplogRegion { start: golem_common::model::OplogIndex::from_u64(params.jump.start), end: golem_common::model::OplogIndex::from_u64(params.jump.end), @@ -1260,9 +1263,11 @@ impl TryFrom for golem_common::model::oplog::OplogEntry { }), oplog::OplogEntry::BeginAtomicRegion(ts) => Ok(Self::BeginAtomicRegion { timestamp: timestamp_from_datetime(ts.timestamp), + entity_parent_start_index: None, }), oplog::OplogEntry::EndAtomicRegion(params) => Ok(Self::EndAtomicRegion { timestamp: timestamp_from_datetime(params.timestamp), + entity_parent_start_index: None, begin_index: golem_common::model::OplogIndex::from_u64(params.begin_index), }), oplog::OplogEntry::PendingAgentInvocation(params) => { @@ -1317,6 +1322,7 @@ impl TryFrom for golem_common::model::oplog::OplogEntry { }), oplog::OplogEntry::CreateResource(params) => Ok(Self::CreateResource { timestamp: timestamp_from_datetime(params.timestamp), + entity_parent_start_index: None, id: golem_common::model::oplog::AgentResourceId(params.id), resource_type_id: golem_common::resource_runtime::ResourceTypeId { name: params.resource_type_id.name, @@ -1325,6 +1331,7 @@ impl TryFrom for golem_common::model::oplog::OplogEntry { }), oplog::OplogEntry::DropResource(params) => Ok(Self::DropResource { timestamp: timestamp_from_datetime(params.timestamp), + entity_parent_start_index: None, id: golem_common::model::oplog::AgentResourceId(params.id), resource_type_id: golem_common::resource_runtime::ResourceTypeId { name: params.resource_type_id.name, @@ -1478,15 +1485,18 @@ impl TryFrom for golem_common::model::oplog::OplogEntry { params.policy.into(); Ok(Self::SetRetryPolicy { timestamp: timestamp_from_datetime(params.timestamp), + entity_parent_start_index: None, policy: named, }) } oplog::OplogEntry::RemoveRetryPolicy(params) => Ok(Self::RemoveRetryPolicy { timestamp: timestamp_from_datetime(params.timestamp), + entity_parent_start_index: None, name: params.name, }), oplog::OplogEntry::CardRevoked(params) => Ok(Self::CardRevoked { timestamp: timestamp_from_datetime(params.timestamp), + entity_parent_start_index: None, queued_event_index: golem_common::model::OplogIndex::from_u64( params.queued_event_index, ), @@ -1495,15 +1505,18 @@ impl TryFrom for golem_common::model::oplog::OplogEntry { }), oplog::OplogEntry::CardExpired(params) => Ok(Self::CardExpired { timestamp: timestamp_from_datetime(params.timestamp), + entity_parent_start_index: None, card_id: card_id_from_wit(params.card_id), wallet_generation: None, }), oplog::OplogEntry::CardEventQueued(params) => Ok(Self::CardEventQueued { timestamp: timestamp_from_datetime(params.timestamp), + entity_parent_start_index: None, event: raw_queued_card_event_from_wit(params.event)?, }), oplog::OplogEntry::CardInstalled(params) => Ok(Self::CardInstalled { timestamp: timestamp_from_datetime(params.timestamp), + entity_parent_start_index: None, queued_event_index: params .queued_event_index .map(golem_common::model::OplogIndex::from_u64), @@ -1512,6 +1525,7 @@ impl TryFrom for golem_common::model::oplog::OplogEntry { }), oplog::OplogEntry::CardInstallFailed(params) => Ok(Self::CardInstallFailed { timestamp: timestamp_from_datetime(params.timestamp), + entity_parent_start_index: None, queued_event_index: golem_common::model::OplogIndex::from_u64( params.queued_event_index, ), @@ -1528,22 +1542,27 @@ impl TryFrom for golem_common::model::oplog::OplogEntry { }), oplog::OplogEntry::StreamRegistered(params) => Ok(Self::StreamRegistered { timestamp: timestamp_from_datetime(params.timestamp), + entity_parent_start_index: None, record: oplog_payload_from_wit(params.record), }), oplog::OplogEntry::StreamItems(params) => Ok(Self::StreamItems { timestamp: timestamp_from_datetime(params.timestamp), + entity_parent_start_index: None, record: oplog_payload_from_wit(params.record), }), oplog::OplogEntry::StreamEnd(params) => Ok(Self::StreamEnd { timestamp: timestamp_from_datetime(params.timestamp), + entity_parent_start_index: None, record: oplog_payload_from_wit(params.record), }), oplog::OplogEntry::StreamCancel(params) => Ok(Self::StreamCancel { timestamp: timestamp_from_datetime(params.timestamp), + entity_parent_start_index: None, record: oplog_payload_from_wit(params.record), }), oplog::OplogEntry::StreamSession(params) => Ok(Self::StreamSession { timestamp: timestamp_from_datetime(params.timestamp), + entity_parent_start_index: None, record: oplog_payload_from_wit(params.record), }), } @@ -2002,6 +2021,7 @@ impl TryFrom for oplog::OplogEntry { retry_from, inside_atomic_region, retry_policy_state, + .. } => Ok(Self::Error(oplog::RawErrorParameters { timestamp: timestamp.into(), error: error.into(), @@ -2009,8 +2029,10 @@ impl TryFrom for oplog::OplogEntry { inside_atomic_region, retry_policy_state: retry_policy_state.map(|s| s.into()), })), - M::NoOp { timestamp } => Ok(Self::NoOp(timestamp.into())), - M::Jump { timestamp, jump } => Ok(Self::Jump(oplog::JumpParameters { + M::NoOp { timestamp, .. } => Ok(Self::NoOp(timestamp.into())), + M::Jump { + timestamp, jump, .. + } => Ok(Self::Jump(oplog::JumpParameters { timestamp: timestamp.into(), jump: oplog::OplogRegion { start: jump.start.into(), @@ -2019,10 +2041,11 @@ impl TryFrom for oplog::OplogEntry { })), M::Interrupted { timestamp } => Ok(Self::Interrupted(timestamp.into())), M::Exited { timestamp } => Ok(Self::Exited(timestamp.into())), - M::BeginAtomicRegion { timestamp } => Ok(Self::BeginAtomicRegion(timestamp.into())), + M::BeginAtomicRegion { timestamp, .. } => Ok(Self::BeginAtomicRegion(timestamp.into())), M::EndAtomicRegion { timestamp, begin_index, + .. } => Ok(Self::EndAtomicRegion(oplog::EndAtomicRegionParameters { timestamp: timestamp.into(), begin_index: begin_index.into(), @@ -2084,16 +2107,14 @@ impl TryFrom for oplog::OplogEntry { timestamp, queued_event_index, card_id, - wallet_generation: _, + .. } => Ok(Self::CardRevoked(oplog::CardRevokedParameters { timestamp: timestamp.into(), queued_event_index: queued_event_index.into(), card_id: card_id_to_wit(card_id), })), M::CardExpired { - timestamp, - card_id, - wallet_generation: _, + timestamp, card_id, .. } => Ok(Self::CardExpired(oplog::CardExpiredParameters { timestamp: timestamp.into(), card_id: card_id_to_wit(card_id), @@ -2109,47 +2130,53 @@ impl TryFrom for oplog::OplogEntry { kind: kind.into(), payload: oplog_payload_to_wit(payload)?, })), - M::StreamRegistered { timestamp, record } => Ok(Self::StreamRegistered( + M::StreamRegistered { + timestamp, record, .. + } => Ok(Self::StreamRegistered( oplog::RawDurableStreamRecordParameters { timestamp: timestamp.into(), record: oplog_payload_to_wit(record)?, }, )), - M::StreamItems { timestamp, record } => { - Ok(Self::StreamItems(oplog::RawDurableStreamRecordParameters { - timestamp: timestamp.into(), - record: oplog_payload_to_wit(record)?, - })) - } - M::StreamEnd { timestamp, record } => { - Ok(Self::StreamEnd(oplog::RawDurableStreamRecordParameters { - timestamp: timestamp.into(), - record: oplog_payload_to_wit(record)?, - })) - } - M::StreamCancel { timestamp, record } => Ok(Self::StreamCancel( + M::StreamItems { + timestamp, record, .. + } => Ok(Self::StreamItems(oplog::RawDurableStreamRecordParameters { + timestamp: timestamp.into(), + record: oplog_payload_to_wit(record)?, + })), + M::StreamEnd { + timestamp, record, .. + } => Ok(Self::StreamEnd(oplog::RawDurableStreamRecordParameters { + timestamp: timestamp.into(), + record: oplog_payload_to_wit(record)?, + })), + M::StreamCancel { + timestamp, record, .. + } => Ok(Self::StreamCancel( oplog::RawDurableStreamRecordParameters { timestamp: timestamp.into(), record: oplog_payload_to_wit(record)?, }, )), - M::StreamSession { timestamp, record } => Ok(Self::StreamSession( + M::StreamSession { + timestamp, record, .. + } => Ok(Self::StreamSession( oplog::RawDurableStreamRecordParameters { timestamp: timestamp.into(), record: oplog_payload_to_wit(record)?, }, )), - M::CardEventQueued { timestamp, event } => { - Ok(Self::CardEventQueued(oplog::CardEventQueuedParameters { - timestamp: timestamp.into(), - event: raw_queued_card_event_to_wit(event)?, - })) - } + M::CardEventQueued { + timestamp, event, .. + } => Ok(Self::CardEventQueued(oplog::CardEventQueuedParameters { + timestamp: timestamp.into(), + event: raw_queued_card_event_to_wit(event)?, + })), M::CardInstalled { timestamp, queued_event_index, card, - wallet_generation: _, + .. } => Ok(Self::CardInstalled(oplog::RawCardInstalledParameters { timestamp: timestamp.into(), queued_event_index: queued_event_index.map(Into::into), @@ -2160,6 +2187,7 @@ impl TryFrom for oplog::OplogEntry { queued_event_index, card_id, reason, + .. } => Ok(Self::CardInstallFailed( oplog::CardInstallFailedParameters { timestamp: timestamp.into(), @@ -2180,6 +2208,7 @@ impl TryFrom for oplog::OplogEntry { timestamp, id, resource_type_id, + .. } => Ok(Self::CreateResource(oplog::RawCreateResourceParameters { timestamp: timestamp.into(), id: id.0, @@ -2192,6 +2221,7 @@ impl TryFrom for oplog::OplogEntry { timestamp, id, resource_type_id, + .. } => Ok(Self::DropResource(oplog::RawDropResourceParameters { timestamp: timestamp.into(), id: id.0, @@ -2361,13 +2391,15 @@ impl TryFrom for oplog::OplogEntry { last_batch_start: last_batch_start.into(), }, )), - M::SetRetryPolicy { timestamp, policy } => { - Ok(Self::SetRetryPolicy(oplog::SetRetryPolicyParameters { - timestamp: timestamp.into(), - policy: policy.into(), - })) - } - M::RemoveRetryPolicy { timestamp, name } => Ok(Self::RemoveRetryPolicy( + M::SetRetryPolicy { + timestamp, policy, .. + } => Ok(Self::SetRetryPolicy(oplog::SetRetryPolicyParameters { + timestamp: timestamp.into(), + policy: policy.into(), + })), + M::RemoveRetryPolicy { + timestamp, name, .. + } => Ok(Self::RemoveRetryPolicy( oplog::RemoveRetryPolicyParameters { timestamp: timestamp.into(), name, @@ -2422,12 +2454,14 @@ mod tests { ( OplogEntry::CardInstalled { timestamp, + entity_parent_start_index: None, queued_event_index: None, card: card.clone().into(), wallet_generation: Some(1), }, OplogEntry::CardInstalled { timestamp, + entity_parent_start_index: None, queued_event_index: None, card: card.clone().into(), wallet_generation: None, @@ -2436,12 +2470,14 @@ mod tests { ( OplogEntry::CardRevoked { timestamp, + entity_parent_start_index: None, queued_event_index: golem_common::model::OplogIndex::NONE, card_id, wallet_generation: Some(3), }, OplogEntry::CardRevoked { timestamp, + entity_parent_start_index: None, queued_event_index: golem_common::model::OplogIndex::NONE, card_id, wallet_generation: None, @@ -2450,11 +2486,13 @@ mod tests { ( OplogEntry::CardExpired { timestamp, + entity_parent_start_index: None, card_id, wallet_generation: Some(4), }, OplogEntry::CardExpired { timestamp, + entity_parent_start_index: None, card_id, wallet_generation: None, }, @@ -2469,6 +2507,7 @@ mod tests { assert!( oplog::OplogEntry::try_from(OplogEntry::CardDerived { timestamp, + entity_parent_start_index: None, card: card.into(), wallet_generation: Some(2), }) @@ -2530,6 +2569,7 @@ mod tests { let payload_card_id = CardId::new(); let entry = OplogEntry::CardTransferred { timestamp: Timestamp::now_utc().rounded(), + entity_parent_start_index: None, transfer_id: Uuid::new_v4(), source_card_id: Some(payload_card_id), installed_card_id: CardId::new(), @@ -2563,6 +2603,7 @@ mod tests { let payload_card_id = CardId::new(); let entry = OplogEntry::CardEventQueued { timestamp: Timestamp::now_utc().rounded(), + entity_parent_start_index: None, event: QueuedCardEvent::TransferStarted(QueuedCardEventTransfer { transfer_id: Uuid::new_v4(), card_id: CardId::new(), @@ -2588,6 +2629,7 @@ mod tests { let source_card_id = CardId::new(); let entry = OplogEntry::CardEventQueued { timestamp: Timestamp::now_utc().rounded(), + entity_parent_start_index: None, event: QueuedCardEvent::TransferReceived(QueuedCardEventTransferReceived { transfer_id: Uuid::new_v4(), source_card_id: Some(source_card_id), diff --git a/golem-worker-executor/src/services/oplog/mod.rs b/golem-worker-executor/src/services/oplog/mod.rs index 5b53cca2c4..98ef59545e 100644 --- a/golem-worker-executor/src/services/oplog/mod.rs +++ b/golem-worker-executor/src/services/oplog/mod.rs @@ -391,41 +391,48 @@ pub struct OrderedOplogStart { } pub enum DurableStreamOplogRecord { - Registered(StreamRegisteredRecordV1), - Items(StreamItemsRecordV1), - End(StreamEndRecordV1), - Cancel(StreamCancelRecordV1), - Session(Box), + Registered(Option, StreamRegisteredRecordV1), + Items(Option, StreamItemsRecordV1), + End(Option, StreamEndRecordV1), + Cancel(Option, StreamCancelRecordV1), + Session(Option, Box), InlineEntry(OplogEntry), } impl DurableStreamOplogRecord { fn serialize(&self) -> Result, String> { match self { - Self::Registered(record) => serialize(record), - Self::Items(record) => serialize(record), - Self::End(record) => serialize(record), - Self::Cancel(record) => serialize(record), - Self::Session(record) => serialize(record), + Self::Registered(_, record) => serialize(record), + Self::Items(_, record) => serialize(record), + Self::End(_, record) => serialize(record), + Self::Cancel(_, record) => serialize(record), + Self::Session(_, record) => serialize(record), Self::InlineEntry(_) => Ok(Vec::new()), } } fn into_entry(self, raw: RawOplogPayload) -> Result { match self { - Self::Registered(record) => Ok(OplogEntry::stream_registered( - raw.into_payload_with_cache(Arc::new(record))?, - )), - Self::Items(record) => Ok(OplogEntry::stream_items( + Self::Registered(entity_parent_start_index, record) => { + Ok(OplogEntry::stream_registered( + entity_parent_start_index, + raw.into_payload_with_cache(Arc::new(record))?, + )) + } + Self::Items(entity_parent_start_index, record) => Ok(OplogEntry::stream_items( + entity_parent_start_index, raw.into_payload_with_cache(Arc::new(record))?, )), - Self::End(record) => Ok(OplogEntry::stream_end( + Self::End(entity_parent_start_index, record) => Ok(OplogEntry::stream_end( + entity_parent_start_index, raw.into_payload_with_cache(Arc::new(record))?, )), - Self::Cancel(record) => Ok(OplogEntry::stream_cancel( + Self::Cancel(entity_parent_start_index, record) => Ok(OplogEntry::stream_cancel( + entity_parent_start_index, raw.into_payload_with_cache(Arc::new(record))?, )), - Self::Session(record) => Ok(OplogEntry::stream_session( + Self::Session(entity_parent_start_index, record) => Ok(OplogEntry::stream_session( + entity_parent_start_index, raw.into_payload_with_cache(Arc::from(record))?, )), Self::InlineEntry(entry) => Ok(entry), @@ -434,15 +441,25 @@ impl DurableStreamOplogRecord { pub fn into_inline_entry(self) -> OplogEntry { match self { - Self::Registered(record) => { - OplogEntry::stream_registered(OplogPayload::Inline(Box::new(record))) - } - Self::Items(record) => OplogEntry::stream_items(OplogPayload::Inline(Box::new(record))), - Self::End(record) => OplogEntry::stream_end(OplogPayload::Inline(Box::new(record))), - Self::Cancel(record) => { - OplogEntry::stream_cancel(OplogPayload::Inline(Box::new(record))) + Self::Registered(entity_parent_start_index, record) => OplogEntry::stream_registered( + entity_parent_start_index, + OplogPayload::Inline(Box::new(record)), + ), + Self::Items(entity_parent_start_index, record) => OplogEntry::stream_items( + entity_parent_start_index, + OplogPayload::Inline(Box::new(record)), + ), + Self::End(entity_parent_start_index, record) => OplogEntry::stream_end( + entity_parent_start_index, + OplogPayload::Inline(Box::new(record)), + ), + Self::Cancel(entity_parent_start_index, record) => OplogEntry::stream_cancel( + entity_parent_start_index, + OplogPayload::Inline(Box::new(record)), + ), + Self::Session(entity_parent_start_index, record) => { + OplogEntry::stream_session(entity_parent_start_index, OplogPayload::Inline(record)) } - Self::Session(record) => OplogEntry::stream_session(OplogPayload::Inline(record)), Self::InlineEntry(entry) => entry, } } diff --git a/golem-worker-executor/src/services/oplog/plugin.rs b/golem-worker-executor/src/services/oplog/plugin.rs index 7757df1731..cc523d2457 100644 --- a/golem-worker-executor/src/services/oplog/plugin.rs +++ b/golem-worker-executor/src/services/oplog/plugin.rs @@ -2753,10 +2753,12 @@ mod tests { let first = oplog.enqueue_add(OplogEntry::NoOp { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, }); let second = oplog .add(OplogEntry::NoOp { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, }) .await; diff --git a/golem-worker-executor/src/services/oplog/rate_limited.rs b/golem-worker-executor/src/services/oplog/rate_limited.rs index 146f46f4d5..59ac07d30f 100644 --- a/golem-worker-executor/src/services/oplog/rate_limited.rs +++ b/golem-worker-executor/src/services/oplog/rate_limited.rs @@ -660,10 +660,13 @@ mod tests { } fn dummy_entry() -> OplogEntry { - OplogEntry::jump(OplogRegion { - start: OplogIndex::from_u64(1), - end: OplogIndex::from_u64(1), - }) + OplogEntry::jump( + None, + OplogRegion { + start: OplogIndex::from_u64(1), + end: OplogIndex::from_u64(1), + }, + ) } // When writes exceed the configured rate, subsequent adds are delayed. diff --git a/golem-worker-executor/src/services/oplog/tests.rs b/golem-worker-executor/src/services/oplog/tests.rs index a29dfd5fb2..c421e04c27 100644 --- a/golem-worker-executor/src/services/oplog/tests.rs +++ b/golem-worker-executor/src/services/oplog/tests.rs @@ -1384,10 +1384,13 @@ async fn open_add_and_read_back(_tracing: &Tracing) { ) .await; - let entry1 = OplogEntry::jump(OplogRegion { - start: OplogIndex::from_u64(5), - end: OplogIndex::from_u64(12), - }) + let entry1 = OplogEntry::jump( + None, + OplogRegion { + start: OplogIndex::from_u64(5), + end: OplogIndex::from_u64(12), + }, + ) .rounded(); let entry2 = OplogEntry::suspend().rounded(); let entry3 = OplogEntry::exited().rounded(); @@ -1606,57 +1609,69 @@ async fn durable_stream_batch_externalizes_every_record_family(_tracing: &Tracin let end_index = item_index.next(); let cancel_index = end_index.next(); vec![ - DurableStreamOplogRecord::Registered(StreamRegisteredRecordV1 { - format_version: 1, - coordinate: StreamRegistrationCoordinateV1::Root { - invocation_id: invocation_id.clone(), - root_kind: StreamRootKindV1::MethodResult, - recursive_value_path: Vec::new(), + DurableStreamOplogRecord::Registered( + None, + StreamRegisteredRecordV1 { + format_version: 1, + coordinate: StreamRegistrationCoordinateV1::Root { + invocation_id: invocation_id.clone(), + root_kind: StreamRootKindV1::MethodResult, + recursive_value_path: Vec::new(), + }, + registration_oplog_index: registration_index, + handle: DurableStreamHandleV1 { + format_version: 1, + stream_id, + producer_environment_id: environment_id, + producer: agent_id, + expected_producer_fingerprint: producer_fingerprint, + source_invocation: invocation_id, + component_revision: ComponentRevision::INITIAL, + element_schema_fingerprint: SchemaFingerprintV1([7; 32]), + }, + source_kind: StreamSourceKindV1::InvocationOutput, + session_mapping: None, }, - registration_oplog_index: registration_index, - handle: DurableStreamHandleV1 { + ), + DurableStreamOplogRecord::Items( + None, + StreamItemsRecordV1 { format_version: 1, stream_id, - producer_environment_id: environment_id, - producer: agent_id, - expected_producer_fingerprint: producer_fingerprint, - source_invocation: invocation_id, - component_revision: ComponentRevision::INITIAL, - element_schema_fingerprint: SchemaFingerprintV1([7; 32]), + producer_fingerprint, + first_sequence: 0, + nested_stream_ids: Vec::new(), + newly_registered_stream_ids: Vec::new(), + payload: StreamItemsPayloadV1::Values(vec![vec![42; 1024]]), + offsets: vec![StreamOffsetV1::new(item_index, 0)], }, - source_kind: StreamSourceKindV1::InvocationOutput, - session_mapping: None, - }), - DurableStreamOplogRecord::Items(StreamItemsRecordV1 { - format_version: 1, - stream_id, - producer_fingerprint, - first_sequence: 0, - nested_stream_ids: Vec::new(), - newly_registered_stream_ids: Vec::new(), - payload: StreamItemsPayloadV1::Values(vec![vec![42; 1024]]), - offsets: vec![StreamOffsetV1::new(item_index, 0)], - }), - DurableStreamOplogRecord::End(StreamEndRecordV1 { - format_version: 1, - stream_id, - producer_fingerprint, - sequence: 1, - offset: StreamOffsetV1::new(end_index, 0), - authored_by: StreamTerminalAuthorV1::Guest, - result: StreamEndResultV1::Ok, - }), - DurableStreamOplogRecord::Cancel(StreamCancelRecordV1 { - format_version: 1, - stream_id, - producer_fingerprint, - sequence: 1, - offset: StreamOffsetV1::new(cancel_index, 0), - authored_by: StreamTerminalAuthorV1::Protocol, - role: StreamCancelRoleV1::OutputConsumer, - reason: StreamCancelReasonV1::Protocol, - details: Some("test cancellation".to_string()), - }), + ), + DurableStreamOplogRecord::End( + None, + StreamEndRecordV1 { + format_version: 1, + stream_id, + producer_fingerprint, + sequence: 1, + offset: StreamOffsetV1::new(end_index, 0), + authored_by: StreamTerminalAuthorV1::Guest, + result: StreamEndResultV1::Ok, + }, + ), + DurableStreamOplogRecord::Cancel( + None, + StreamCancelRecordV1 { + format_version: 1, + stream_id, + producer_fingerprint, + sequence: 1, + offset: StreamOffsetV1::new(cancel_index, 0), + authored_by: StreamTerminalAuthorV1::Protocol, + role: StreamCancelRoleV1::OutputConsumer, + reason: StreamCancelReasonV1::Protocol, + details: Some("test cancellation".to_string()), + }, + ), ] })) .await @@ -1739,6 +1754,7 @@ async fn durable_stream_producer_recovers_from_sqlite_storage_restart(_tracing: element_schema_fingerprint: SchemaFingerprintV1([7; 32]), source_kind: StreamSourceKindV1::InvocationOutput, session_mapping: None, + entity_parent_start_index: None, }; let indexed_storage: Arc = @@ -1884,15 +1900,18 @@ async fn open_add_and_read_back_many(_tracing: &Tracing) { ) .await; - let entry1 = OplogEntry::jump(OplogRegion { - start: OplogIndex::from_u64(5), - end: OplogIndex::from_u64(12), - }) + let entry1 = OplogEntry::jump( + None, + OplogRegion { + start: OplogIndex::from_u64(5), + end: OplogIndex::from_u64(12), + }, + ) .rounded(); let entry2 = OplogEntry::suspend().rounded(); let entry3 = OplogEntry::exited().rounded(); let entry4 = OplogEntry::interrupted().rounded(); - let entry5 = OplogEntry::no_op().rounded(); + let entry5 = OplogEntry::no_op(None).rounded(); oplog.add(entry1.clone()).await; oplog.add(entry2.clone()).await; @@ -1991,10 +2010,13 @@ async fn open_add_and_read_back_ephemeral(_tracing: &Tracing) { ) .await; - let entry1 = OplogEntry::jump(OplogRegion { - start: OplogIndex::from_u64(5), - end: OplogIndex::from_u64(12), - }) + let entry1 = OplogEntry::jump( + None, + OplogRegion { + start: OplogIndex::from_u64(5), + end: OplogIndex::from_u64(12), + }, + ) .rounded(); let entry2 = OplogEntry::suspend().rounded(); let entry3 = OplogEntry::exited().rounded(); @@ -2079,10 +2101,13 @@ async fn open_add_and_read_back_many_ephemeral(_tracing: &Tracing) { ) .await; - let entry1 = OplogEntry::jump(OplogRegion { - start: OplogIndex::from_u64(5), - end: OplogIndex::from_u64(12), - }) + let entry1 = OplogEntry::jump( + None, + OplogRegion { + start: OplogIndex::from_u64(5), + end: OplogIndex::from_u64(12), + }, + ) .rounded(); let entry2 = OplogEntry::suspend().rounded(); let entry3 = OplogEntry::exited().rounded(); @@ -2278,6 +2303,7 @@ async fn ephemeral_read_exact_partial_range(_tracing: &Tracing) { for i in 0..10 { let entry = OplogEntry::Error { timestamp, + entity_parent_start_index: None, error: AgentError::Unknown(i.to_string()), retry_from: OplogIndex::NONE, inside_atomic_region: false, @@ -2380,6 +2406,7 @@ async fn ephemeral_read_exact_across_archive_layers(_tracing: &Tracing) { .map(|i| { OplogEntry::Error { timestamp, + entity_parent_start_index: None, error: AgentError::Unknown(i.to_string()), retry_from: OplogIndex::NONE, inside_atomic_region: false, @@ -3270,6 +3297,7 @@ async fn read_from_archive_impl(use_blob: bool) { .map(|i| { OplogEntry::Error { timestamp, + entity_parent_start_index: None, error: AgentError::Unknown(i.to_string()), retry_from: OplogIndex::NONE, inside_atomic_region: false, @@ -3653,7 +3681,7 @@ async fn blob_write_after_archive_reopen_full(_tracing: &Tracing) { fn transfer_test_entries() -> BTreeMap { [ - (OplogIndex::INITIAL, OplogEntry::no_op().rounded()), + (OplogIndex::INITIAL, OplogEntry::no_op(None).rounded()), (OplogIndex::from_u64(2), OplogEntry::suspend().rounded()), ] .into_iter() @@ -3801,6 +3829,7 @@ async fn compressed_transfer_verification_bypasses_append_cache(_tracing: &Traci async fn blob_transfer_verifies_the_persisted_entry_representation(_tracing: &Tracing) { let entry = OplogEntry::NoOp { timestamp: "2026-08-27T13:09:36.123456Z".parse().unwrap(), + entity_parent_start_index: None, }; assert_ne!(entry, entry.clone().rounded()); @@ -3881,7 +3910,7 @@ async fn open_multilayer_oplog_retains_stale_index_after_service_deletion(_traci .create( &owned_agent_id, AgentMode::Durable, - OplogEntry::no_op(), + OplogEntry::no_op(None), make_agent_metadata(agent_id, account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), @@ -3983,7 +4012,7 @@ async fn deleting_worker_fences_in_flight_archive_transfers_impl(agent_mode: Age ) .await; - oplog.add(OplogEntry::no_op()).await; + oplog.add(OplogEntry::no_op(None)).await; oplog.commit(CommitLevel::Always).await; if agent_mode == AgentMode::Ephemeral { EphemeralOplog::try_archive(&oplog) @@ -4083,6 +4112,7 @@ async fn write_after_archive_impl(use_blob: bool, reopen: Reopen) { .map(|i| { OplogEntry::Error { timestamp, + entity_parent_start_index: None, error: AgentError::Unknown(i.to_string()), retry_from: OplogIndex::NONE, inside_atomic_region: false, @@ -4177,6 +4207,7 @@ async fn write_after_archive_impl(use_blob: bool, reopen: Reopen) { .map(|i| { OplogEntry::Error { timestamp, + entity_parent_start_index: None, error: AgentError::Unknown(i.to_string()), retry_from: OplogIndex::NONE, inside_atomic_region: false, @@ -4272,6 +4303,7 @@ async fn write_after_archive_impl(use_blob: bool, reopen: Reopen) { .add( OplogEntry::Error { timestamp, + entity_parent_start_index: None, error: AgentError::Unknown("last".to_string()), retry_from: OplogIndex::NONE, inside_atomic_region: false, @@ -4320,6 +4352,7 @@ async fn write_after_archive_impl(use_blob: bool, reopen: Reopen) { entry1.get(&OplogIndex::INITIAL).unwrap().clone(), OplogEntry::Error { timestamp, + entity_parent_start_index: None, error: AgentError::Unknown("0".to_string()), retry_from: OplogIndex::NONE, inside_atomic_region: false, @@ -4331,6 +4364,7 @@ async fn write_after_archive_impl(use_blob: bool, reopen: Reopen) { entry2.get(&OplogIndex::from_u64(100)).unwrap().clone(), OplogEntry::Error { timestamp, + entity_parent_start_index: None, error: AgentError::Unknown("99".to_string()), retry_from: OplogIndex::NONE, inside_atomic_region: false, @@ -4342,6 +4376,7 @@ async fn write_after_archive_impl(use_blob: bool, reopen: Reopen) { entry3.get(&OplogIndex::from_u64(1000)).unwrap().clone(), OplogEntry::Error { timestamp, + entity_parent_start_index: None, error: AgentError::Unknown("999".to_string()), retry_from: OplogIndex::NONE, inside_atomic_region: false, @@ -4353,6 +4388,7 @@ async fn write_after_archive_impl(use_blob: bool, reopen: Reopen) { entry4.get(&OplogIndex::from_u64(1001)).unwrap().clone(), OplogEntry::Error { timestamp, + entity_parent_start_index: None, error: AgentError::Unknown("last".to_string()), retry_from: OplogIndex::NONE, inside_atomic_region: false, @@ -4439,6 +4475,7 @@ async fn empty_layer_gets_deleted_impl(use_blob: bool) { .map(|i| { OplogEntry::Error { timestamp, + entity_parent_start_index: None, error: AgentError::Unknown(i.to_string()), retry_from: OplogIndex::NONE, inside_atomic_region: false, @@ -4564,6 +4601,7 @@ async fn scheduled_archive_impl(use_blob: bool) { .map(|i| { OplogEntry::Error { timestamp, + entity_parent_start_index: None, error: AgentError::Unknown(i.to_string()), retry_from: OplogIndex::NONE, inside_atomic_region: false, @@ -5020,10 +5058,13 @@ async fn concurrent_get_or_open_does_not_cause_unique_key_violation(_tracing: &T .create( &owned_agent_id, AgentMode::Durable, - OplogEntry::jump(OplogRegion { - start: OplogIndex::from_u64(0), - end: OplogIndex::from_u64(0), - }), + OplogEntry::jump( + None, + OplogRegion { + start: OplogIndex::from_u64(0), + end: OplogIndex::from_u64(0), + }, + ), make_agent_metadata(worker_id.clone(), account_id, environment_id), default_last_known_status(), default_execution_status(AgentMode::Durable), diff --git a/golem-worker-executor/src/services/worker_fork.rs b/golem-worker-executor/src/services/worker_fork.rs index 93495d5f81..cbd27affc7 100644 --- a/golem-worker-executor/src/services/worker_fork.rs +++ b/golem-worker-executor/src/services/worker_fork.rs @@ -1010,6 +1010,7 @@ mod tests { let remote = agent_id("remote"); let entry = OplogEntry::CardTransferStarted { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, transfer_id: Uuid::new_v4(), card_id: CardId::new(), source_holder: Some(CardHolder::Agent(AgentCardHolder { diff --git a/golem-worker-executor/src/worker/cut_point.rs b/golem-worker-executor/src/worker/cut_point.rs index b3f97afd51..621a12b5bd 100644 --- a/golem-worker-executor/src/worker/cut_point.rs +++ b/golem-worker-executor/src/worker/cut_point.rs @@ -275,7 +275,7 @@ mod tests { let entry = entries .get(&u64::from(i)) .cloned() - .unwrap_or_else(OplogEntry::no_op); + .unwrap_or_else(|| OplogEntry::no_op(None)); async move { entry } }, idx(cut), @@ -295,7 +295,7 @@ mod tests { let entry = entries .get(&u64::from(i)) .cloned() - .unwrap_or_else(OplogEntry::no_op); + .unwrap_or_else(|| OplogEntry::no_op(None)); async move { entry } }, idx(start), @@ -305,18 +305,21 @@ mod tests { } fn stream_entry() -> OplogEntry { - OplogEntry::stream_session(OplogPayload::Inline(Box::new( - StreamSessionRecordV1::ConsumerDeleting(StreamConsumerDeletingRecordV1 { - format_version: DURABLE_STREAM_FORMAT_VERSION, - consumer_environment_id: EnvironmentId(Uuid::from_u128(1)), - consumer: AgentId { - component_id: ComponentId(Uuid::from_u128(2)), - agent_id: "consumer".to_string(), + OplogEntry::stream_session( + None, + OplogPayload::Inline(Box::new(StreamSessionRecordV1::ConsumerDeleting( + StreamConsumerDeletingRecordV1 { + format_version: DURABLE_STREAM_FORMAT_VERSION, + consumer_environment_id: EnvironmentId(Uuid::from_u128(1)), + consumer: AgentId { + component_id: ComponentId(Uuid::from_u128(2)), + agent_id: "consumer".to_string(), + }, + consumer_fingerprint: AgentFingerprint(Uuid::from_u128(3)), + deleting_at_millis: 100, }, - consumer_fingerprint: AgentFingerprint(Uuid::from_u128(3)), - deleting_at_millis: 100, - }), - ))) + ))), + ) } #[test] @@ -416,7 +419,7 @@ mod tests { #[test] async fn end_atomic_region_after_cut_is_rejected() { - let entries = HashMap::from([(6, OplogEntry::end_atomic_region(idx(2)))]); + let entries = HashMap::from([(6, OplogEntry::end_atomic_region(None, idx(2)))]); assert_eq!( scan(&entries, 4, 6, &deleted(vec![])).await, Some(SpanningConstruct::AtomicRegion { @@ -487,6 +490,7 @@ mod tests { 3, OplogEntry::CardTransferStarted { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, transfer_id, card_id: source_card_id, source_holder: None, @@ -498,6 +502,7 @@ mod tests { 5, OplogEntry::CardTransferConfirmed { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, transfer_id, source_card_id, installed_card_id, @@ -532,7 +537,7 @@ mod tests { #[test] async fn first_spanning_construct_is_reported() { let entries = HashMap::from([ - (5, OplogEntry::end_atomic_region(idx(2))), + (5, OplogEntry::end_atomic_region(None, idx(2))), (6, OplogEntry::end(idx(3), None, false)), ]); assert_eq!( diff --git a/golem-worker-executor/src/worker/mod.rs b/golem-worker-executor/src/worker/mod.rs index f3db93cfd7..c8b98e3762 100644 --- a/golem-worker-executor/src/worker/mod.rs +++ b/golem-worker-executor/src/worker/mod.rs @@ -3699,16 +3699,19 @@ impl Worker { .add_pair( pending, Box::new(move |pending_invocation_oplog_index| { - OplogEntry::stream_session(OplogPayload::Inline(Box::new( - StreamSessionRecordV1::Attached(StreamSessionAttachedRecordV1 { - format_version: 1, - session_key: attached_attempt.session_key, - attachment_id: attached_attempt.attachment_id, - attempt_id: attached_attempt.attempt_id, - epoch: 1, - pending_invocation_oplog_index, - }), - ))) + OplogEntry::stream_session( + None, + OplogPayload::Inline(Box::new(StreamSessionRecordV1::Attached( + StreamSessionAttachedRecordV1 { + format_version: 1, + session_key: attached_attempt.session_key, + attachment_id: attached_attempt.attachment_id, + attempt_id: attached_attempt.attempt_id, + epoch: 1, + pending_invocation_oplog_index, + }, + ))), + ) }), ) .await; @@ -5021,9 +5024,10 @@ impl Worker { let mut queued_event_indices = Vec::with_capacity(card_ids.len()); for card_id in card_ids { queued_event_indices.push( - self.add_to_oplog(OplogEntry::card_event_queued(QueuedCardEvent::revoke( - card_id, - ))) + self.add_to_oplog(OplogEntry::card_event_queued( + None, + QueuedCardEvent::revoke(card_id), + )) .await, ); } @@ -5083,11 +5087,10 @@ impl Worker { let boundary_guard = self.card_event_boundary_lock.clone().lock_owned().await; self.state_actor .append_and_commit_attached( - OplogEntry::card_event_queued(QueuedCardEvent::transfer_received( - transfer_id, - source_card_id, - card, - )), + OplogEntry::card_event_queued( + None, + QueuedCardEvent::transfer_received(transfer_id, source_card_id, card), + ), self.clone(), instance_guard, boundary_guard, diff --git a/golem-worker-executor/src/worker/status.rs b/golem-worker-executor/src/worker/status.rs index 3416d8cb81..5d6294a3ac 100644 --- a/golem-worker-executor/src/worker/status.rs +++ b/golem-worker-executor/src/worker/status.rs @@ -773,10 +773,15 @@ pub(crate) fn calculate_pending_card_events( for (oplog_idx, entry) in entries { match entry { - OplogEntry::CardEventQueued { timestamp, event } => { + OplogEntry::CardEventQueued { + timestamp, + entity_parent_start_index, + event, + } => { result.push(PendingCardEventRef { timestamp: *timestamp, oplog_index: *oplog_idx, + entity_parent_start_index: *entity_parent_start_index, event: event.clone(), }); } @@ -1172,6 +1177,7 @@ fn collect_resources( id, timestamp, resource_type_id, + .. } => { result.insert( *id, @@ -1435,6 +1441,7 @@ mod test { .agent_invocation_started("a", vec![], idempotency_key.clone()) .add( OplogEntry::error( + None, AgentError::TransientError("transient".to_string()), retry_from, false, @@ -1467,6 +1474,7 @@ mod test { .agent_invocation_started("a", vec![], idempotency_key.clone()) .add( OplogEntry::error( + None, AgentError::TransientError("transient".to_string()), retry_from, false, @@ -2452,7 +2460,7 @@ mod test { let old_status = self.entries[u64::from(target) as usize - 1] .expected_status .clone(); - self.add(OplogEntry::jump(region.clone()), move |mut status| { + self.add(OplogEntry::jump(None, region.clone()), move |mut status| { status.status = old_status.status; status.component_revision = old_status.component_revision; status.current_idempotency_key = old_status.current_idempotency_key; @@ -2544,6 +2552,7 @@ mod test { pub fn permission_denied_pending_invocation(self, idempotency_key: IdempotencyKey) -> Self { self.cancel_pending_invocation(idempotency_key.clone()).add( OplogEntry::error( + None, AgentError::PermissionDenied("permission denied".to_string()), OplogIndex::INITIAL, false, @@ -3358,6 +3367,7 @@ mod test { OplogIndex::from_u64(1), OplogEntry::CardRevoked { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, queued_event_index: OplogIndex::from_u64(1), card_id, wallet_generation: None, @@ -3395,7 +3405,7 @@ mod test { let card_id = golem_common::model::card::CardId::new(); let entries = BTreeMap::from([( OplogIndex::from_u64(1), - OplogEntry::card_event_queued(QueuedCardEvent::revoke(card_id)), + OplogEntry::card_event_queued(None, QueuedCardEvent::revoke(card_id)), )]); let status = super::update_status_with_new_entries( @@ -3423,11 +3433,11 @@ mod test { let entries = BTreeMap::from([ ( OplogIndex::from_u64(1), - OplogEntry::card_event_queued(QueuedCardEvent::revoke(card_id)), + OplogEntry::card_event_queued(None, QueuedCardEvent::revoke(card_id)), ), ( OplogIndex::from_u64(2), - OplogEntry::card_revoked(OplogIndex::from_u64(1), card_id, None), + OplogEntry::card_revoked(None, OplogIndex::from_u64(1), card_id, None), ), ]); @@ -3451,6 +3461,7 @@ mod test { OplogIndex::from_u64(1), OplogEntry::CardRevokedCascade { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, revoked_card_ids: vec![first_card_id, second_card_id], affected_wallets: Vec::new(), local_wallet_generation: None, @@ -3476,6 +3487,7 @@ mod test { OplogIndex::from_u64(1), OplogEntry::CardRevokedCascade { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, revoked_card_ids: vec![card_id], affected_wallets: Vec::new(), local_wallet_generation: Some(1), @@ -3492,7 +3504,7 @@ mod test { let installed = BTreeMap::from([( OplogIndex::from_u64(2), - OplogEntry::card_installed(None, test_card(card_id).into(), Some(2)), + OplogEntry::card_installed(None, None, test_card(card_id).into(), Some(2)), )]); let status_after_install = super::update_status_with_new_entries( AgentMode::Durable, @@ -3516,20 +3528,21 @@ mod test { let entries = BTreeMap::from([ ( OplogIndex::from_u64(1), - OplogEntry::card_event_queued(QueuedCardEvent::revoke(first_card_id)), + OplogEntry::card_event_queued(None, QueuedCardEvent::revoke(first_card_id)), ), ( OplogIndex::from_u64(2), - OplogEntry::card_event_queued(QueuedCardEvent::revoke(second_card_id)), + OplogEntry::card_event_queued(None, QueuedCardEvent::revoke(second_card_id)), ), ( OplogIndex::from_u64(3), - OplogEntry::card_event_queued(QueuedCardEvent::revoke(unrelated_card_id)), + OplogEntry::card_event_queued(None, QueuedCardEvent::revoke(unrelated_card_id)), ), ( OplogIndex::from_u64(4), OplogEntry::CardRevokedCascade { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, revoked_card_ids: vec![first_card_id, second_card_id], affected_wallets: Vec::new(), local_wallet_generation: Some(1), @@ -3562,6 +3575,7 @@ mod test { OplogIndex::from_u64(1), OplogEntry::CardRevokedCascade { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, revoked_card_ids: vec![card_id], affected_wallets: Vec::new(), local_wallet_generation: None, @@ -3604,23 +3618,30 @@ mod test { let entries = BTreeMap::from([ ( OplogIndex::from_u64(1), - OplogEntry::card_event_queued(QueuedCardEvent::transfer_started( - completed_transfer_id, - transferred_card.clone(), - target_holder.clone(), - )), + OplogEntry::card_event_queued( + None, + QueuedCardEvent::transfer_started( + completed_transfer_id, + transferred_card.clone(), + target_holder.clone(), + ), + ), ), ( OplogIndex::from_u64(2), - OplogEntry::card_event_queued(QueuedCardEvent::transfer_started( - pending_transfer_id, - pending_card.clone(), - target_holder.clone(), - )), + OplogEntry::card_event_queued( + None, + QueuedCardEvent::transfer_started( + pending_transfer_id, + pending_card.clone(), + target_holder.clone(), + ), + ), ), ( OplogIndex::from_u64(3), OplogEntry::card_transfer_confirmed( + None, completed_transfer_id, transferred_card.card_id, transferred_card.card_id, @@ -3668,16 +3689,20 @@ mod test { let entries = BTreeMap::from([ ( OplogIndex::from_u64(1), - OplogEntry::card_event_queued(QueuedCardEvent::transfer_started_with_source( - transfer_id, - source_card_id, - installed_child.clone(), - target_holder.clone(), - )), + OplogEntry::card_event_queued( + None, + QueuedCardEvent::transfer_started_with_source( + transfer_id, + source_card_id, + installed_child.clone(), + target_holder.clone(), + ), + ), ), ( OplogIndex::from_u64(2), OplogEntry::card_transfer_confirmed( + None, transfer_id, source_card_id, installed_child.card_id, @@ -3713,16 +3738,20 @@ mod test { let entries = BTreeMap::from([ ( OplogIndex::from_u64(1), - OplogEntry::card_event_queued(QueuedCardEvent::transfer_started_with_source( - transfer_id, - source_card_id, - installed_child.clone(), - target_holder.clone(), - )), + OplogEntry::card_event_queued( + None, + QueuedCardEvent::transfer_started_with_source( + transfer_id, + source_card_id, + installed_child.clone(), + target_holder.clone(), + ), + ), ), ( OplogIndex::from_u64(2), OplogEntry::card_transfer_confirmed( + None, transfer_id, golem_common::model::card::CardId::new(), installed_child.card_id, @@ -3750,11 +3779,10 @@ mod test { let source_card_id = golem_common::model::card::CardId::new(); let entries = BTreeMap::from([( OplogIndex::from_u64(1), - OplogEntry::card_event_queued(QueuedCardEvent::transfer_received( - transfer_id, - source_card_id, - card.clone(), - )), + OplogEntry::card_event_queued( + None, + QueuedCardEvent::transfer_received(transfer_id, source_card_id, card.clone()), + ), )]); let status = super::update_status_with_new_entries( @@ -3803,7 +3831,7 @@ mod test { AgentStatusRecord::default(), BTreeMap::from([( OplogIndex::from_u64(1), - OplogEntry::card_event_queued(legacy_receipt), + OplogEntry::card_event_queued(None, legacy_receipt), )]), &RetryConfig::default(), ) @@ -3821,11 +3849,14 @@ mod test { status, BTreeMap::from([( OplogIndex::from_u64(2), - OplogEntry::card_event_queued(QueuedCardEvent::transfer_received( - transfer_id, - source_card_id, - stored_card.clone(), - )), + OplogEntry::card_event_queued( + None, + QueuedCardEvent::transfer_received( + transfer_id, + source_card_id, + stored_card.clone(), + ), + ), )]), &RetryConfig::default(), ) @@ -3843,11 +3874,14 @@ mod test { status, BTreeMap::from([( OplogIndex::from_u64(3), - OplogEntry::card_event_queued(QueuedCardEvent::transfer_received( - transfer_id, - golem_common::model::card::CardId::new(), - stored_card, - )), + OplogEntry::card_event_queued( + None, + QueuedCardEvent::transfer_received( + transfer_id, + golem_common::model::card::CardId::new(), + stored_card, + ), + ), )]), &RetryConfig::default(), ) @@ -3867,26 +3901,35 @@ mod test { let entries = BTreeMap::from([ ( OplogIndex::from_u64(1), - OplogEntry::card_event_queued(QueuedCardEvent::transfer_received( - skipped_transfer_id, - source_card_id, - card.clone(), - )), + OplogEntry::card_event_queued( + None, + QueuedCardEvent::transfer_received( + skipped_transfer_id, + source_card_id, + card.clone(), + ), + ), ), ( OplogIndex::from_u64(2), - OplogEntry::jump(OplogRegion { - start: OplogIndex::from_u64(1), - end: OplogIndex::from_u64(1), - }), + OplogEntry::jump( + None, + OplogRegion { + start: OplogIndex::from_u64(1), + end: OplogIndex::from_u64(1), + }, + ), ), ( OplogIndex::from_u64(3), - OplogEntry::card_event_queued(QueuedCardEvent::transfer_received( - deleted_transfer_id, - source_card_id, - card.clone(), - )), + OplogEntry::card_event_queued( + None, + QueuedCardEvent::transfer_received( + deleted_transfer_id, + source_card_id, + card.clone(), + ), + ), ), ( OplogIndex::from_u64(4), @@ -3933,15 +3976,15 @@ mod test { let entries = BTreeMap::from([ ( OplogIndex::from_u64(1), - OplogEntry::card_event_queued(QueuedCardEvent::transfer_received( - transfer_id, - source_card_id, - card.clone(), - )), + OplogEntry::card_event_queued( + None, + QueuedCardEvent::transfer_received(transfer_id, source_card_id, card.clone()), + ), ), ( OplogIndex::from_u64(2), OplogEntry::card_transferred( + None, transfer_id, Some(source_card_id), card.card_id, @@ -3979,15 +4022,15 @@ mod test { let entries = BTreeMap::from([ ( OplogIndex::from_u64(1), - OplogEntry::card_event_queued(QueuedCardEvent::transfer_received( - transfer_id, - source_card_id, - card.clone(), - )), + OplogEntry::card_event_queued( + None, + QueuedCardEvent::transfer_received(transfer_id, source_card_id, card.clone()), + ), ), ( OplogIndex::from_u64(2), OplogEntry::card_transferred( + None, transfer_id, None, card.card_id, @@ -4025,15 +4068,15 @@ mod test { let entries = BTreeMap::from([ ( OplogIndex::from_u64(1), - OplogEntry::card_event_queued(QueuedCardEvent::transfer_received( - transfer_id, - source_card_id, - card.clone(), - )), + OplogEntry::card_event_queued( + None, + QueuedCardEvent::transfer_received(transfer_id, source_card_id, card.clone()), + ), ), ( OplogIndex::from_u64(2), OplogEntry::card_transferred( + None, transfer_id, Some(golem_common::model::card::CardId::new()), card.card_id, @@ -4078,15 +4121,19 @@ mod test { let entries = BTreeMap::from([ ( OplogIndex::from_u64(1), - OplogEntry::card_event_queued(QueuedCardEvent::transfer_started( - transfer_id, - pending_card.clone(), - target_holder.clone(), - )), + OplogEntry::card_event_queued( + None, + QueuedCardEvent::transfer_started( + transfer_id, + pending_card.clone(), + target_holder.clone(), + ), + ), ), ( OplogIndex::from_u64(2), OplogEntry::card_transfer_confirmed( + None, transfer_id, conflicting_card.card_id, conflicting_card.card_id, @@ -4128,15 +4175,19 @@ mod test { let entries = BTreeMap::from([ ( OplogIndex::from_u64(1), - OplogEntry::card_event_queued(QueuedCardEvent::transfer_started( - transfer_id, - card.clone(), - target_holder.clone(), - )), + OplogEntry::card_event_queued( + None, + QueuedCardEvent::transfer_started( + transfer_id, + card.clone(), + target_holder.clone(), + ), + ), ), ( OplogIndex::from_u64(2), OplogEntry::card_install_failed( + None, OplogIndex::from_u64(1), card.card_id, golem_common::base_model::oplog::CardInstallFailure::NotFound, @@ -4182,15 +4233,19 @@ mod test { let entries = BTreeMap::from([ ( OplogIndex::from_u64(1), - OplogEntry::card_event_queued(QueuedCardEvent::transfer_started( - transfer_id, - card.clone(), - target_holder.clone(), - )), + OplogEntry::card_event_queued( + None, + QueuedCardEvent::transfer_started( + transfer_id, + card.clone(), + target_holder.clone(), + ), + ), ), ( OplogIndex::from_u64(2), OplogEntry::card_transferred( + None, transfer_id, Some(card.card_id), card.card_id, @@ -4234,15 +4289,19 @@ mod test { let entries = BTreeMap::from([ ( OplogIndex::from_u64(1), - OplogEntry::card_event_queued(QueuedCardEvent::transfer_started( - transfer_id, - card.clone(), - target_holder.clone(), - )), + OplogEntry::card_event_queued( + None, + QueuedCardEvent::transfer_started( + transfer_id, + card.clone(), + target_holder.clone(), + ), + ), ), ( OplogIndex::from_u64(2), OplogEntry::card_transfer_confirmed( + None, transfer_id, card.card_id, card.card_id, @@ -4275,15 +4334,15 @@ mod test { let entries = BTreeMap::from([ ( OplogIndex::from_u64(1), - OplogEntry::card_event_queued(QueuedCardEvent::revoke(card_id)), + OplogEntry::card_event_queued(None, QueuedCardEvent::revoke(card_id)), ), ( OplogIndex::from_u64(2), - OplogEntry::card_event_queued(QueuedCardEvent::revoke(card_id)), + OplogEntry::card_event_queued(None, QueuedCardEvent::revoke(card_id)), ), ( OplogIndex::from_u64(3), - OplogEntry::card_revoked(OplogIndex::from_u64(1), card_id, None), + OplogEntry::card_revoked(None, OplogIndex::from_u64(1), card_id, None), ), ]); @@ -4309,7 +4368,10 @@ mod test { let card = test_card(card_id); let entries = BTreeMap::from([( OplogIndex::from_u64(1), - OplogEntry::card_event_queued(QueuedCardEvent::install(card.clone())), + OplogEntry::card_event_queued( + Some(OplogIndex::from_u64(42)), + QueuedCardEvent::install(card.clone()), + ), )]); let status = super::update_status_with_new_entries( @@ -4321,6 +4383,10 @@ mod test { .unwrap(); assert_eq!(status.pending_card_events.len(), 1); + assert_eq!( + status.pending_card_events[0].entity_parent_start_index, + Some(OplogIndex::from_u64(42)) + ); assert_eq!( status.pending_card_events[0].event, QueuedCardEvent::install(card) @@ -4334,11 +4400,11 @@ mod test { let entries = BTreeMap::from([ ( OplogIndex::from_u64(1), - OplogEntry::card_event_queued(QueuedCardEvent::install(card.clone())), + OplogEntry::card_event_queued(None, QueuedCardEvent::install(card.clone())), ), ( OplogIndex::from_u64(2), - OplogEntry::card_installed(Some(OplogIndex::from_u64(1)), card.into(), None), + OplogEntry::card_installed(None, Some(OplogIndex::from_u64(1)), card.into(), None), ), ]); @@ -4360,11 +4426,12 @@ mod test { let entries = BTreeMap::from([ ( OplogIndex::from_u64(1), - OplogEntry::card_event_queued(QueuedCardEvent::install(card)), + OplogEntry::card_event_queued(None, QueuedCardEvent::install(card)), ), ( OplogIndex::from_u64(2), OplogEntry::card_install_failed( + None, OplogIndex::from_u64(1), card_id, CardInstallFailure::CardRevoked, @@ -4389,7 +4456,7 @@ mod test { let entries = BTreeMap::from([ ( OplogIndex::from_u64(2), - OplogEntry::card_event_queued(QueuedCardEvent::revoke(card_id)), + OplogEntry::card_event_queued(None, QueuedCardEvent::revoke(card_id)), ), ( OplogIndex::from_u64(3), @@ -4418,11 +4485,11 @@ mod test { let entries = BTreeMap::from([ ( OplogIndex::from_u64(1), - OplogEntry::card_event_queued(QueuedCardEvent::install(card.clone())), + OplogEntry::card_event_queued(None, QueuedCardEvent::install(card.clone())), ), ( OplogIndex::from_u64(2), - OplogEntry::card_installed(Some(OplogIndex::from_u64(1)), card.into(), None), + OplogEntry::card_installed(None, Some(OplogIndex::from_u64(1)), card.into(), None), ), ( OplogIndex::from_u64(3), @@ -4452,6 +4519,7 @@ mod test { OplogIndex::from_u64(2), OplogEntry::CardRevoked { timestamp: Timestamp::now_utc(), + entity_parent_start_index: None, queued_event_index: OplogIndex::from_u64(1), card_id, wallet_generation: None, diff --git a/golem-worker-executor/tests/api.rs b/golem-worker-executor/tests/api.rs index fbf03d200f..9d22f8df2e 100644 --- a/golem-worker-executor/tests/api.rs +++ b/golem-worker-executor/tests/api.rs @@ -693,6 +693,7 @@ async fn card_transfer_delivery_is_durable_idempotent_and_rejects_payload_confli .commit_oplog_entry_bypassing_worker_status( &target_agent_id, golem_common::model::oplog::OplogEntry::card_event_queued( + None, QueuedCardEvent::transfer_received( detached_status_transfer_id, source_card_id, @@ -1136,12 +1137,15 @@ async fn pending_source_card_transfers_resume_only_after_replay_reaches_live_mod executor .commit_oplog_entry_bypassing_worker_status( &source_agent_id, - OplogEntry::card_event_queued(QueuedCardEvent::transfer_started_with_source( - pending_transfer_id, - card.card_id(), - card.clone(), - target_holder.clone(), - )), + OplogEntry::card_event_queued( + None, + QueuedCardEvent::transfer_started_with_source( + pending_transfer_id, + card.card_id(), + card.clone(), + target_holder.clone(), + ), + ), ) .await?; assert_eq!( @@ -1152,29 +1156,36 @@ async fn pending_source_card_transfers_resume_only_after_replay_reaches_live_mod executor .commit_oplog_entry_bypassing_worker_status( &source_agent_id, - OplogEntry::card_event_queued(QueuedCardEvent::transfer_started_with_source( - completed_transfer_id, - card.card_id(), - card.clone(), - target_holder.clone(), - )), + OplogEntry::card_event_queued( + None, + QueuedCardEvent::transfer_started_with_source( + completed_transfer_id, + card.card_id(), + card.clone(), + target_holder.clone(), + ), + ), ) .await?; executor .commit_oplog_entry_bypassing_worker_status( &source_agent_id, - OplogEntry::card_event_queued(QueuedCardEvent::transfer_started_with_source( - started_transfer_id, - card.card_id(), - card.clone(), - target_holder.clone(), - )), + OplogEntry::card_event_queued( + None, + QueuedCardEvent::transfer_started_with_source( + started_transfer_id, + card.card_id(), + card.clone(), + target_holder.clone(), + ), + ), ) .await?; executor .commit_oplog_entry_bypassing_worker_status( &source_agent_id, OplogEntry::card_transfer_started( + None, started_transfer_id, card.card_id(), Some(source_holder.clone()), @@ -1187,6 +1198,7 @@ async fn pending_source_card_transfers_resume_only_after_replay_reaches_live_mod .commit_oplog_entry_bypassing_worker_status( &source_agent_id, OplogEntry::card_transfer_started( + None, completed_transfer_id, card.card_id(), Some(source_holder), @@ -1199,6 +1211,7 @@ async fn pending_source_card_transfers_resume_only_after_replay_reaches_live_mod .commit_oplog_entry_bypassing_worker_status( &source_agent_id, OplogEntry::card_transfer_confirmed( + None, completed_transfer_id, card.card_id(), card.card_id(), @@ -1367,12 +1380,15 @@ async fn pending_self_card_transfer_recovery_does_not_deadlock( executor .commit_oplog_entry_bypassing_worker_status( &source_agent_id, - OplogEntry::card_event_queued(QueuedCardEvent::transfer_started_with_source( - transfer_id, - card.card_id(), - card.clone(), - self_holder, - )), + OplogEntry::card_event_queued( + None, + QueuedCardEvent::transfer_started_with_source( + transfer_id, + card.card_id(), + card.clone(), + self_holder, + ), + ), ) .await?; @@ -1498,18 +1514,22 @@ async fn lost_card_transfer_response_converges_after_source_and_target_restart( executor .commit_oplog_entry_bypassing_worker_status( &source_agent_id, - OplogEntry::card_event_queued(QueuedCardEvent::transfer_started_with_source( - transfer_id, - card.card_id(), - card.clone(), - target_holder.clone(), - )), + OplogEntry::card_event_queued( + None, + QueuedCardEvent::transfer_started_with_source( + transfer_id, + card.card_id(), + card.clone(), + target_holder.clone(), + ), + ), ) .await?; executor .commit_oplog_entry_bypassing_worker_status( &source_agent_id, OplogEntry::card_transfer_started( + None, transfer_id, card.card_id(), Some(source_holder), diff --git a/golem-worker-service/src/service/worker/client.rs b/golem-worker-service/src/service/worker/client.rs index 28c3c583c5..4805c85236 100644 --- a/golem-worker-service/src/service/worker/client.rs +++ b/golem-worker-service/src/service/worker/client.rs @@ -49,7 +49,7 @@ use golem_common::model::component::{ CanonicalFilePath, ComponentId, ComponentRevision, PluginPriority, }; use golem_common::model::environment::EnvironmentId; -use golem_common::model::oplog::{OplogCursor, PublicOplogEntry}; +use golem_common::model::oplog::OplogCursor; use golem_common::model::oplog::{OplogIndex, PublicOplogEntryWithIndex}; use golem_common::model::worker::AgentConfigEntryDto; use golem_common::model::worker::AgentUpdateMode; @@ -1132,7 +1132,7 @@ impl WorkerClient for WorkerExecutorWorkerClient { }, )), } => { - let entries: Vec = entries + let entries: Vec = entries .into_iter() .map(|e| e.try_into()) .collect::, _>>() @@ -1142,16 +1142,7 @@ impl WorkerClient for WorkerExecutorWorkerClient { )) })?; Ok(GetOplogResponse { - entries: entries - .into_iter() - .enumerate() - .map(|(idx, entry)| PublicOplogEntryWithIndex { - oplog_index: OplogIndex::from_u64( - (first_index_in_chunk) + idx as u64, - ), - entry, - }) - .collect(), + entries, next: next.map(|c| c.into()), first_index_in_chunk, last_index, diff --git a/openapi/golem-service.yaml b/openapi/golem-service.yaml index 1f77093a5e..50710df4b6 100644 --- a/openapi/golem-service.yaml +++ b/openapi/golem-service.yaml @@ -14279,6 +14279,22 @@ components: properties: error: type: string + PublicAgentEntity: + title: PublicAgentEntity + type: object + properties: + kind: + $ref: '#/components/schemas/PublicAgentEntityKind' + name: + type: string + required: + - kind + - name + PublicAgentEntityKind: + type: string + enum: + - tool + - toolMiddleware PublicAgentInvocation: discriminator: propertyName: type @@ -14532,6 +14548,66 @@ components: required: - type - $ref: '#/components/schemas/WriteRemoteTransactionParameters' + PublicEntityCallMode: + type: string + enum: + - synchronous + - asynchronous + - fireAndForget + PublicEntityInvocation: + title: PublicEntityInvocation + description: One entity invocation in an owner-oplog execution chain. + type: object + properties: + entity: + $ref: '#/components/schemas/PublicAgentEntity' + startIndex: + type: integer + format: uint64 + callMode: + $ref: '#/components/schemas/PublicEntityCallMode' + operation: + $ref: '#/components/schemas/PublicEntityInvocationOperation' + required: + - entity + - startIndex + - callMode + PublicEntityInvocationContext: + title: PublicEntityInvocationContext + description: |- + Attribution for an entry executed by an entity. Ancestors are ordered from the root entity + invocation to the immediate parent of `invocation`. + type: object + properties: + invocation: + $ref: '#/components/schemas/PublicEntityInvocation' + ancestors: + type: array + items: + $ref: '#/components/schemas/PublicEntityInvocation' + required: + - invocation + - ancestors + PublicEntityInvocationOperation: + discriminator: + propertyName: type + mapping: + Tool: '#/components/schemas/PublicEntityInvocationOperation_PublicToolInvocationOperation' + type: object + oneOf: + - $ref: '#/components/schemas/PublicEntityInvocationOperation_PublicToolInvocationOperation' + PublicEntityInvocationOperation_PublicToolInvocationOperation: + allOf: + - type: object + properties: + type: + example: Tool + type: string + enum: + - Tool + required: + - type + - $ref: '#/components/schemas/PublicToolInvocationOperation' PublicExternalSpanData: title: PublicExternalSpanData type: object @@ -14723,6 +14799,40 @@ components: - $ref: '#/components/schemas/PublicOplogEntry_r#StreamSessionParams' - $ref: '#/components/schemas/PublicOplogEntry_r#CompletionDiscardedParams' - $ref: '#/components/schemas/PublicOplogEntry_r#CompletionDeliveredParams' + PublicOplogEntryAttribution: + discriminator: + propertyName: type + mapping: + Agent: '#/components/schemas/PublicOplogEntryAttribution_Empty' + Entity: '#/components/schemas/PublicOplogEntryAttribution_PublicEntityInvocationContext' + type: object + oneOf: + - $ref: '#/components/schemas/PublicOplogEntryAttribution_Empty' + - $ref: '#/components/schemas/PublicOplogEntryAttribution_PublicEntityInvocationContext' + PublicOplogEntryAttribution_Empty: + allOf: + - type: object + properties: + type: + example: Agent + type: string + enum: + - Agent + required: + - type + - $ref: '#/components/schemas/Empty' + PublicOplogEntryAttribution_PublicEntityInvocationContext: + allOf: + - type: object + properties: + type: + example: Entity + type: string + enum: + - Entity + required: + - type + - $ref: '#/components/schemas/PublicEntityInvocationContext' PublicOplogEntryWithIndex: title: PublicOplogEntryWithIndex type: object @@ -14730,10 +14840,13 @@ components: oplogIndex: type: integer format: uint64 + attribution: + $ref: '#/components/schemas/PublicOplogEntryAttribution' entry: $ref: '#/components/schemas/PublicOplogEntry' required: - oplogIndex + - attribution - entry PublicOplogEntry_r#ActivatePluginParams: allOf: @@ -15780,6 +15893,30 @@ components: required: - type - $ref: '#/components/schemas/PublicLocalSpanData' + PublicToolInvocationOperation: + title: PublicToolInvocationOperation + type: object + properties: + commandPath: + type: array + items: + type: string + hasStdin: + description: Whether a live stdin attachment was requested. + type: boolean + hasStdout: + description: |- + Whether a live stdout attachment was requested. Stdout bytes are not recorded in the + oplog. + type: boolean + declaresStdout: + description: Whether the tool declares stdout support. Stdout bytes are not recorded in the oplog. + type: boolean + required: + - commandPath + - hasStdin + - hasStdout + - declaresStdout PublicTypedAgentConfigEntry: title: PublicTypedAgentConfigEntry description: |- diff --git a/openapi/golem-worker-service.yaml b/openapi/golem-worker-service.yaml index de848ad862..6263c05b3f 100644 --- a/openapi/golem-worker-service.yaml +++ b/openapi/golem-worker-service.yaml @@ -4362,6 +4362,22 @@ components: properties: error: type: string + PublicAgentEntity: + type: object + title: PublicAgentEntity + required: + - kind + - name + properties: + kind: + $ref: '#/components/schemas/PublicAgentEntityKind' + name: + type: string + PublicAgentEntityKind: + type: string + enum: + - tool + - toolMiddleware PublicAgentInvocation: type: object oneOf: @@ -4615,6 +4631,66 @@ components: - WriteRemoteTransaction example: WriteRemoteTransaction - $ref: '#/components/schemas/WriteRemoteTransactionParameters' + PublicEntityCallMode: + type: string + enum: + - synchronous + - asynchronous + - fireAndForget + PublicEntityInvocation: + type: object + title: PublicEntityInvocation + description: One entity invocation in an owner-oplog execution chain. + required: + - entity + - startIndex + - callMode + properties: + entity: + $ref: '#/components/schemas/PublicAgentEntity' + startIndex: + type: integer + format: uint64 + callMode: + $ref: '#/components/schemas/PublicEntityCallMode' + operation: + $ref: '#/components/schemas/PublicEntityInvocationOperation' + PublicEntityInvocationContext: + type: object + title: PublicEntityInvocationContext + description: |- + Attribution for an entry executed by an entity. Ancestors are ordered from the root entity + invocation to the immediate parent of `invocation`. + required: + - invocation + - ancestors + properties: + invocation: + $ref: '#/components/schemas/PublicEntityInvocation' + ancestors: + type: array + items: + $ref: '#/components/schemas/PublicEntityInvocation' + PublicEntityInvocationOperation: + type: object + oneOf: + - $ref: '#/components/schemas/PublicEntityInvocationOperation_PublicToolInvocationOperation' + discriminator: + propertyName: type + mapping: + Tool: '#/components/schemas/PublicEntityInvocationOperation_PublicToolInvocationOperation' + PublicEntityInvocationOperation_PublicToolInvocationOperation: + allOf: + - type: object + required: + - type + properties: + type: + type: string + enum: + - Tool + example: Tool + - $ref: '#/components/schemas/PublicToolInvocationOperation' PublicExternalSpanData: type: object title: PublicExternalSpanData @@ -4806,16 +4882,53 @@ components: StreamSession: '#/components/schemas/PublicOplogEntry_r#StreamSessionParams' CompletionDiscarded: '#/components/schemas/PublicOplogEntry_r#CompletionDiscardedParams' CompletionDelivered: '#/components/schemas/PublicOplogEntry_r#CompletionDeliveredParams' + PublicOplogEntryAttribution: + type: object + oneOf: + - $ref: '#/components/schemas/PublicOplogEntryAttribution_Empty' + - $ref: '#/components/schemas/PublicOplogEntryAttribution_PublicEntityInvocationContext' + discriminator: + propertyName: type + mapping: + Agent: '#/components/schemas/PublicOplogEntryAttribution_Empty' + Entity: '#/components/schemas/PublicOplogEntryAttribution_PublicEntityInvocationContext' + PublicOplogEntryAttribution_Empty: + allOf: + - type: object + required: + - type + properties: + type: + type: string + enum: + - Agent + example: Agent + - $ref: '#/components/schemas/Empty' + PublicOplogEntryAttribution_PublicEntityInvocationContext: + allOf: + - type: object + required: + - type + properties: + type: + type: string + enum: + - Entity + example: Entity + - $ref: '#/components/schemas/PublicEntityInvocationContext' PublicOplogEntryWithIndex: type: object title: PublicOplogEntryWithIndex required: - oplogIndex + - attribution - entry properties: oplogIndex: type: integer format: uint64 + attribution: + $ref: '#/components/schemas/PublicOplogEntryAttribution' entry: $ref: '#/components/schemas/PublicOplogEntry' PublicOplogEntry_r#ActivatePluginParams: @@ -5863,6 +5976,30 @@ components: - LocalSpan example: LocalSpan - $ref: '#/components/schemas/PublicLocalSpanData' + PublicToolInvocationOperation: + type: object + title: PublicToolInvocationOperation + required: + - commandPath + - hasStdin + - hasStdout + - declaresStdout + properties: + commandPath: + type: array + items: + type: string + hasStdin: + type: boolean + description: Whether a live stdin attachment was requested. + hasStdout: + type: boolean + description: |- + Whether a live stdout attachment was requested. Stdout bytes are not recorded in the + oplog. + declaresStdout: + type: boolean + description: Whether the tool declares stdout support. Stdout bytes are not recorded in the oplog. PublicTypedAgentConfigEntry: type: object title: PublicTypedAgentConfigEntry