From a93cd9edbc4eb3a9bf8cd12113ace678a4b0a2e5 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 18 Sep 2026 11:22:13 -0700 Subject: [PATCH 01/11] feat(journaling): add journaled state lifecycle hooks --- .../grains/journaling/runtime-behavior.md | 27 + .../IJournaledStateManager.cs | 13 + src/Orleans.Journaling/IStateMachine.cs | 69 +- .../JournaledStateManager.cs | 74 ++ .../Orleans.Journaling/Orleans.Journaling.cs | 8 + .../JournaledGrainCompositionTests.cs | 7 +- .../KeyedJournalingRegistrationTests.cs | 86 +++ .../StateManagerLifecycleTests.cs | 632 ++++++++++++++++++ .../StateManagerTests.cs | 2 +- 9 files changed, 915 insertions(+), 3 deletions(-) create mode 100644 test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index dddcba9ea91..8fa22a2a66f 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -102,6 +102,33 @@ storage outcome, so the caller reconciles that outcome before retrying the comma An initialization failure preserves stored data for diagnosis. Restore the required format/codec registration or repair the backing data before creating a fresh manager or retrying activation. +## Custom state lifecycle + +Custom implementations share the manager's single logical execution thread. +Resolve write codecs through to use the +owning manager's configured format, including before recovery of an empty journal. Delegating managers forward +codec resolution to that owner. + + and +perform pure validation in the requesting caller's context. An admission rejection leaves the manager healthy. +Deletion validates all states again in serialized execution, then calls + on every state before awaiting storage deletion. +Successful deletion resets states before completing callers. + +Before each append or snapshot capture, the manager checks +and awaits for each unprepared state. Preparation +establishes that state's readiness and retains valid state-owned prerequisites across rechecks. Every await +is followed by another all-state readiness pass. The final successful pass flows directly into synchronous +capture in the same work-loop continuation. This also applies to writes which flush only committed entries +or produce zero bytes. Preparation uses the manager's shutdown token; caller cancellation only ends that caller's wait. + +After storage acknowledges captured bytes, +performs durable-completion bookkeeping. A zero-byte write completes without this callback. +An admitted preparation, validation, capture, or storage failure fences the manager, records the original +exception, and calls on every registered state before +faulting current and queued waiters. Notification failures are logged while the original failure remains +the operation's outcome. Idle shutdown completes normally; cancellation during admitted work is terminal. + ## Compaction Each provider reports when its journal crosses a configured storage threshold. The next `WriteStateAsync`: diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index d619a8f37f7..167eeebf35f 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -41,6 +41,19 @@ public interface IJournaledStateManager : IAsyncDisposable /// if the state machine is registered; otherwise, . bool TryGetStateMachine(string name, [NotNullWhen(true)] out IStateMachine? stateMachine); + /// + /// Resolves a command codec for this manager's configured write journal format. + /// + /// The command codec service type. + /// The codec registered for this manager's write format. + /// + /// Codecs are available before recovery, including for an empty journal. Delegating managers forward + /// this call to their owning manager. The default implementation throws . + /// + /// The manager does not support command codec resolution. + TCodec GetRequiredCommandCodec() where TCodec : notnull + => throw new NotSupportedException("This journaled state manager does not support write command codec resolution."); + /// /// Persists pending changes from the registered state machines to the journal. /// diff --git a/src/Orleans.Journaling/IStateMachine.cs b/src/Orleans.Journaling/IStateMachine.cs index 39f824a6db7..2bb0b7eda22 100644 --- a/src/Orleans.Journaling/IStateMachine.cs +++ b/src/Orleans.Journaling/IStateMachine.cs @@ -14,7 +14,8 @@ namespace Orleans.Journaling; /// both apply the mutation locally and emit the corresponding command to the journal). /// /// -/// When the application requests a write, the journaled state manager calls +/// When the application requests a write, the journaled state manager validates the request, +/// prepares all states, and calls /// (and occasionally ) to materialize /// the pending changes, then flushes the journal to durable storage. /// @@ -67,6 +68,72 @@ public interface IStateMachine /// void OnRecoveryCompleted() { } + /// + /// Gets whether this state has the prerequisites required for synchronous write capture. + /// The default is . + /// + /// + /// This synchronous check is pure. The manager evaluates it inside the admitted operation's + /// failure boundary, including writes which only flush committed entries or produce zero bytes. + /// An unrecoverable state-local error can be reported by throwing, which fences the manager. + /// + bool IsWritePrepared => true; + + /// + /// Acquires state-owned prerequisites for synchronous write capture when is false. + /// The default implementation completes synchronously. + /// + /// The manager operation and shutdown token. + /// A task which completes when this state is prepared. + /// + /// Completion must establish this state's readiness. Retain valid prepared resources across readiness + /// rechecks. After preparation awaits, the manager rechecks every state and captures synchronously + /// in the same continuation as the final ready pass. Caller wait cancellation leaves preparation running. + /// Preparation failures fence the manager. + /// + ValueTask PrepareWriteAsync(CancellationToken cancellationToken) => default; + + /// + /// Validates a write request in the public caller's context before it is queued. + /// The default implementation accepts the request. + /// + /// + /// Validation is pure. Throwing rejects this request and leaves the manager healthy. + /// + void ValidateWrite() { } + + /// + /// Validates deletion at public request admission and again during serialized execution. + /// The default implementation accepts deletion. + /// + /// + /// Validation is pure. An admission failure rejects the request and leaves the manager healthy. + /// An execution-time failure fences the manager. All states pass execution-time validation + /// before the manager calls on any state. + /// + void ValidateDelete() { } + + /// + /// Notifies the state that deletion is starting, after all execution-time validation succeeds + /// and before the storage operation begins. The default implementation performs no action. + /// + /// + /// A successful storage deletion is followed by before deletion waiters complete. + /// + void OnDeleteStarted() { } + + /// + /// Notifies the state of the manager's first terminal failure, before current and queued operation waiters fault. + /// The default implementation performs no action. + /// + /// The original failure recorded by the manager. + /// + /// The manager is already fenced when this callback runs. Every registered state is notified even if + /// another notification throws; notification errors are logged and the original failure is preserved. + /// Idle shutdown completes normally. Cancellation during admitted preparation or storage work is terminal. + /// + void OnFaulted(Exception exception) { } + /// /// Writes pending state changes to the journal. /// diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 720e6789d5e..6359f32bb1a 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -100,6 +100,9 @@ internal static IJournalStorage CreateStorage(IJournalStorageProvider storagePro internal IServiceProvider ServiceProvider => _grainContext is { } context ? context.ActivationServices : _shared.ServiceProvider; + public TCodec GetRequiredCommandCodec() where TCodec : notnull + => JournalFormatServices.GetRequiredCommandCodec(ServiceProvider, _shared.JournalFormatKey); + public bool TryGetStateMachine(string name, [NotNullWhen(true)] out IStateMachine? stateMachine) { ArgumentException.ThrowIfNullOrEmpty(name); @@ -253,6 +256,31 @@ private async Task WorkLoop() case AppendJournalWorkItem: case WriteSnapshotWorkItem: { + // Keep the final readiness pass and synchronous capture in this continuation. + bool prepared; + do + { + prepared = true; + foreach (var (name, state) in _states) + { + if (state.IsWritePrepared) + { + continue; + } + + await state.PrepareWriteAsync(_shutdownCancellation.Token).ConfigureAwait(true); + if (!state.IsWritePrepared) + { + throw new InvalidOperationException( + $"Journaled state '{name}' completed write preparation without becoming prepared."); + } + + prepared = false; + break; + } + } + while (!prepared); + // TODO: decide whether it's best to snapshot or append. Eg, by summing the size of the most recent snapshots and the current journal length. // If the current journal length is greater than the snapshot size, then take a snapshot instead of appending more journal entries. var isSnapshot = workItem is WriteSnapshotWorkItem @@ -447,6 +475,16 @@ private async Task WorkLoop() case DeleteStateWorkItem: { + foreach (var state in _states.Values) + { + state.ValidateDelete(); + } + + foreach (var state in _states.Values) + { + state.OnDeleteStarted(); + } + // Clear storage. await DeleteStorageAsync(_shutdownCancellation.Token).ConfigureAwait(true); @@ -548,6 +586,10 @@ private async Task WorkLoop() } } } + catch (OperationCanceledException) when (_shutdownCancellation.IsCancellationRequested) + { + return; + } catch (Exception exception) { Fence(exception); @@ -560,8 +602,25 @@ private void Fence(Exception exception) { lock (_lock) { + if (_state is ManagerState.Fenced) + { + return; + } + _state = ManagerState.Fenced; _failure = exception; + + foreach (var (name, state) in _states) + { + try + { + state.OnFaulted(exception); + } + catch (Exception notificationException) + { + LogErrorNotifyingFaultedState(_shared.Logger, notificationException, name); + } + } } try @@ -660,6 +719,11 @@ public async ValueTask DeleteStateAsync(CancellationToken cancellationToken = de lock (_lock) { ThrowIfStateOperationsUnavailable(); + foreach (var state in _states.Values) + { + state.ValidateDelete(); + } + task = EnqueueOrGetPendingWorkItem(out didEnqueue); } @@ -849,6 +913,11 @@ public async ValueTask WriteStateAsync(CancellationToken cancellationToken = def lock (_lock) { ThrowIfStateOperationsUnavailable(); + foreach (var state in _states.Values) + { + state.ValidateWrite(); + } + var isSnapshot = _migrationSnapshotRequired || _storage.IsCompactionRequested; operation = isSnapshot ? JournalingInstruments.OperationSnapshot : JournalingInstruments.OperationAppend; pendingWrite = isSnapshot @@ -1307,6 +1376,11 @@ void IStateMachine.WritePendingEntries(JournalStreamWriter writer) { } Message = "Error processing work items.")] private static partial void LogErrorProcessingWorkItems(ILogger logger, Exception exception); + [LoggerMessage( + Level = LogLevel.Error, + Message = "Error notifying journaled state \"{Name}\" of a terminal failure.")] + private static partial void LogErrorNotifyingFaultedState(ILogger logger, Exception exception, string name); + [LoggerMessage( Level = LogLevel.Information, Message = "State \"{Name}\" was not found. I have substituted a placeholder for graceful time-based retirement.")] diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index f29f6db150e..f1a76cf9719 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -224,6 +224,7 @@ public partial interface IJournaledStateManager : System.IAsyncDisposable long PendingWriteByteCount { get; } System.Threading.Tasks.ValueTask DeleteStateAsync(System.Threading.CancellationToken cancellationToken = default); + TCodec GetRequiredCommandCodec(); System.Threading.Tasks.ValueTask InitializeAsync(System.Threading.CancellationToken cancellationToken = default); void RegisterStateMachine(string name, IStateMachine stateMachine); System.Threading.Tasks.ValueTask System.IAsyncDisposable.DisposeAsync(); @@ -305,10 +306,17 @@ public partial interface IPreservedJournalEntry public partial interface IStateMachine { + bool IsWritePrepared { get; } + + void OnDeleteStarted(); + void OnFaulted(System.Exception exception); void OnRecoveryCompleted(); void OnWriteCompleted(); + System.Threading.Tasks.ValueTask PrepareWriteAsync(System.Threading.CancellationToken cancellationToken); void ReplayEntry(JournalEntry entry, JournalReplayContext context); void Reset(JournalStreamWriter writer); + void ValidateDelete(); + void ValidateWrite(); void WritePendingEntries(JournalStreamWriter writer); void WriteSnapshot(JournalStreamWriter writer); } diff --git a/test/Orleans.Journaling.Tests/JournaledGrainCompositionTests.cs b/test/Orleans.Journaling.Tests/JournaledGrainCompositionTests.cs index ad162decba4..1036fb5838f 100644 --- a/test/Orleans.Journaling.Tests/JournaledGrainCompositionTests.cs +++ b/test/Orleans.Journaling.Tests/JournaledGrainCompositionTests.cs @@ -336,7 +336,12 @@ private static CompositionSiloBuilder CreateBuilder() builder.Services.AddSingleton(TimeProvider.System); builder.Services.AddKeyedSingleton(KeyedService.AnyKey, static (services, _) => services.GetRequiredService()); builder.AddVolatileJournalStorage().UseJsonJournalFormat(JournalingTestsJsonContext.Default); - builder.Services.AddStateMachine(static (_, _) => Substitute.For()); + builder.Services.AddStateMachine(static (_, _) => + { + var state = Substitute.For(); + state.IsWritePrepared.Returns(true); + return state; + }); return builder; } diff --git a/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs b/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs index 3f72fdc0e4f..3fe7eff258f 100644 --- a/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs +++ b/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs @@ -382,6 +382,63 @@ public void DurableService_ResolvesCommandCodecFromJournalFormatKey() scope.ServiceProvider.GetRequiredService().ObservableLifecycle).Subscriptions); } + [Fact] + public async Task ManagerCommandCodec_UsesOwningNamedFormatOnEmptyJournal() + { + var builder = CreateNamedProviderBuilder(); + builder.AddVolatileJournalStorage(); + var customStorage = new VolatileJournalStorage(CustomFormatKey); + builder.Services.AddKeyedSingleton(CustomFormatKey, static (services, _) => + new NamedBinaryJournalFormat(services.GetRequiredService())); + builder.Services.AddKeyedSingleton(typeof(IDurableDictionaryCommandCodec<,>), CustomFormatKey, + typeof(OrleansBinaryDurableDictionaryCommandCodec<,>)); + builder.Services.AddKeyedSingleton>(CustomFormatKey, static (services, _) => + new OrleansBinaryDurableValueCommandCodec( + services.GetRequiredService().GetCodec(), + services.GetRequiredService())); + builder.Services.AddKeyedSingleton("custom", (services, _) => + new JournaledStateManagerFactory( + new JournaledStateManagerShared( + services.GetRequiredService>(), + Options.Create(new JournaledStateManagerOptions { JournalFormatKey = CustomFormatKey }), + TimeProvider.System, + services), + new TestJournalStorageProvider(customStorage))); + await using var services = builder.Services.BuildServiceProvider(); + var factory = services.GetRequiredKeyedService("custom"); + await using var defaultManager = services.GetRequiredService().CreateStandalone(new JournalId("default")); + await using var customManager = factory.CreateStandalone(new JournalId("custom")); + IJournaledStateManager delegating = new DelegatingCodecManager(customManager); + var codec = delegating.GetRequiredCommandCodec>(); + Assert.Same(services.GetRequiredKeyedService>(CustomFormatKey), codec); + Assert.Same(services.GetRequiredKeyedService>(JsonLinesJournalFormat.JournalFormatKey), + defaultManager.GetRequiredCommandCodec>()); + Assert.NotSame(defaultManager.GetRequiredCommandCodec>(), codec); + var state = new DurableValue("value", delegating, codec); + await delegating.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(0, state.Value); + Assert.Empty(customStorage.Segments); + state.Value = 42; + await delegating.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Single(customStorage.Segments); + await using var recovered = factory.CreateStandalone(new JournalId("custom")); + var recoveredState = new DurableValue("value", recovered, recovered.GetRequiredCommandCodec>()); + await recovered.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(42, recoveredState.Value); + + var missing = Assert.Throws(() => customManager.GetRequiredCommandCodec>()); + Assert.Contains(CustomFormatKey, missing.Message); + Assert.Contains(nameof(IDurableQueueCommandCodec), missing.Message); + } + + [Fact] + public void ManagerCommandCodec_DefaultReportsUnsupportedResolution() + { + IJournaledStateManager manager = new UnsupportedCodecManager(); + var exception = Assert.Throws(() => manager.GetRequiredCommandCodec>()); + Assert.Contains("write command codec resolution", exception.Message); + } + [Fact] public async Task StateManagerFactory_CreatesManagerForJournalId() { @@ -421,6 +478,35 @@ private sealed class TestJournalStorageProvider(IJournalStorage storage) : IJour public IJournalStorage CreateStorage(JournalId journalId) => storage; } + private sealed class NamedBinaryJournalFormat(IJournalFormat inner) : IJournalFormat + { + public string FormatKey => CustomFormatKey; + public string? MimeType => inner.MimeType; + public JournalBufferWriter CreateWriter() => inner.CreateWriter(); + public void Replay(JournalBufferReader input, JournalReplayContext context) => inner.Replay(input, context); + } + + private sealed class DelegatingCodecManager(IJournaledStateManager inner) : IJournaledStateManager + { + public TCodec GetRequiredCommandCodec() where TCodec : notnull => inner.GetRequiredCommandCodec(); + public ValueTask InitializeAsync(CancellationToken cancellationToken) => inner.InitializeAsync(cancellationToken); + public void RegisterStateMachine(string name, IStateMachine state) => inner.RegisterStateMachine(name, state); + public bool TryGetStateMachine(string name, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out IStateMachine? state) + => inner.TryGetStateMachine(name, out state); + public ValueTask WriteStateAsync(CancellationToken cancellationToken) => inner.WriteStateAsync(cancellationToken); + public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => inner.DeleteStateAsync(cancellationToken); + } + + private sealed class UnsupportedCodecManager : IJournaledStateManager + { + public ValueTask InitializeAsync(CancellationToken cancellationToken) => throw new NotSupportedException(); + public void RegisterStateMachine(string name, IStateMachine state) => throw new NotSupportedException(); + public bool TryGetStateMachine(string name, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out IStateMachine? state) + => throw new NotSupportedException(); + public ValueTask WriteStateAsync(CancellationToken cancellationToken) => throw new NotSupportedException(); + public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => throw new NotSupportedException(); + } + private sealed class LifecycleJournalStorageProvider : IJournalStorageProvider, IJournalStorageCatalog, ILifecycleParticipant { private readonly VolatileJournalStorageProvider _storage = new(); diff --git a/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs b/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs new file mode 100644 index 00000000000..820b8de8cb4 --- /dev/null +++ b/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs @@ -0,0 +1,632 @@ +using System.Threading.Channels; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NSubstitute; +using Xunit; + +namespace Orleans.Journaling.Tests; + +public partial class StateManagerTests +{ + [Fact] + public async Task StateHooks_DefaultsAcceptExistingStates() + { + IStateMachine state = new AlwaysWritingState(); + Assert.True(state.IsWritePrepared); + Assert.True(state.PrepareWriteAsync(CancellationToken.None).IsCompletedSuccessfully); + state.ValidateWrite(); + state.ValidateDelete(); + state.OnDeleteStarted(); + state.OnFaulted(new IOException("Default notification.")); + + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + manager.RegisterStateMachine("state", state); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + await manager.DeleteStateAsync(TestContext.Current.CancellationToken); + Assert.Single(storage.Appends); + Assert.Equal(1, storage.DeleteCount); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task AdmissionVeto_UsesCallerContextAndLeavesManagerHealthy(bool delete) + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var state = new HookState(); + manager.RegisterStateMachine("state", state); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var callerContext = new AsyncLocal { Value = "request" }; + var insideAdmission = true; + var expected = new InvalidOperationException("Request rejected."); + var calls = 0; + void Validate() + { + Assert.True(insideAdmission); + Assert.Equal("request", callerContext.Value); + calls++; + throw expected; + } + + if (delete) state.ValidateDeleteAction = Validate; + else state.ValidateWriteAction = Validate; + var request = delete + ? manager.DeleteStateAsync(TestContext.Current.CancellationToken) + : manager.WriteStateAsync(TestContext.Current.CancellationToken); + insideAdmission = false; + Assert.Equal(1, calls); + Assert.Same(expected, await Record.ExceptionAsync(() => request.AsTask())); + Assert.Null(state.Failure); + Assert.Equal(0, state.DeleteStartedCount); + Assert.Equal(1, state.ResetCount); + Assert.Empty(storage.Appends); + Assert.Equal(0, storage.DeleteCount); + + state.ValidateWriteAction = null; + state.ValidateDeleteAction = null; + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + await manager.DeleteStateAsync(TestContext.Current.CancellationToken); + Assert.Single(storage.Appends); + Assert.Equal(1, storage.DeleteCount); + Assert.Null(state.Failure); + } + + [Fact] + public async Task Delete_RechecksAllValidationBeforeAnyStart() + { + var storage = new CapturingStorage { BlockNextAppend = true }; + await using var manager = CreateTestSystem(storage).Manager; + var first = new HookState(); + var second = new HookState(); + manager.RegisterStateMachine("first", first); + manager.RegisterStateMachine("second", second); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await WaitFor(storage.BlockedAppendStarted.Task); + + var validations = new List(); + first.ValidateDeleteAction = () => validations.Add("first"); + var reject = false; + var expected = new InvalidOperationException("Deletion became unsafe."); + second.ValidateDeleteAction = () => + { + validations.Add("second"); + if (reject) throw expected; + }; + var delete = manager.DeleteStateAsync(TestContext.Current.CancellationToken).AsTask(); + Assert.Equal(["first", "second"], validations); + reject = true; + storage.ReleaseAppend.SetResult(); + await WaitFor(write); + Assert.Same(expected, await Record.ExceptionAsync(() => WaitFor(delete))); + Assert.Equal(["first", "second", "first", "second"], validations); + Assert.Equal(0, first.DeleteStartedCount); + Assert.Equal(0, second.DeleteStartedCount); + Assert.Equal(0, storage.DeleteCount); + Assert.Same(expected, first.Failure); + Assert.Same(expected, second.Failure); + } + + [Fact] + public async Task Delete_StartsAfterAllValidationAndResetsBeforeWaiter() + { + var storage = new BlockingDeleteStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var events = new List(); + var first = new HookState(); + var second = new HookState(); + manager.RegisterStateMachine("first", first); + manager.RegisterStateMachine("second", second); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + first.ValidateDeleteAction = () => events.Add("validate first"); + second.ValidateDeleteAction = () => events.Add("validate second"); + first.DeleteStartedAction = () => events.Add("start first"); + second.DeleteStartedAction = () => events.Add("start second"); + first.ResetAction = () => events.Add("reset first"); + second.ResetAction = () => events.Add("reset second"); + + var delete = manager.DeleteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await WaitFor(storage.FirstDeleteStarted.Task); + Assert.Equal( + ["validate first", "validate second", "validate first", "validate second", "start first", "start second"], + events); + Assert.Equal(1, first.ResetCount); + Assert.Equal(1, second.ResetCount); + Assert.False(delete.IsCompleted); + storage.AllowFirstDelete.SetResult(); + await WaitFor(delete); + Assert.Equal(["reset first", "reset second"], events.TakeLast(2)); + Assert.Equal(2, first.ResetCount); + Assert.Equal(2, second.ResetCount); + } + + [Theory] + [InlineData("abc", false)] + [InlineData("acb", false)] + [InlineData("bac", false)] + [InlineData("bca", false)] + [InlineData("cab", false)] + [InlineData("cba", false)] + [InlineData("abc", true)] + [InlineData("acb", true)] + [InlineData("bac", true)] + [InlineData("bca", true)] + [InlineData("cab", true)] + [InlineData("cba", true)] + public async Task Readiness_RechecksAllStatesAfterAwaitBeforeFirstCapture(string order, bool snapshot) + { + var storage = new CapturingStorage { IsCompactionRequested = snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var entered = NewSignal(); + var release = NewSignal(); + var first = new HookState(); + var blocking = new HookState { Prepared = false }; + var retained = new HookState { Prepared = false }; + var states = new Dictionary { ['a'] = first, ['b'] = blocking, ['c'] = retained }; + blocking.PrepareAction = async token => + { + entered.SetResult(); + await release.Task.WaitAsync(token); + blocking.Prepared = true; + }; + foreach (var key in order) + { + var state = states[key]; + manager.RegisterStateMachine(key.ToString(), state); + state.CaptureAction = () => Assert.All(states.Values, value => Assert.True(value.Prepared)); + } + + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await WaitFor(entered.Task); + Assert.All(states.Values, state => Assert.Equal(0, state.CaptureCount)); + first.Prepared = false; + release.SetResult(); + await WaitFor(write); + + Assert.All(states.Values, state => + { + Assert.Equal(1, state.PrepareCount); + Assert.Equal(1, state.CaptureCount); + Assert.Equal(1, state.WriteCompletedCount); + }); + Assert.Equal(snapshot ? 1 : 0, storage.Replaces.Count); + Assert.Equal(snapshot ? 0 : 1, storage.Appends.Count); + } + + [Fact] + public async Task Readiness_FinalPassAndCaptureShareSchedulerTurn() + { + var context = new QueuedSynchronizationContext(); + await context.Run(async () => + { + await using var manager = CreateTestSystem().Manager; + var state = new HookState { Prepared = false }; + var readyTurn = -1; + state.PrepareAction = async _ => + { + await Task.Yield(); + state.Prepared = true; + }; + state.ReadinessAction = () => + { + if (state.Prepared) readyTurn = context.Turn; + return state.Prepared; + }; + state.CaptureAction = () => Assert.Equal(readyTurn, context.Turn); + manager.RegisterStateMachine("state", state); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, state.PrepareCount); + Assert.Equal(1, state.CaptureCount); + Assert.Equal(1, state.WriteCompletedCount); + }); + } + + [Fact] + public async Task ZeroByteWrite_PreparesStateWithoutWriteCompleted() + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var state = new HookState { EmitEntry = false }; + manager.RegisterStateMachine("state", state); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(0, manager.PendingWriteByteCount); + Assert.Equal(1, state.WriteCompletedCount); + state.Prepared = false; + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, state.PrepareCount); + Assert.Equal(2, state.CaptureCount); + Assert.Equal(1, state.WriteCompletedCount); + Assert.Single(storage.Appends); + } + + [Theory] + [InlineData("readiness")] + [InlineData("prepare")] + [InlineData("incomplete")] + public async Task PreparationFailure_FencesBeforeCapture(string failure) + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var expected = new InvalidOperationException("Preparation failed."); + var state = new HookState { Prepared = false }; + manager.RegisterStateMachine("state", state); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + if (failure == "readiness") state.ReadinessAction = () => throw expected; + else if (failure == "prepare") state.PrepareAction = _ => throw expected; + else state.PrepareAction = _ => default; + + var exception = await Assert.ThrowsAsync(() => + WaitFor(manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask())); + if (failure == "incomplete") Assert.Contains("without becoming prepared", exception.Message); + else Assert.Same(expected, exception); + Assert.Same(exception, state.Failure); + Assert.Equal(failure == "readiness" ? 0 : 1, state.PrepareCount); + Assert.Equal(0, state.CaptureCount); + Assert.Equal(0, state.WriteCompletedCount); + Assert.Empty(storage.Appends); + var rejected = await Assert.ThrowsAsync(() => + manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask()); + Assert.Same(exception, rejected.InnerException); + } + + [Fact] + public async Task CommittedOnlyWrite_ChecksLatchedStateFailure() + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var state = new HookState(); + manager.RegisterStateMachine("state", state); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var expected = new IOException("Latched state failure."); + state.ReadinessAction = () => throw expected; + var write = StartWhileEntryIsOpen(); + Assert.Same(expected, await Record.ExceptionAsync(() => WaitFor(write))); + Assert.Same(expected, state.Failure); + Assert.Equal(0, state.CaptureCount); + Assert.Empty(storage.Appends); + + Task StartWhileEntryIsOpen() + { + using var entry = state.Writer.BeginEntry(); + var result = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + Assert.True(SpinWait.SpinUntil(() => result.IsCompleted, TimeSpan.FromSeconds(10)), + "The committed-only write must observe readiness while the lexical entry is open."); + return result; + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Fault_NotifiesEveryStateBeforeWaitersAndPreservesOriginal(bool failPreparation) + { + var expected = new IOException("Original failure."); + var notificationFailure = new InvalidOperationException("Notification failed."); + var storage = new CapturingStorage { BlockNextAppend = !failPreparation, NextAppendException = expected }; + var logger = Substitute.For(); + logger.IsEnabled(Arg.Any()).Returns(true); + var loggerFactory = Substitute.For(); + loggerFactory.CreateLogger(Arg.Any()).Returns(logger); + var shared = new JournaledStateManagerShared(new Logger(loggerFactory), + Options.Create(ManagerOptions), TimeProvider.System, ServiceProvider); + await using var manager = new JournaledStateManager(shared, storage); + var entered = NewSignal(); + var release = NewSignal(); + var first = new HookState { Prepared = !failPreparation }; + var second = new HookState(); + manager.RegisterStateMachine("first", first); + manager.RegisterStateMachine("second", second); + first.PrepareAction = async token => + { + entered.SetResult(); + await release.Task.WaitAsync(token); + throw expected; + }; + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var current = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await WaitFor(failPreparation ? entered.Task : storage.BlockedAppendStarted.Task); + var queued = new[] + { + current, + manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(), + manager.DeleteStateAsync(TestContext.Current.CancellationToken).AsTask(), + manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask() + }; + var notified = new List(); + first.FaultAction = exception => + { + Assert.Same(expected, exception); + Assert.All(queued, task => Assert.False(task.IsCompleted)); + notified.Add("first"); + throw notificationFailure; + }; + second.FaultAction = exception => + { + Assert.Same(expected, exception); + Assert.All(queued, task => Assert.False(task.IsCompleted)); + var rejected = Assert.Throws(() => manager.RegisterStateMachine("late", new HookState())); + Assert.Same(expected, rejected.InnerException); + notified.Add("second"); + }; + + if (failPreparation) release.SetResult(); + else storage.ReleaseAppend.SetResult(); + foreach (var task in queued) + { + Assert.Same(expected, await Record.ExceptionAsync(() => WaitFor(task))); + } + + Assert.Equal(["first", "second"], notified); + Assert.Equal(1, first.FaultCount); + Assert.Equal(1, second.FaultCount); + Assert.Contains(logger.ReceivedCalls(), call => + call.GetMethodInfo().Name == nameof(ILogger.Log) + && Equals(call.GetArguments()[0], LogLevel.Error) + && ReferenceEquals(call.GetArguments()[3], notificationFailure)); + var late = await Assert.ThrowsAsync(() => manager.WriteStateAsync(CancellationToken.None).AsTask()); + Assert.Same(expected, late.InnerException); + Assert.Empty(storage.Appends); + } + + [Fact] + public async Task LatchedFailure_PreservesAlreadyCapturedWriteAcknowledgement() + { + var storage = new CapturingStorage { BlockNextAppend = true }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new HookState(); + manager.RegisterStateMachine("state", state); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var capturedWrite = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await WaitFor(storage.BlockedAppendStarted.Task); + var expected = new IOException("Failure after capture."); + state.ReadinessAction = () => throw expected; + var failingWrite = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + Assert.Null(state.Failure); + Assert.Equal(0, state.WriteCompletedCount); + storage.ReleaseAppend.SetResult(); + await WaitFor(capturedWrite); + Assert.Same(expected, await Record.ExceptionAsync(() => WaitFor(failingWrite))); + Assert.Equal(1, state.CaptureCount); + Assert.Equal(1, state.WriteCompletedCount); + Assert.Same(expected, state.Failure); + Assert.Single(storage.Appends); + } + + [Fact] + public async Task InitializationFailure_NotifiesRegisteredStatesOnce() + { + var expected = new IOException("Recovery failed."); + var storage = new CapturingStorage { NextReadException = expected }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new HookState(); + manager.RegisterStateMachine("state", state); + Assert.Same(expected, await Record.ExceptionAsync(() => manager.InitializeAsync(CancellationToken.None).AsTask())); + Assert.Same(expected, state.Failure); + var retry = await Assert.ThrowsAsync(() => manager.InitializeAsync(CancellationToken.None).AsTask()); + Assert.Same(expected, retry.InnerException); + Assert.Equal(1, state.FaultCount); + } + + [Fact] + public async Task IdleShutdown_CompletesWithoutFaultNotification() + { + var sut = CreateTestSystem(); + await using var manager = sut.Manager; + var state = new HookState(); + manager.RegisterStateMachine("state", state); + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); + await sut.Lifecycle.OnStop(TestContext.Current.CancellationToken); + Assert.Null(state.Failure); + Assert.Equal(0, state.FaultCount); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task AdmittedShutdownCancellation_NotifiesAndFaultsWaiters(bool preparation) + { + var storage = new CapturingStorage { BlockNextAppend = !preparation }; + var sut = CreateTestSystem(storage); + await using var manager = sut.Manager; + var state = new HookState { Prepared = !preparation }; + var entered = NewSignal(); + state.PrepareAction = async token => + { + entered.SetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, token); + }; + manager.RegisterStateMachine("state", state); + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); + var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await WaitFor(preparation ? entered.Task : storage.BlockedAppendStarted.Task); + var queued = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await sut.Lifecycle.OnStop(TestContext.Current.CancellationToken); + var exception = await Assert.ThrowsAnyAsync(() => WaitFor(write)); + Assert.Same(exception, state.Failure); + Assert.Same(exception, await Record.ExceptionAsync(() => WaitFor(queued))); + Assert.Equal(1, state.FaultCount); + Assert.Equal(0, state.WriteCompletedCount); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task CallerCancellation_PreservesOwnedPreparationStorageAndAck(bool cancelDuringPreparation) + { + var storage = new CapturingStorage { BlockNextAppend = true }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new HookState { Prepared = false }; + var entered = NewSignal(); + var release = NewSignal(); + var acknowledged = NewSignal(); + var ownedToken = CancellationToken.None; + state.PrepareAction = async token => + { + ownedToken = token; + entered.SetResult(); + await release.Task.WaitAsync(token); + state.Prepared = true; + }; + state.WriteCompletedAction = () => acknowledged.SetResult(); + manager.RegisterStateMachine("state", state); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + using var caller = new CancellationTokenSource(); + var write = manager.WriteStateAsync(caller.Token).AsTask(); + await WaitFor(entered.Task); + Assert.True(ownedToken.CanBeCanceled); + Assert.NotEqual(caller.Token, ownedToken); + if (cancelDuringPreparation) + { + caller.Cancel(); + await Assert.ThrowsAnyAsync(() => write); + Assert.False(ownedToken.IsCancellationRequested); + } + + release.SetResult(); + await WaitFor(storage.BlockedAppendStarted.Task); + if (!cancelDuringPreparation) + { + caller.Cancel(); + await Assert.ThrowsAnyAsync(() => write); + } + + Assert.False(ownedToken.IsCancellationRequested); + Assert.False(acknowledged.Task.IsCompleted); + state.EmitEntry = false; + var nextWrite = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + storage.ReleaseAppend.SetResult(); + await WaitFor(acknowledged.Task); + await WaitFor(nextWrite); + Assert.Equal(1, state.PrepareCount); + Assert.Equal(1, state.WriteCompletedCount); + Assert.Equal(0, manager.PendingWriteByteCount); + Assert.Null(state.Failure); + Assert.Single(storage.Appends); + } + + private static TaskCompletionSource NewSignal() => new(TaskCreationOptions.RunContinuationsAsynchronously); + + private static Task WaitFor(Task task) => task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + private sealed class QueuedSynchronizationContext : SynchronizationContext + { + private readonly Channel<(SendOrPostCallback Callback, object? State)> _queue = + Channel.CreateUnbounded<(SendOrPostCallback, object?)>(); + + public int Turn { get; private set; } + + public override void Post(SendOrPostCallback callback, object? state) => Assert.True(_queue.Writer.TryWrite((callback, state))); + + public async Task Run(Func action) + { + Task task = Task.CompletedTask; + Invoke(_ => task = action(), null); + while (!task.IsCompleted) + { + var next = await _queue.Reader.ReadAsync(TestContext.Current.CancellationToken).AsTask().WaitAsync(TimeSpan.FromSeconds(10)); + Invoke(next.Callback, next.State); + } + + await task; + } + + private void Invoke(SendOrPostCallback callback, object? state) + { + var previous = Current; + SetSynchronizationContext(this); + try + { + Turn++; + callback(state); + } + finally + { + SetSynchronizationContext(previous); + } + } + } + + private sealed class HookState : IStateMachine + { + public bool Prepared { get; set; } = true; + public bool EmitEntry { get; set; } = true; + public Func? ReadinessAction { get; set; } + public Func? PrepareAction { get; set; } + public Action? ValidateWriteAction { get; set; } + public Action? ValidateDeleteAction { get; set; } + public Action? DeleteStartedAction { get; set; } + public Action? ResetAction { get; set; } + public Action? CaptureAction { get; set; } + public Action? WriteCompletedAction { get; set; } + public Action? FaultAction { get; set; } + public JournalStreamWriter Writer { get; private set; } + public int ResetCount { get; private set; } + public int PrepareCount { get; private set; } + public int CaptureCount { get; private set; } + public int DeleteStartedCount { get; private set; } + public int WriteCompletedCount { get; private set; } + public int FaultCount { get; private set; } + public Exception? Failure { get; private set; } + public bool IsWritePrepared => ReadinessAction?.Invoke() ?? Prepared; + + public ValueTask PrepareWriteAsync(CancellationToken cancellationToken) + { + PrepareCount++; + if (PrepareAction is { } prepare) return prepare(cancellationToken); + Prepared = true; + return default; + } + + public void ValidateWrite() => ValidateWriteAction?.Invoke(); + public void ValidateDelete() => ValidateDeleteAction?.Invoke(); + + public void OnDeleteStarted() + { + DeleteStartedCount++; + DeleteStartedAction?.Invoke(); + } + + public void Reset(JournalStreamWriter writer) + { + Writer = writer; + ResetCount++; + ResetAction?.Invoke(); + } + + public void WritePendingEntries(JournalStreamWriter writer) + { + CaptureAction?.Invoke(); + CaptureCount++; + if (EmitEntry) + { + using var entry = writer.BeginEntry(); + entry.Writer.GetSpan(1)[0] = 1; + entry.Writer.Advance(1); + entry.Commit(); + } + } + + public void WriteSnapshot(JournalStreamWriter writer) => WritePendingEntries(writer); + + public void OnWriteCompleted() + { + WriteCompletedCount++; + WriteCompletedAction?.Invoke(); + } + + public void OnFaulted(Exception exception) + { + FaultCount++; + Failure = exception; + FaultAction?.Invoke(exception); + } + + public void ReplayEntry(JournalEntry entry, JournalReplayContext context) => throw new NotSupportedException(); + } +} diff --git a/test/Orleans.Journaling.Tests/StateManagerTests.cs b/test/Orleans.Journaling.Tests/StateManagerTests.cs index 73e072838c6..a7c06840779 100644 --- a/test/Orleans.Journaling.Tests/StateManagerTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerTests.cs @@ -26,7 +26,7 @@ namespace Orleans.Journaling.Tests; [TestSuite("BVT")] [TestProvider("None")] [TestCategory("BVT")] -public class StateManagerTests : JournalingTestBase +public partial class StateManagerTests : JournalingTestBase { /// /// Tests the registration and basic operation of multiple states. From d33017dbe8aab0b3908d74aba5929bcc91fc70ef Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 18 Sep 2026 14:30:27 -0700 Subject: [PATCH 02/11] fix(journaling): notify faulted states outside manager lock --- .../JournaledStateManager.cs | 7 ++- .../StateManagerLifecycleTests.cs | 59 +++++++++++++++++++ 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 6359f32bb1a..3ae384ed28f 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -609,7 +609,11 @@ private void Fence(Exception exception) _state = ManagerState.Fenced; _failure = exception; + } + try + { + // Fencing prevents registration, so callbacks can use the stable registry outside the lock. foreach (var (name, state) in _states) { try @@ -621,10 +625,7 @@ private void Fence(Exception exception) LogErrorNotifyingFaultedState(_shared.Logger, notificationException, name); } } - } - try - { if (!_shutdownCancellation.IsCancellationRequested) { LogErrorProcessingWorkItems(_shared.Logger, exception); diff --git a/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs b/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs index 820b8de8cb4..6fa9690111b 100644 --- a/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs @@ -374,6 +374,65 @@ public async Task Fault_NotifiesEveryStateBeforeWaitersAndPreservesOriginal(bool Assert.Empty(storage.Appends); } + [Fact] + public async Task Fault_CallbackCanCoordinateCrossThreadManagerReentry() + { + var expected = new IOException("Original storage failure."); + var storage = new CapturingStorage { BlockNextAppend = true, NextAppendException = expected }; + await using var manager = CreateTestSystem(storage).Manager; + var first = new HookState(); + var second = new HookState(); + manager.RegisterState("first", first); + manager.RegisterState("second", second); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var current = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await WaitFor(storage.BlockedAppendStarted.Task); + var queued = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + var callbackEntered = NewSignal(); + var workerReady = NewSignal(); + var worker = Task.Run(async () => + { + workerReady.SetResult(); + await WaitFor(callbackEntered.Task); + return (ThreadId: Environment.CurrentManagedThreadId, + Failure: Record.Exception(() => manager.RegisterState("late", new HookState()))); + }, TestContext.Current.CancellationToken); + await WaitFor(workerReady.Task); + Exception? callbackError = null; + var notifications = new List(); + first.FaultAction = exception => + { + callbackError = Record.Exception(() => + { + Assert.Same(expected, exception); + callbackEntered.SetResult(); + var result = worker.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken).GetAwaiter().GetResult(); + Assert.NotEqual(Environment.CurrentManagedThreadId, result.ThreadId); + Assert.Same(expected, Assert.IsType(result.Failure).InnerException); + Assert.False(current.IsCompleted); + Assert.False(queued.IsCompleted); + notifications.Add("first"); + }); + }; + second.FaultAction = exception => + { + Assert.Same(expected, exception); + Assert.False(current.IsCompleted); + Assert.False(queued.IsCompleted); + notifications.Add("second"); + }; + + storage.ReleaseAppend.SetResult(); + Assert.Same(expected, await Record.ExceptionAsync(() => WaitFor(current))); + Assert.Same(expected, await Record.ExceptionAsync(() => WaitFor(queued))); + await WaitFor(worker); + Assert.Null(callbackError); + Assert.Equal(["first", "second"], notifications); + Assert.Equal(1, first.FaultCount); + Assert.Equal(1, second.FaultCount); + Assert.False(manager.TryGetState("late", out _)); + } + [Fact] public async Task LatchedFailure_PreservesAlreadyCapturedWriteAcknowledgement() { From 0237ba44db0d2485941798baadff6615db806cbf Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Fri, 18 Sep 2026 17:09:56 -0700 Subject: [PATCH 03/11] test(journaling): complete state machine API port coverage --- .../grains/journaling/runtime-behavior.md | 3 +- .../IJournaledStateManager.cs | 3 +- .../KeyedJournalingRegistrationTests.cs | 43 +++++++++++++++++++ .../StateManagerLifecycleTests.cs | 9 ++-- 4 files changed, 52 insertions(+), 6 deletions(-) diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index 8fa22a2a66f..d8afbed9c6a 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -107,7 +107,8 @@ or repair the backing data before creating a fresh manager or retrying activatio Custom implementations share the manager's single logical execution thread. Resolve write codecs through to use the owning manager's configured format, including before recovery of an empty journal. Delegating managers forward -codec resolution to that owner. +codec resolution to that owner. Grain-bound managers resolve codecs from the activation's services; +standalone owners use shared application services. and perform pure validation in the requesting caller's context. An admission rejection leaves the manager healthy. diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index 167eeebf35f..5987d11dcb8 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -47,7 +47,8 @@ public interface IJournaledStateManager : IAsyncDisposable /// The command codec service type. /// The codec registered for this manager's write format. /// - /// Codecs are available before recovery, including for an empty journal. Delegating managers forward + /// Codecs are resolved from the owning activation's services for grain-bound managers and from shared + /// application services for standalone owners. They are available before recovery, including for an empty journal. Delegating managers forward /// this call to their owning manager. The default implementation throws . /// /// The manager does not support command codec resolution. diff --git a/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs b/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs index 3fe7eff258f..5281aec9408 100644 --- a/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs +++ b/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs @@ -382,6 +382,49 @@ public void DurableService_ResolvesCommandCodecFromJournalFormatKey() scope.ServiceProvider.GetRequiredService().ObservableLifecycle).Subscriptions); } + [Fact] + public async Task ManagerCommandCodec_UsesActivationScopeBeforeRecovery() + { + var builder = CreateNamedProviderBuilder(); + builder.AddVolatileJournalStorage(); + builder.Services.Configure(options => + options.JournalFormatKey = OrleansBinaryJournalFormat.JournalFormatKey); + builder.Services.AddScoped(services => + { + var context = Substitute.For(); + context.GrainId.Returns(GrainId.Create("codec-scope", Guid.NewGuid().ToString("N"))); + context.ActivationServices.Returns(services); + context.ObservableLifecycle.Returns(new CompositionTestLifecycle()); + return context; + }); + builder.Services.AddKeyedScoped>(OrleansBinaryJournalFormat.JournalFormatKey, + static (services, _) => new OrleansBinaryDurableValueCommandCodec( + services.GetRequiredService().GetCodec(), + services.GetRequiredService())); + await using var services = builder.Services.BuildServiceProvider(validateScopes: true); + await using var first = services.CreateAsyncScope(); + await using var second = services.CreateAsyncScope(); + var owner = first.ServiceProvider.GetRequiredService(); + var manager = first.ServiceProvider.GetRequiredService(); + Assert.Same(owner, manager); + var codec = owner.GetRequiredCommandCodec>(); + Assert.Same(first.ServiceProvider.GetRequiredKeyedService>(OrleansBinaryJournalFormat.JournalFormatKey), codec); + Assert.NotSame(codec, second.ServiceProvider.GetRequiredService() + .GetRequiredCommandCodec>()); + Assert.Throws(() => + services.GetRequiredKeyedService>(OrleansBinaryJournalFormat.JournalFormatKey)); + var state = new DurableValue("value", owner, codec); + Assert.Same(state, manager.GetOrAddValue("value")); + var lifecycle = Assert.IsType(first.ServiceProvider.GetRequiredService().ObservableLifecycle); + Assert.Equal(1, lifecycle.Subscriptions); + await lifecycle.OnStart(TestContext.Current.CancellationToken); + Assert.Equal(0, state.Value); + state.Value = 42; + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(0, owner.PendingWriteByteCount); + await lifecycle.OnStop(TestContext.Current.CancellationToken); + } + [Fact] public async Task ManagerCommandCodec_UsesOwningNamedFormatOnEmptyJournal() { diff --git a/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs b/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs index 6fa9690111b..6ed04633cb7 100644 --- a/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs @@ -382,8 +382,8 @@ public async Task Fault_CallbackCanCoordinateCrossThreadManagerReentry() await using var manager = CreateTestSystem(storage).Manager; var first = new HookState(); var second = new HookState(); - manager.RegisterState("first", first); - manager.RegisterState("second", second); + manager.RegisterStateMachine("first", first); + manager.RegisterStateMachine("second", second); await manager.InitializeAsync(TestContext.Current.CancellationToken); var current = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); await WaitFor(storage.BlockedAppendStarted.Task); @@ -395,7 +395,7 @@ public async Task Fault_CallbackCanCoordinateCrossThreadManagerReentry() workerReady.SetResult(); await WaitFor(callbackEntered.Task); return (ThreadId: Environment.CurrentManagedThreadId, - Failure: Record.Exception(() => manager.RegisterState("late", new HookState()))); + Failure: Record.Exception(() => manager.RegisterStateMachine("late", new HookState()))); }, TestContext.Current.CancellationToken); await WaitFor(workerReady.Task); Exception? callbackError = null; @@ -430,7 +430,8 @@ public async Task Fault_CallbackCanCoordinateCrossThreadManagerReentry() Assert.Equal(["first", "second"], notifications); Assert.Equal(1, first.FaultCount); Assert.Equal(1, second.FaultCount); - Assert.False(manager.TryGetState("late", out _)); + var lookup = Assert.Throws(() => manager.TryGetStateMachine("late", out _)); + Assert.Same(expected, lookup.InnerException); } [Fact] From f4285c6ea222e4feaf38aa1f45dc8ddeb474b4c0 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 20 Sep 2026 15:02:05 -0700 Subject: [PATCH 04/11] refactor(journaling): validate pending changes synchronously --- .../grains/journaling/runtime-behavior.md | 20 +- src/Orleans.Journaling/IStateMachine.cs | 35 +- .../JournaledStateManager.cs | 24 +- .../Orleans.Journaling/Orleans.Journaling.cs | 4 +- .../JournaledGrainCompositionTests.cs | 7 +- .../StateManagerLifecycleTests.cs | 519 +++++++++++------- 6 files changed, 353 insertions(+), 256 deletions(-) diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index d8afbed9c6a..a6c9fda4bdd 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -116,16 +116,22 @@ Deletion validates all states again in serialized execution, then calls on every state before awaiting storage deletion. Successful deletion resets states before completing callers. -Before each append or snapshot capture, the manager checks -and awaits for each unprepared state. Preparation -establishes that state's readiness and retains valid state-owned prerequisites across rechecks. Every await -is followed by another all-state readiness pass. The final successful pass flows directly into synchronous -capture in the same work-loop continuation. This also applies to writes which flush only committed entries -or produce zero bytes. Preparation uses the manager's shutdown token; caller cancellation only ends that caller's wait. +Before each append or snapshot capture, the manager calls + on every registered state. +This pure synchronous check validates state-owned failure latches and pending-change invariants inside +the admitted operation's failure boundary. Every state passes before capture begins in the same work-loop +continuation, including writes which flush only committed entries or produce zero bytes. + +Features acquire asynchronous prerequisites before staging their changes, then apply prepared mutations +synchronously and request an ordinary write. Independent operation-local preparation can continue while +the manager captures previously staged valid changes. `ValidateWrite` remains caller-context admission +validation; a terminal state-local error belongs in `ValidatePendingChanges` so the manager fences even +when the reporting write is its first admitted operation. A previously captured write retains its actual +storage outcome and acknowledgement bookkeeping. After storage acknowledges captured bytes, performs durable-completion bookkeeping. A zero-byte write completes without this callback. -An admitted preparation, validation, capture, or storage failure fences the manager, records the original +An admitted validation, capture, or storage failure fences the manager, records the original exception, and calls on every registered state before faulting current and queued waiters. Notification failures are logged while the original failure remains the operation's outcome. Idle shutdown completes normally; cancellation during admitted work is terminal. diff --git a/src/Orleans.Journaling/IStateMachine.cs b/src/Orleans.Journaling/IStateMachine.cs index 2bb0b7eda22..9d6ca36e953 100644 --- a/src/Orleans.Journaling/IStateMachine.cs +++ b/src/Orleans.Journaling/IStateMachine.cs @@ -15,7 +15,7 @@ namespace Orleans.Journaling; /// /// /// When the application requests a write, the journaled state manager validates the request, -/// prepares all states, and calls +/// validates the pending changes of all states, and calls /// (and occasionally ) to materialize /// the pending changes, then flushes the journal to durable storage. /// @@ -69,36 +69,25 @@ public interface IStateMachine void OnRecoveryCompleted() { } /// - /// Gets whether this state has the prerequisites required for synchronous write capture. - /// The default is . + /// Validates this state's pending changes immediately before an admitted write captures journal entries or a snapshot. + /// The default implementation accepts the pending changes. /// /// - /// This synchronous check is pure. The manager evaluates it inside the admitted operation's - /// failure boundary, including writes which only flush committed entries or produce zero bytes. - /// An unrecoverable state-local error can be reported by throwing, which fences the manager. + /// Validation is pure and synchronous. All registered states pass validation before any state is captured, + /// including writes which only flush committed entries or produce zero bytes. Validation and capture run + /// in the same work-loop continuation. Throwing reports a terminal state-local failure and fences the manager. + /// Callers acquire asynchronous prerequisites before staging changes. Independent operation-local preparation + /// can proceed while previously staged valid changes are captured. /// - bool IsWritePrepared => true; - - /// - /// Acquires state-owned prerequisites for synchronous write capture when is false. - /// The default implementation completes synchronously. - /// - /// The manager operation and shutdown token. - /// A task which completes when this state is prepared. - /// - /// Completion must establish this state's readiness. Retain valid prepared resources across readiness - /// rechecks. After preparation awaits, the manager rechecks every state and captures synchronously - /// in the same continuation as the final ready pass. Caller wait cancellation leaves preparation running. - /// Preparation failures fence the manager. - /// - ValueTask PrepareWriteAsync(CancellationToken cancellationToken) => default; + void ValidatePendingChanges() { } /// /// Validates a write request in the public caller's context before it is queued. /// The default implementation accepts the request. /// /// - /// Validation is pure. Throwing rejects this request and leaves the manager healthy. + /// Validation is pure and runs only at request admission. Throwing rejects this request and leaves the manager healthy. + /// Use to report a terminal state-local failure inside admitted execution. /// void ValidateWrite() { } @@ -130,7 +119,7 @@ void OnDeleteStarted() { } /// /// The manager is already fenced when this callback runs. Every registered state is notified even if /// another notification throws; notification errors are logged and the original failure is preserved. - /// Idle shutdown completes normally. Cancellation during admitted preparation or storage work is terminal. + /// Idle shutdown completes normally. Cancellation during admitted validation or storage work is terminal. /// void OnFaulted(Exception exception) { } diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 3ae384ed28f..b8ce48810ec 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -256,30 +256,10 @@ private async Task WorkLoop() case AppendJournalWorkItem: case WriteSnapshotWorkItem: { - // Keep the final readiness pass and synchronous capture in this continuation. - bool prepared; - do + foreach (var state in _states.Values) { - prepared = true; - foreach (var (name, state) in _states) - { - if (state.IsWritePrepared) - { - continue; - } - - await state.PrepareWriteAsync(_shutdownCancellation.Token).ConfigureAwait(true); - if (!state.IsWritePrepared) - { - throw new InvalidOperationException( - $"Journaled state '{name}' completed write preparation without becoming prepared."); - } - - prepared = false; - break; - } + state.ValidatePendingChanges(); } - while (!prepared); // TODO: decide whether it's best to snapshot or append. Eg, by summing the size of the most recent snapshots and the current journal length. // If the current journal length is greater than the snapshot size, then take a snapshot instead of appending more journal entries. diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index f1a76cf9719..c2cb5bd35d6 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -306,16 +306,14 @@ public partial interface IPreservedJournalEntry public partial interface IStateMachine { - bool IsWritePrepared { get; } - void OnDeleteStarted(); void OnFaulted(System.Exception exception); void OnRecoveryCompleted(); void OnWriteCompleted(); - System.Threading.Tasks.ValueTask PrepareWriteAsync(System.Threading.CancellationToken cancellationToken); void ReplayEntry(JournalEntry entry, JournalReplayContext context); void Reset(JournalStreamWriter writer); void ValidateDelete(); + void ValidatePendingChanges(); void ValidateWrite(); void WritePendingEntries(JournalStreamWriter writer); void WriteSnapshot(JournalStreamWriter writer); diff --git a/test/Orleans.Journaling.Tests/JournaledGrainCompositionTests.cs b/test/Orleans.Journaling.Tests/JournaledGrainCompositionTests.cs index 1036fb5838f..ad162decba4 100644 --- a/test/Orleans.Journaling.Tests/JournaledGrainCompositionTests.cs +++ b/test/Orleans.Journaling.Tests/JournaledGrainCompositionTests.cs @@ -336,12 +336,7 @@ private static CompositionSiloBuilder CreateBuilder() builder.Services.AddSingleton(TimeProvider.System); builder.Services.AddKeyedSingleton(KeyedService.AnyKey, static (services, _) => services.GetRequiredService()); builder.AddVolatileJournalStorage().UseJsonJournalFormat(JournalingTestsJsonContext.Default); - builder.Services.AddStateMachine(static (_, _) => - { - var state = Substitute.For(); - state.IsWritePrepared.Returns(true); - return state; - }); + builder.Services.AddStateMachine(static (_, _) => Substitute.For()); return builder; } diff --git a/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs b/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs index 6ed04633cb7..dec84f68380 100644 --- a/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs @@ -12,8 +12,7 @@ public partial class StateManagerTests public async Task StateHooks_DefaultsAcceptExistingStates() { IStateMachine state = new AlwaysWritingState(); - Assert.True(state.IsWritePrepared); - Assert.True(state.PrepareWriteAsync(CancellationToken.None).IsCompletedSuccessfully); + state.ValidatePendingChanges(); state.ValidateWrite(); state.ValidateDelete(); state.OnDeleteStarted(); @@ -156,78 +155,73 @@ public async Task Delete_StartsAfterAllValidationAndResetsBeforeWaiter() [InlineData("bca", true)] [InlineData("cab", true)] [InlineData("cba", true)] - public async Task Readiness_RechecksAllStatesAfterAwaitBeforeFirstCapture(string order, bool snapshot) + public async Task PendingValidation_ValidatesAllStatesBeforeAnyCapture(string order, bool snapshot) { var storage = new CapturingStorage { IsCompactionRequested = snapshot }; - await using var manager = CreateTestSystem(storage).Manager; - var entered = NewSignal(); - var release = NewSignal(); - var first = new HookState(); - var blocking = new HookState { Prepared = false }; - var retained = new HookState { Prepared = false }; - var states = new Dictionary { ['a'] = first, ['b'] = blocking, ['c'] = retained }; - blocking.PrepareAction = async token => - { - entered.SetResult(); - await release.Task.WaitAsync(token); - blocking.Prepared = true; - }; + var format = new TrackingJournalFormat(SessionPool); + await using var manager = CreateTestSystem(storage, journalFormat: format).Manager; + var states = new Dictionary { ['a'] = new(), ['b'] = new(), ['c'] = new() }; + var events = new List(); foreach (var key in order) { var state = states[key]; manager.RegisterStateMachine(key.ToString(), state); - state.CaptureAction = () => Assert.All(states.Values, value => Assert.True(value.Prepared)); + state.CaptureAction = () => + { + Assert.All(states.Values, value => Assert.Equal(1, value.PendingValidationCount)); + events.Add($"capture {key}"); + }; } await manager.InitializeAsync(TestContext.Current.CancellationToken); - var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - await WaitFor(entered.Task); - Assert.All(states.Values, state => Assert.Equal(0, state.CaptureCount)); - first.Prepared = false; - release.SetResult(); - await WaitFor(write); + var initialWriter = Assert.Single(format.Writers); + var initialEntries = initialWriter.BeganEntryIds.Count; + foreach (var key in order) + { + states[key].PendingValidationAction = () => + { + Assert.Same(initialWriter, Assert.Single(format.Writers)); + Assert.Equal(initialEntries, initialWriter.BeganEntryIds.Count); + Assert.All(states.Values, value => Assert.Equal(0, value.CaptureCount)); + events.Add($"validate {key}"); + }; + } + + await manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.All(states.Values, state => { - Assert.Equal(1, state.PrepareCount); + Assert.Equal(1, state.PendingValidationCount); Assert.Equal(1, state.CaptureCount); Assert.Equal(1, state.WriteCompletedCount); }); + Assert.Equal(order.Select(key => $"validate {key}").Concat(order.Select(key => $"capture {key}")), events); Assert.Equal(snapshot ? 1 : 0, storage.Replaces.Count); Assert.Equal(snapshot ? 0 : 1, storage.Appends.Count); } [Fact] - public async Task Readiness_FinalPassAndCaptureShareSchedulerTurn() + public async Task PendingValidation_AndCaptureShareSchedulerTurn() { var context = new QueuedSynchronizationContext(); await context.Run(async () => { await using var manager = CreateTestSystem().Manager; - var state = new HookState { Prepared = false }; - var readyTurn = -1; - state.PrepareAction = async _ => - { - await Task.Yield(); - state.Prepared = true; - }; - state.ReadinessAction = () => - { - if (state.Prepared) readyTurn = context.Turn; - return state.Prepared; - }; - state.CaptureAction = () => Assert.Equal(readyTurn, context.Turn); + var state = new HookState(); + var validationTurn = -1; + state.PendingValidationAction = () => validationTurn = context.Turn; + state.CaptureAction = () => Assert.Equal(validationTurn, context.Turn); manager.RegisterStateMachine("state", state); await manager.InitializeAsync(TestContext.Current.CancellationToken); await manager.WriteStateAsync(TestContext.Current.CancellationToken); - Assert.Equal(1, state.PrepareCount); + Assert.Equal(1, state.PendingValidationCount); Assert.Equal(1, state.CaptureCount); Assert.Equal(1, state.WriteCompletedCount); }); } [Fact] - public async Task ZeroByteWrite_PreparesStateWithoutWriteCompleted() + public async Task ZeroByteWrite_ValidatesPendingChangesWithoutWriteCompleted() { var storage = new CapturingStorage(); await using var manager = CreateTestSystem(storage).Manager; @@ -237,66 +231,205 @@ public async Task ZeroByteWrite_PreparesStateWithoutWriteCompleted() await manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Equal(0, manager.PendingWriteByteCount); Assert.Equal(1, state.WriteCompletedCount); - state.Prepared = false; await manager.WriteStateAsync(TestContext.Current.CancellationToken); - Assert.Equal(1, state.PrepareCount); + Assert.Equal(2, state.PendingValidationCount); Assert.Equal(2, state.CaptureCount); Assert.Equal(1, state.WriteCompletedCount); Assert.Single(storage.Appends); } [Theory] - [InlineData("readiness")] - [InlineData("prepare")] - [InlineData("incomplete")] - public async Task PreparationFailure_FencesBeforeCapture(string failure) + [InlineData("append", false, false)] + [InlineData("append", false, true)] + [InlineData("append", true, false)] + [InlineData("append", true, true)] + [InlineData("snapshot", false, false)] + [InlineData("snapshot", false, true)] + [InlineData("snapshot", true, false)] + [InlineData("snapshot", true, true)] + [InlineData("empty-buffer", false, false)] + [InlineData("empty-buffer", false, true)] + [InlineData("empty-buffer", true, false)] + [InlineData("empty-buffer", true, true)] + public async Task PartialApplyFailure_FencesBeforeCapture(string path, bool previouslyAdmitted, bool stateFirst) { - var storage = new CapturingStorage(); - await using var manager = CreateTestSystem(storage).Manager; - var expected = new InvalidOperationException("Preparation failed."); - var state = new HookState { Prepared = false }; - manager.RegisterStateMachine("state", state); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - if (failure == "readiness") state.ReadinessAction = () => throw expected; - else if (failure == "prepare") state.PrepareAction = _ => throw expected; - else state.PrepareAction = _ => default; - - var exception = await Assert.ThrowsAsync(() => - WaitFor(manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask())); - if (failure == "incomplete") Assert.Contains("without becoming prepared", exception.Message); - else Assert.Same(expected, exception); - Assert.Same(exception, state.Failure); - Assert.Equal(failure == "readiness" ? 0 : 1, state.PrepareCount); - Assert.Equal(0, state.CaptureCount); - Assert.Equal(0, state.WriteCompletedCount); - Assert.Empty(storage.Appends); - var rejected = await Assert.ThrowsAsync(() => - manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask()); - Assert.Same(exception, rejected.InnerException); + var context = new QueuedSynchronizationContext(); + await context.Run(async () => + { + var storage = new CapturingStorage(); + var format = new TrackingJournalFormat(SessionPool); + await using var manager = CreateTestSystem(storage, journalFormat: format).Manager; + var state = new HookState { EmitEntry = false }; + var other = new HookState { EmitEntry = false }; + if (stateFirst) manager.RegisterStateMachine("state", state); + var value = new DurableValue("business", manager, CreateValueCodec()); + manager.RegisterStateMachine("other", other); + if (!stateFirst) manager.RegisterStateMachine("state", state); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + if (path == "empty-buffer") + { + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(0, manager.PendingWriteByteCount); + } + + storage.IsCompactionRequested = path == "snapshot"; + var previousWrites = storage.Appends.Count; + var previousCaptures = state.CaptureCount; + var previousAcks = state.WriteCompletedCount; + var initialWriter = Assert.Single(format.Writers); + var initialEntries = initialWriter.BeganEntryIds.Count; + var waiters = new List(); + var expected = new InvalidOperationException("Synchronous apply failed after changing business state."); + Exception? latchedFailure = null; + var handlerContext = new AsyncLocal(); + var admissions = 0; + state.ValidateWriteAction = () => + { + Assert.False(handlerContext.Value); + admissions++; + }; + state.PendingValidationAction = () => + { + Assert.Same(initialWriter, Assert.Single(format.Writers)); + Assert.Equal(initialEntries, initialWriter.BeganEntryIds.Count); + Assert.Same(expected, latchedFailure); + throw latchedFailure!; + }; + if (previouslyAdmitted) waiters.Add(manager.WriteStateAsync(CancellationToken.None).AsTask()); + handlerContext.Value = true; + try + { + value.Value = 42; + throw expected; + } + catch (InvalidOperationException exception) + { + latchedFailure = exception; + } + finally + { + handlerContext.Value = false; + } + + if (path == "empty-buffer") Assert.Equal(0, manager.PendingWriteByteCount); + waiters.Add(manager.WriteStateAsync(CancellationToken.None).AsTask()); + waiters.Add(manager.DeleteStateAsync(CancellationToken.None).AsTask()); + waiters.Add(manager.InitializeAsync(CancellationToken.None).AsTask()); + var notifications = new List(); + var stateOwnedWaiter = NewSignal(); + state.FaultAction = exception => + { + Assert.Same(expected, exception); + Assert.All(waiters, waiter => Assert.False(waiter.IsCompleted)); + stateOwnedWaiter.TrySetException(exception); + notifications.Add("state"); + }; + other.FaultAction = exception => + { + Assert.Same(expected, exception); + Assert.All(waiters, waiter => Assert.False(waiter.IsCompleted)); + notifications.Add("other"); + }; + Assert.Equal(previouslyAdmitted ? 2 : 1, admissions); + foreach (var waiter in waiters) + { + Assert.Same(expected, await Record.ExceptionAsync(() => WaitFor(waiter))); + } + + Assert.Same(expected, await Record.ExceptionAsync(() => WaitFor(stateOwnedWaiter.Task))); + Assert.Equal(stateFirst ? new[] { "state", "other" } : ["other", "state"], notifications); + Assert.Equal(42, value.Value); + Assert.Equal(previousCaptures, state.CaptureCount); + Assert.Equal(previousAcks, state.WriteCompletedCount); + Assert.Equal(previousWrites, storage.Appends.Count); + Assert.Empty(storage.Replaces); + Assert.Equal(0, storage.DeleteCount); + var rejected = await Assert.ThrowsAsync(() => manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask()); + Assert.Same(expected, rejected.InnerException); + }); } [Fact] - public async Task CommittedOnlyWrite_ChecksLatchedStateFailure() + public async Task PendingValidation_AllowsIndependentWriteDuringLocalPreparation() + { + var context = new QueuedSynchronizationContext(); + await context.Run(async () => + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var state = new HookState(); + manager.RegisterStateMachine("state", state); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var handlerContext = new AsyncLocal(); + var preparing = false; + var admissionCalls = 0; + var rejected = new InvalidOperationException("Handler preparation cannot commit state."); + state.ValidateWriteAction = () => + { + admissionCalls++; + if (handlerContext.Value) throw rejected; + }; + state.PendingValidationAction = () => + { + Assert.True(preparing); + Assert.False(handlerContext.Value); + Assert.Equal(2, admissionCalls); + }; + + var independent = manager.WriteStateAsync(CancellationToken.None).AsTask(); + preparing = true; + handlerContext.Value = true; + Assert.Same(rejected, await Record.ExceptionAsync(() => manager.WriteStateAsync(CancellationToken.None).AsTask())); + await independent; + Assert.Equal(1, state.PendingValidationCount); + Assert.Equal(1, state.WriteCompletedCount); + Assert.Single(storage.Appends); + Assert.Null(state.Failure); + + handlerContext.Value = false; + preparing = false; + state.PendingValidationAction = null; + await manager.WriteStateAsync(CancellationToken.None); + Assert.Equal(3, admissionCalls); + Assert.Equal(2, state.WriteCompletedCount); + Assert.Null(state.Failure); + }); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task CommittedOnlyWrite_ChecksLatchedStateFailure(bool emptyPrefix) { var storage = new CapturingStorage(); await using var manager = CreateTestSystem(storage).Manager; var state = new HookState(); manager.RegisterStateMachine("state", state); await manager.InitializeAsync(TestContext.Current.CancellationToken); + if (emptyPrefix) + { + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(0, manager.PendingWriteByteCount); + } + + var previousCaptures = state.CaptureCount; + var previousWrites = storage.Appends.Count; var expected = new IOException("Latched state failure."); - state.ReadinessAction = () => throw expected; + state.PendingValidationAction = () => throw expected; var write = StartWhileEntryIsOpen(); Assert.Same(expected, await Record.ExceptionAsync(() => WaitFor(write))); Assert.Same(expected, state.Failure); - Assert.Equal(0, state.CaptureCount); - Assert.Empty(storage.Appends); + Assert.Equal(previousCaptures, state.CaptureCount); + Assert.Equal(previousWrites, storage.Appends.Count); Task StartWhileEntryIsOpen() { using var entry = state.Writer.BeginEntry(); var result = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - Assert.True(SpinWait.SpinUntil(() => result.IsCompleted, TimeSpan.FromSeconds(10)), - "The committed-only write must observe readiness while the lexical entry is open."); + using var completed = new ManualResetEventSlim(); + result.GetAwaiter().OnCompleted(completed.Set); + Assert.True(completed.Wait(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken), + "The committed-only write must validate pending changes while the lexical entry is open."); return result; } } @@ -304,74 +437,70 @@ Task StartWhileEntryIsOpen() [Theory] [InlineData(false)] [InlineData(true)] - public async Task Fault_NotifiesEveryStateBeforeWaitersAndPreservesOriginal(bool failPreparation) + public async Task Fault_NotifiesEveryStateBeforeWaitersAndPreservesOriginal(bool failValidation) { - var expected = new IOException("Original failure."); - var notificationFailure = new InvalidOperationException("Notification failed."); - var storage = new CapturingStorage { BlockNextAppend = !failPreparation, NextAppendException = expected }; - var logger = Substitute.For(); - logger.IsEnabled(Arg.Any()).Returns(true); - var loggerFactory = Substitute.For(); - loggerFactory.CreateLogger(Arg.Any()).Returns(logger); - var shared = new JournaledStateManagerShared(new Logger(loggerFactory), - Options.Create(ManagerOptions), TimeProvider.System, ServiceProvider); - await using var manager = new JournaledStateManager(shared, storage); - var entered = NewSignal(); - var release = NewSignal(); - var first = new HookState { Prepared = !failPreparation }; - var second = new HookState(); - manager.RegisterStateMachine("first", first); - manager.RegisterStateMachine("second", second); - first.PrepareAction = async token => - { - entered.SetResult(); - await release.Task.WaitAsync(token); - throw expected; - }; - await manager.InitializeAsync(TestContext.Current.CancellationToken); - var current = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - await WaitFor(failPreparation ? entered.Task : storage.BlockedAppendStarted.Task); - var queued = new[] - { - current, - manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(), - manager.DeleteStateAsync(TestContext.Current.CancellationToken).AsTask(), - manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask() - }; - var notified = new List(); - first.FaultAction = exception => - { - Assert.Same(expected, exception); - Assert.All(queued, task => Assert.False(task.IsCompleted)); - notified.Add("first"); - throw notificationFailure; - }; - second.FaultAction = exception => + var context = new QueuedSynchronizationContext(); + await context.Run(async () => { - Assert.Same(expected, exception); - Assert.All(queued, task => Assert.False(task.IsCompleted)); - var rejected = Assert.Throws(() => manager.RegisterStateMachine("late", new HookState())); - Assert.Same(expected, rejected.InnerException); - notified.Add("second"); - }; + var expected = new IOException("Original failure."); + var notificationFailure = new InvalidOperationException("Notification failed."); + var storage = new CapturingStorage { BlockNextAppend = !failValidation, NextAppendException = expected }; + var logger = Substitute.For(); + logger.IsEnabled(Arg.Any()).Returns(true); + var loggerFactory = Substitute.For(); + loggerFactory.CreateLogger(Arg.Any()).Returns(logger); + var shared = new JournaledStateManagerShared(new Logger(loggerFactory), + Options.Create(ManagerOptions), TimeProvider.System, ServiceProvider); + await using var manager = new JournaledStateManager(shared, storage); + var first = new HookState(); + var second = new HookState(); + manager.RegisterStateMachine("first", first); + manager.RegisterStateMachine("second", second); + if (failValidation) first.PendingValidationAction = () => throw expected; + await manager.InitializeAsync(TestContext.Current.CancellationToken); + var current = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + if (!failValidation) await WaitFor(storage.BlockedAppendStarted.Task); + var queued = new[] + { + current, + manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(), + manager.DeleteStateAsync(TestContext.Current.CancellationToken).AsTask(), + manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask() + }; + var notified = new List(); + first.FaultAction = exception => + { + Assert.Same(expected, exception); + Assert.All(queued, task => Assert.False(task.IsCompleted)); + notified.Add("first"); + throw notificationFailure; + }; + second.FaultAction = exception => + { + Assert.Same(expected, exception); + Assert.All(queued, task => Assert.False(task.IsCompleted)); + var rejected = Assert.Throws(() => manager.RegisterStateMachine("late", new HookState())); + Assert.Same(expected, rejected.InnerException); + notified.Add("second"); + }; - if (failPreparation) release.SetResult(); - else storage.ReleaseAppend.SetResult(); - foreach (var task in queued) - { - Assert.Same(expected, await Record.ExceptionAsync(() => WaitFor(task))); - } + if (!failValidation) storage.ReleaseAppend.SetResult(); + foreach (var task in queued) + { + Assert.Same(expected, await Record.ExceptionAsync(() => WaitFor(task))); + } - Assert.Equal(["first", "second"], notified); - Assert.Equal(1, first.FaultCount); - Assert.Equal(1, second.FaultCount); - Assert.Contains(logger.ReceivedCalls(), call => - call.GetMethodInfo().Name == nameof(ILogger.Log) - && Equals(call.GetArguments()[0], LogLevel.Error) - && ReferenceEquals(call.GetArguments()[3], notificationFailure)); - var late = await Assert.ThrowsAsync(() => manager.WriteStateAsync(CancellationToken.None).AsTask()); - Assert.Same(expected, late.InnerException); - Assert.Empty(storage.Appends); + Assert.Equal(["first", "second"], notified); + Assert.Equal(1, first.FaultCount); + Assert.Equal(1, second.FaultCount); + Assert.Contains(logger.ReceivedCalls(), call => + call.GetMethodInfo().Name == nameof(ILogger.Log) + && Equals(call.GetArguments()[0], LogLevel.Error) + && ReferenceEquals(call.GetArguments()[3], notificationFailure)); + var late = await Assert.ThrowsAsync(() => manager.WriteStateAsync(CancellationToken.None).AsTask()); + Assert.Same(expected, late.InnerException); + Assert.Empty(storage.Appends); + }); } [Fact] @@ -445,7 +574,7 @@ public async Task LatchedFailure_PreservesAlreadyCapturedWriteAcknowledgement() var capturedWrite = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); await WaitFor(storage.BlockedAppendStarted.Task); var expected = new IOException("Failure after capture."); - state.ReadinessAction = () => throw expected; + state.PendingValidationAction = () => throw expected; var failingWrite = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); Assert.Null(state.Failure); Assert.Equal(0, state.WriteCompletedCount); @@ -489,22 +618,21 @@ public async Task IdleShutdown_CompletesWithoutFaultNotification() [Theory] [InlineData(false)] [InlineData(true)] - public async Task AdmittedShutdownCancellation_NotifiesAndFaultsWaiters(bool preparation) + public async Task AdmittedShutdownCancellation_NotifiesAndFaultsWaiters(bool snapshot) { - var storage = new CapturingStorage { BlockNextAppend = !preparation }; - var sut = CreateTestSystem(storage); - await using var manager = sut.Manager; - var state = new HookState { Prepared = !preparation }; - var entered = NewSignal(); - state.PrepareAction = async token => + var storage = new CapturingStorage { - entered.SetResult(); - await Task.Delay(Timeout.InfiniteTimeSpan, token); + IsCompactionRequested = snapshot, + BlockNextAppend = !snapshot, + BlockNextReplace = snapshot }; + var sut = CreateTestSystem(storage); + await using var manager = sut.Manager; + var state = new HookState(); manager.RegisterStateMachine("state", state); await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - await WaitFor(preparation ? entered.Task : storage.BlockedAppendStarted.Task); + await WaitFor(snapshot ? storage.ReplaceEntered.Task : storage.BlockedAppendStarted.Task); var queued = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); await sut.Lifecycle.OnStop(TestContext.Current.CancellationToken); var exception = await Assert.ThrowsAnyAsync(() => WaitFor(write)); @@ -517,57 +645,64 @@ public async Task AdmittedShutdownCancellation_NotifiesAndFaultsWaiters(bool pre [Theory] [InlineData(false)] [InlineData(true)] - public async Task CallerCancellation_PreservesOwnedPreparationStorageAndAck(bool cancelDuringPreparation) + public async Task CallerCancellation_PreservesOwnedStorageAndAck(bool snapshot) { - var storage = new CapturingStorage { BlockNextAppend = true }; - await using var manager = CreateTestSystem(storage).Manager; - var state = new HookState { Prepared = false }; - var entered = NewSignal(); - var release = NewSignal(); - var acknowledged = NewSignal(); - var ownedToken = CancellationToken.None; - state.PrepareAction = async token => + var storage = new CapturingStorage { - ownedToken = token; - entered.SetResult(); - await release.Task.WaitAsync(token); - state.Prepared = true; + IsCompactionRequested = snapshot, + BlockNextAppend = !snapshot, + BlockNextReplace = snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var state = new HookState(); + var acknowledged = NewSignal(); state.WriteCompletedAction = () => acknowledged.SetResult(); manager.RegisterStateMachine("state", state); await manager.InitializeAsync(TestContext.Current.CancellationToken); using var caller = new CancellationTokenSource(); var write = manager.WriteStateAsync(caller.Token).AsTask(); - await WaitFor(entered.Task); - Assert.True(ownedToken.CanBeCanceled); - Assert.NotEqual(caller.Token, ownedToken); - if (cancelDuringPreparation) - { - caller.Cancel(); - await Assert.ThrowsAnyAsync(() => write); - Assert.False(ownedToken.IsCancellationRequested); - } - - release.SetResult(); - await WaitFor(storage.BlockedAppendStarted.Task); - if (!cancelDuringPreparation) - { - caller.Cancel(); - await Assert.ThrowsAnyAsync(() => write); - } - - Assert.False(ownedToken.IsCancellationRequested); + await WaitFor(snapshot ? storage.ReplaceEntered.Task : storage.BlockedAppendStarted.Task); + caller.Cancel(); + await Assert.ThrowsAnyAsync(() => write); Assert.False(acknowledged.Task.IsCompleted); state.EmitEntry = false; + storage.IsCompactionRequested = false; var nextWrite = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - storage.ReleaseAppend.SetResult(); + if (snapshot) storage.ReleaseReplace.SetResult(); + else storage.ReleaseAppend.SetResult(); await WaitFor(acknowledged.Task); await WaitFor(nextWrite); - Assert.Equal(1, state.PrepareCount); + Assert.Equal(2, state.PendingValidationCount); Assert.Equal(1, state.WriteCompletedCount); Assert.Equal(0, manager.PendingWriteByteCount); Assert.Null(state.Failure); - Assert.Single(storage.Appends); + Assert.Equal(snapshot ? 0 : 1, storage.Appends.Count); + Assert.Equal(snapshot ? 1 : 0, storage.Replaces.Count); + } + + [Fact] + public async Task CallerCancellation_PreservesQueuedWrite() + { + var context = new QueuedSynchronizationContext(); + await context.Run(async () => + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var state = new HookState(); + manager.RegisterStateMachine("state", state); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + using var caller = new CancellationTokenSource(); + var write = manager.WriteStateAsync(caller.Token).AsTask(); + Assert.Equal(0, state.PendingValidationCount); + caller.Cancel(); + var remainingWaiter = manager.WriteStateAsync(CancellationToken.None).AsTask(); + await Assert.ThrowsAnyAsync(() => write); + await remainingWaiter; + Assert.Single(storage.Appends); + Assert.Equal(1, state.PendingValidationCount); + Assert.Equal(1, state.WriteCompletedCount); + Assert.Null(state.Failure); + }); } private static TaskCompletionSource NewSignal() => new(TaskCreationOptions.RunContinuationsAsynchronously); @@ -614,10 +749,8 @@ private void Invoke(SendOrPostCallback callback, object? state) private sealed class HookState : IStateMachine { - public bool Prepared { get; set; } = true; public bool EmitEntry { get; set; } = true; - public Func? ReadinessAction { get; set; } - public Func? PrepareAction { get; set; } + public Action? PendingValidationAction { get; set; } public Action? ValidateWriteAction { get; set; } public Action? ValidateDeleteAction { get; set; } public Action? DeleteStartedAction { get; set; } @@ -627,20 +760,16 @@ private sealed class HookState : IStateMachine public Action? FaultAction { get; set; } public JournalStreamWriter Writer { get; private set; } public int ResetCount { get; private set; } - public int PrepareCount { get; private set; } + public int PendingValidationCount { get; private set; } public int CaptureCount { get; private set; } public int DeleteStartedCount { get; private set; } public int WriteCompletedCount { get; private set; } public int FaultCount { get; private set; } public Exception? Failure { get; private set; } - public bool IsWritePrepared => ReadinessAction?.Invoke() ?? Prepared; - - public ValueTask PrepareWriteAsync(CancellationToken cancellationToken) + public void ValidatePendingChanges() { - PrepareCount++; - if (PrepareAction is { } prepare) return prepare(cancellationToken); - Prepared = true; - return default; + PendingValidationCount++; + PendingValidationAction?.Invoke(); } public void ValidateWrite() => ValidateWriteAction?.Invoke(); From 6e82a781da811e0f5b323f642b2fe232b28f596d Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Sun, 20 Sep 2026 15:33:12 -0700 Subject: [PATCH 05/11] fix(journaling): treat owner-canceled recovery as shutdown --- .../grains/journaling/runtime-behavior.md | 7 +- .../IJournaledStateManager.cs | 4 +- src/Orleans.Journaling/IStateMachine.cs | 3 +- .../JournaledStateManager.cs | 4 + .../StateManagerLifecycleTests.cs | 203 ++++++++++++++++++ .../StateManagerTests.cs | 3 + 6 files changed, 221 insertions(+), 3 deletions(-) diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index a6c9fda4bdd..0e650ccefcf 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -102,6 +102,10 @@ storage outcome, so the caller reconciles that outcome before retrying the comma An initialization failure preserves stored data for diagnosis. Restore the required format/codec registration or repair the backing data before creating a fresh manager or retrying activation. +Owner shutdown which cancels initial recovery cancels all initialization waiters and leaves the manager +stopped. Disposal waits for the owned read to finish before releasing journal resources. Cancelling an +individual initialization caller's token ends only its wait; owned recovery continues for other callers. + ## Custom state lifecycle Custom implementations share the manager's single logical execution thread. @@ -134,7 +138,8 @@ performs durable-completion bookkeeping. A zero-byte write completes without thi An admitted validation, capture, or storage failure fences the manager, records the original exception, and calls on every registered state before faulting current and queued waiters. Notification failures are logged while the original failure remains -the operation's outcome. Idle shutdown completes normally; cancellation during admitted work is terminal. +the operation's outcome. Owner-canceled initial recovery and idle shutdown complete through normal shutdown. +Cancellation during admitted validation or write/delete storage work is terminal. ## Compaction diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index 5987d11dcb8..af08e64a0e3 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -19,7 +19,9 @@ public interface IJournaledStateManager : IAsyncDisposable /// Initializes the state manager by replaying its journal. /// /// - /// A failed initialization permanently fences this instance. Recover by creating a new manager and new state instances. + /// A recovery failure permanently fences this instance. Recover by creating a new manager and new state instances. + /// Owner shutdown which cancels recovery cancels initialization and leaves this instance stopped. + /// Cancelling the caller's token ends only that caller's wait while owned recovery continues. /// /// The cancellation token. /// A which represents the operation. diff --git a/src/Orleans.Journaling/IStateMachine.cs b/src/Orleans.Journaling/IStateMachine.cs index 9d6ca36e953..f6ecb826822 100644 --- a/src/Orleans.Journaling/IStateMachine.cs +++ b/src/Orleans.Journaling/IStateMachine.cs @@ -119,7 +119,8 @@ void OnDeleteStarted() { } /// /// The manager is already fenced when this callback runs. Every registered state is notified even if /// another notification throws; notification errors are logged and the original failure is preserved. - /// Idle shutdown completes normally. Cancellation during admitted validation or storage work is terminal. + /// Owner shutdown during initial recovery and idle shutdown complete through normal shutdown. + /// Cancellation during admitted validation or write/delete storage work is terminal. /// void OnFaulted(Exception exception) { } diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index b8ce48810ec..c24f9582592 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -194,6 +194,10 @@ private async Task WorkLoop() { await RecoverAsync(_shutdownCancellation.Token).ConfigureAwait(true); } + catch (OperationCanceledException) when (_shutdownCancellation.IsCancellationRequested) + { + return; + } catch (Exception exception) { Fence(exception); diff --git a/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs b/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs index dec84f68380..c887c0248d2 100644 --- a/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs @@ -602,6 +602,177 @@ public async Task InitializationFailure_NotifiesRegisteredStatesOnce() Assert.Equal(1, state.FaultCount); } + [Fact] + public async Task InitializationCallerCancellation_LeavesOwnedRecoveryRunning() + { + var storage = new MutableReadStorage(1, Array.Empty()); + await using var manager = CreateTestSystem(storage).Manager; + var state = new HookState(); + manager.RegisterStateMachine("state", state); + using var caller = new CancellationTokenSource(); + var canceledWaiter = manager.InitializeAsync(caller.Token).AsTask(); + await WaitFor(storage.BlockedReadStarted.Task); + var remainingWaiter = manager.InitializeAsync(CancellationToken.None).AsTask(); + caller.Cancel(); + await Assert.ThrowsAnyAsync(() => WaitFor(canceledWaiter)); + Assert.False(storage.ReadToken.IsCancellationRequested); + Assert.False(remainingWaiter.IsCompleted); + Assert.Equal(0, state.FaultCount); + Assert.Equal(0, state.RecoveryCompletedCount); + + storage.AllowBlockedRead.SetResult(); + await WaitFor(remainingWaiter); + Assert.Equal(1, state.RecoveryCompletedCount); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(["append"], storage.OperationLog); + Assert.Equal(1, state.WriteCompletedCount); + Assert.Null(state.Failure); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RecoveryShutdown_CancelsInitializationWithoutFaultNotification(bool lifecycleStop) + { + var storage = new MutableReadStorage(1, Array.Empty()); + var sut = CreateTestSystem(storage); + await using var manager = sut.Manager; + var first = new HookState(); + var second = new HookState(); + manager.RegisterStateMachine("first", first); + manager.RegisterStateMachine("second", second); + var startup = lifecycleStop + ? sut.Lifecycle.OnStart(CancellationToken.None) + : manager.InitializeAsync(CancellationToken.None).AsTask(); + await WaitFor(storage.BlockedReadStarted.Task); + var another = manager.InitializeAsync(CancellationToken.None).AsTask(); + Assert.False(startup.IsCompleted); + Assert.False(another.IsCompleted); + + await WaitFor(lifecycleStop + ? sut.Lifecycle.OnStop(TestContext.Current.CancellationToken) + : manager.DisposeAsync().AsTask()); + foreach (var waiter in new[] { startup, another }) + { + await Assert.ThrowsAnyAsync(() => WaitFor(waiter)); + Assert.True(waiter.IsCanceled); + } + + Assert.True(storage.ReadToken.IsCancellationRequested); + Assert.False(storage.AllowBlockedRead.Task.IsCompleted); + Assert.Empty(storage.OperationLog); + Assert.All(new[] { first, second }, state => + { + Assert.Equal(0, state.FaultCount); + Assert.Equal(0, state.RecoveryCompletedCount); + Assert.Equal(0, state.PendingValidationCount); + Assert.Equal(0, state.CaptureCount); + Assert.Equal(0, state.WriteCompletedCount); + Assert.Null(state.Failure); + }); + if (lifecycleStop) + { + await Assert.ThrowsAnyAsync(() => manager.WriteStateAsync(CancellationToken.None).AsTask()); + } + + await WaitFor(manager.DisposeAsync().AsTask()); + await Assert.ThrowsAsync(() => manager.InitializeAsync(CancellationToken.None).AsTask()); + } + + [Theory] + [InlineData("provider-cancellation")] + [InlineData("provider-io")] + [InlineData("replay")] + public async Task RecoveryFailure_FencesAndNotifiesAllStates(string failure) + { + var context = new QueuedSynchronizationContext(); + await context.Run(async () => + { + Exception expected = failure == "provider-cancellation" + ? new OperationCanceledException("Provider read cancellation.", new CancellationToken(canceled: true)) + : new IOException("Provider read failure."); + IJournalStorage storage = failure == "replay" + ? new RawReadStorage([1, 2, 3]) + : new CapturingStorage { NextReadException = expected }; + await using var manager = CreateTestSystem(storage).Manager; + var states = new[] { new HookState(), new HookState() }; + manager.RegisterStateMachine("first", states[0]); + manager.RegisterStateMachine("second", states[1]); + var waiters = new[] + { + manager.InitializeAsync(CancellationToken.None).AsTask(), + manager.InitializeAsync(CancellationToken.None).AsTask() + }; + var notified = new List(); + foreach (var state in states) + { + state.FaultAction = exception => + { + Assert.All(waiters, waiter => Assert.False(waiter.IsCompleted)); + notified.Add(exception); + }; + } + + var observed = await Record.ExceptionAsync(() => WaitFor(waiters[0])); + if (failure == "replay") + { + Assert.Contains("Failed to recover journaling state", Assert.IsType(observed).Message); + Assert.NotNull(observed.InnerException); + } + else + { + Assert.Same(expected, observed); + } + + Assert.Same(observed, await Record.ExceptionAsync(() => WaitFor(waiters[1]))); + Assert.Equal(2, notified.Count); + Assert.All(notified, exception => Assert.Same(observed, exception)); + Assert.All(states, state => + { + Assert.Same(observed, state.Failure); + Assert.Equal(1, state.FaultCount); + Assert.Equal(0, state.RecoveryCompletedCount); + }); + var rejected = await Assert.ThrowsAsync(() => manager.InitializeAsync(CancellationToken.None).AsTask()); + Assert.Same(observed, rejected.InnerException); + }); + } + + [Fact] + public async Task RecoveryIoFailure_DuringShutdownPreservesOriginalCause() + { + var expected = new IOException("Read failure during shutdown."); + var entered = NewSignal(); + var storage = Substitute.For(); + storage.ReadAsync(Arg.Any(), Arg.Any()) + .Returns(call => ReadAsync(call.Arg())); + await using var manager = CreateTestSystem(storage).Manager; + var state = new HookState(); + manager.RegisterStateMachine("state", state); + var initializing = manager.InitializeAsync(CancellationToken.None).AsTask(); + await WaitFor(entered.Task); + var another = manager.InitializeAsync(CancellationToken.None).AsTask(); + await WaitFor(manager.DisposeAsync().AsTask()); + Assert.Same(expected, await Record.ExceptionAsync(() => WaitFor(initializing))); + Assert.Same(expected, await Record.ExceptionAsync(() => WaitFor(another))); + Assert.Same(expected, state.Failure); + Assert.Equal(1, state.FaultCount); + Assert.Equal(0, state.RecoveryCompletedCount); + + async ValueTask ReadAsync(CancellationToken token) + { + entered.SetResult(); + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, token); + } + catch (OperationCanceledException) when (token.IsCancellationRequested) + { + throw expected; + } + } + } + [Fact] public async Task IdleShutdown_CompletesWithoutFaultNotification() { @@ -680,6 +851,36 @@ public async Task CallerCancellation_PreservesOwnedStorageAndAck(bool snapshot) Assert.Equal(snapshot ? 1 : 0, storage.Replaces.Count); } + [Fact] + public async Task AdmittedDeleteShutdownCancellation_NotifiesAndFaultsWaiters() + { + var storage = new BlockingDeleteStorage(); + var sut = CreateTestSystem(storage); + await using var manager = sut.Manager; + var state = new HookState(); + manager.RegisterStateMachine("state", state); + await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); + var deleting = manager.DeleteStateAsync(CancellationToken.None).AsTask(); + await WaitFor(storage.FirstDeleteStarted.Task); + var queued = manager.WriteStateAsync(CancellationToken.None).AsTask(); + var notified = false; + state.FaultAction = _ => + { + Assert.False(deleting.IsCompleted); + Assert.False(queued.IsCompleted); + notified = true; + }; + await WaitFor(sut.Lifecycle.OnStop(TestContext.Current.CancellationToken)); + var exception = await Assert.ThrowsAnyAsync(() => WaitFor(deleting)); + Assert.Same(exception, state.Failure); + Assert.Same(exception, await Record.ExceptionAsync(() => WaitFor(queued))); + Assert.True(notified); + Assert.Equal(1, state.FaultCount); + Assert.Equal(1, state.DeleteStartedCount); + Assert.Equal(1, state.ResetCount); + Assert.Equal(0, state.WriteCompletedCount); + } + [Fact] public async Task CallerCancellation_PreservesQueuedWrite() { @@ -765,6 +966,7 @@ private sealed class HookState : IStateMachine public int DeleteStartedCount { get; private set; } public int WriteCompletedCount { get; private set; } public int FaultCount { get; private set; } + public int RecoveryCompletedCount { get; private set; } public Exception? Failure { get; private set; } public void ValidatePendingChanges() { @@ -774,6 +976,7 @@ public void ValidatePendingChanges() public void ValidateWrite() => ValidateWriteAction?.Invoke(); public void ValidateDelete() => ValidateDeleteAction?.Invoke(); + public void OnRecoveryCompleted() => RecoveryCompletedCount++; public void OnDeleteStarted() { diff --git a/test/Orleans.Journaling.Tests/StateManagerTests.cs b/test/Orleans.Journaling.Tests/StateManagerTests.cs index a7c06840779..89cc65294c1 100644 --- a/test/Orleans.Journaling.Tests/StateManagerTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerTests.cs @@ -2330,9 +2330,12 @@ public byte[] Bytes public TaskCompletionSource AllowBlockedRead { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public CancellationToken ReadToken { get; private set; } + public async ValueTask ReadAsync(IJournalStorageConsumer consumer, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(consumer); + ReadToken = cancellationToken; cancellationToken.ThrowIfCancellationRequested(); if (Interlocked.Increment(ref _readCount) == _blockedReadNumber) { From a8bbce0c35da251b6e88634cf17099cf10691f36 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 21 Sep 2026 08:37:12 -0700 Subject: [PATCH 06/11] refactor(journaling): remove feature policy from state machines --- .../grains/journaling/runtime-behavior.md | 49 +- .../IJournaledStateManager.cs | 3 +- src/Orleans.Journaling/IStateMachine.cs | 59 +- .../JournaledStateManager.cs | 43 - .../Orleans.Journaling/Orleans.Journaling.cs | 5 - .../StateManagerLifecycleTests.cs | 766 ++++-------------- 6 files changed, 180 insertions(+), 745 deletions(-) diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index 0e650ccefcf..f1e6a6becd1 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -62,13 +62,11 @@ Concurrent calls made while the same kind of write is queued can share that queu ## Safe-to-commit staging All interleaved callers share the manager's pending journal. Prepare fallible work, external acknowledgements, -and proposed output in operation-local data. After establishing that an outcome is safe to commit, apply its -mutations to durable state and initiate a write. Coordinate that transition with other interleaved operations -which can affect the same decision. Any caller's write can include staged mutations from other calls. - -If an application error occurs after staging and makes those mutations unsafe to commit, end the activation's -use of the manager and request deactivation. In-flight methods can retain local decisions and references -across awaits; a fresh activation reconstructs both application and durable state together. +and proposed output in operation-local data. After the final preparation await, check the relevant +preconditions and apply the complete safe-to-commit update synchronously, then request an ordinary write. +Orleans executes that synchronous block on a single activation thread. Another grain turn can run when +the operation awaits, so keep shared state safe to commit at each await. Any caller's write can include +staged mutations from other calls. ## Consistency and competing writers @@ -114,32 +112,21 @@ owning manager's configured format, including before recovery of an empty journa codec resolution to that owner. Grain-bound managers resolve codecs from the activation's services; standalone owners use shared application services. - and -perform pure validation in the requesting caller's context. An admission rejection leaves the manager healthy. -Deletion validates all states again in serialized execution, then calls - on every state before awaiting storage deletion. -Successful deletion resets states before completing callers. - -Before each append or snapshot capture, the manager calls - on every registered state. -This pure synchronous check validates state-owned failure latches and pending-change invariants inside -the admitted operation's failure boundary. Every state passes before capture begins in the same work-loop -continuation, including writes which flush only committed entries or produce zero bytes. - -Features acquire asynchronous prerequisites before staging their changes, then apply prepared mutations -synchronously and request an ordinary write. Independent operation-local preparation can continue while -the manager captures previously staged valid changes. `ValidateWrite` remains caller-context admission -validation; a terminal state-local error belongs in `ValidatePendingChanges` so the manager fences even -when the reporting write is its first admitted operation. A previously captured write retains its actual -storage outcome and acknowledgement bookkeeping. - +States synchronously encode their pending changes through +or their current contents through . After storage acknowledges captured bytes, performs durable-completion bookkeeping. A zero-byte write completes without this callback. -An admitted validation, capture, or storage failure fences the manager, records the original -exception, and calls on every registered state before -faulting current and queued waiters. Notification failures are logged while the original failure remains -the operation's outcome. Owner-canceled initial recovery and idle shutdown complete through normal shutdown. -Cancellation during admitted validation or write/delete storage work is terminal. + +The journal owner keeps feature operations quiescent through deletion's storage and reset outcome, +including when a caller cancels its wait. Successful deletion calls +before completing deletion waiters. + +The manager records the first capture or storage failure, fences further persistence, faults current +and queued manager waiters, and requests grain deactivation. Features observe their write failures and +complete their own operation waiters and resource cleanup through their operation and lifecycle ownership. +Standalone callers own that cleanup explicitly. A previously captured write retains its actual storage +outcome and acknowledgement bookkeeping. Owner-canceled initial recovery and idle shutdown complete +through normal shutdown; admitted write/delete storage cancellation remains terminal. ## Compaction diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index af08e64a0e3..cf80ccd29ba 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -74,7 +74,8 @@ TCodec GetRequiredCommandCodec() where TCodec : notnull /// Resets this instance, removing any persistent state. /// /// - /// Quiesce other operations before deleting state: deletion resets every registered state machine. + /// The caller keeps other operations quiescent through completion: deletion resets every registered state machine. + /// Cancellation ends the caller's wait; an already queued deletion continues to its storage and reset outcome. /// A failed deletion permanently fences the manager and requests deactivation of its owning grain. /// /// The cancellation token. diff --git a/src/Orleans.Journaling/IStateMachine.cs b/src/Orleans.Journaling/IStateMachine.cs index f6ecb826822..39f824a6db7 100644 --- a/src/Orleans.Journaling/IStateMachine.cs +++ b/src/Orleans.Journaling/IStateMachine.cs @@ -14,8 +14,7 @@ namespace Orleans.Journaling; /// both apply the mutation locally and emit the corresponding command to the journal). /// /// -/// When the application requests a write, the journaled state manager validates the request, -/// validates the pending changes of all states, and calls +/// When the application requests a write, the journaled state manager calls /// (and occasionally ) to materialize /// the pending changes, then flushes the journal to durable storage. /// @@ -68,62 +67,6 @@ public interface IStateMachine /// void OnRecoveryCompleted() { } - /// - /// Validates this state's pending changes immediately before an admitted write captures journal entries or a snapshot. - /// The default implementation accepts the pending changes. - /// - /// - /// Validation is pure and synchronous. All registered states pass validation before any state is captured, - /// including writes which only flush committed entries or produce zero bytes. Validation and capture run - /// in the same work-loop continuation. Throwing reports a terminal state-local failure and fences the manager. - /// Callers acquire asynchronous prerequisites before staging changes. Independent operation-local preparation - /// can proceed while previously staged valid changes are captured. - /// - void ValidatePendingChanges() { } - - /// - /// Validates a write request in the public caller's context before it is queued. - /// The default implementation accepts the request. - /// - /// - /// Validation is pure and runs only at request admission. Throwing rejects this request and leaves the manager healthy. - /// Use to report a terminal state-local failure inside admitted execution. - /// - void ValidateWrite() { } - - /// - /// Validates deletion at public request admission and again during serialized execution. - /// The default implementation accepts deletion. - /// - /// - /// Validation is pure. An admission failure rejects the request and leaves the manager healthy. - /// An execution-time failure fences the manager. All states pass execution-time validation - /// before the manager calls on any state. - /// - void ValidateDelete() { } - - /// - /// Notifies the state that deletion is starting, after all execution-time validation succeeds - /// and before the storage operation begins. The default implementation performs no action. - /// - /// - /// A successful storage deletion is followed by before deletion waiters complete. - /// - void OnDeleteStarted() { } - - /// - /// Notifies the state of the manager's first terminal failure, before current and queued operation waiters fault. - /// The default implementation performs no action. - /// - /// The original failure recorded by the manager. - /// - /// The manager is already fenced when this callback runs. Every registered state is notified even if - /// another notification throws; notification errors are logged and the original failure is preserved. - /// Owner shutdown during initial recovery and idle shutdown complete through normal shutdown. - /// Cancellation during admitted validation or write/delete storage work is terminal. - /// - void OnFaulted(Exception exception) { } - /// /// Writes pending state changes to the journal. /// diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index c24f9582592..676dfe4ef5e 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -260,11 +260,6 @@ private async Task WorkLoop() case AppendJournalWorkItem: case WriteSnapshotWorkItem: { - foreach (var state in _states.Values) - { - state.ValidatePendingChanges(); - } - // TODO: decide whether it's best to snapshot or append. Eg, by summing the size of the most recent snapshots and the current journal length. // If the current journal length is greater than the snapshot size, then take a snapshot instead of appending more journal entries. var isSnapshot = workItem is WriteSnapshotWorkItem @@ -459,16 +454,6 @@ private async Task WorkLoop() case DeleteStateWorkItem: { - foreach (var state in _states.Values) - { - state.ValidateDelete(); - } - - foreach (var state in _states.Values) - { - state.OnDeleteStarted(); - } - // Clear storage. await DeleteStorageAsync(_shutdownCancellation.Token).ConfigureAwait(true); @@ -597,19 +582,6 @@ private void Fence(Exception exception) try { - // Fencing prevents registration, so callbacks can use the stable registry outside the lock. - foreach (var (name, state) in _states) - { - try - { - state.OnFaulted(exception); - } - catch (Exception notificationException) - { - LogErrorNotifyingFaultedState(_shared.Logger, notificationException, name); - } - } - if (!_shutdownCancellation.IsCancellationRequested) { LogErrorProcessingWorkItems(_shared.Logger, exception); @@ -704,11 +676,6 @@ public async ValueTask DeleteStateAsync(CancellationToken cancellationToken = de lock (_lock) { ThrowIfStateOperationsUnavailable(); - foreach (var state in _states.Values) - { - state.ValidateDelete(); - } - task = EnqueueOrGetPendingWorkItem(out didEnqueue); } @@ -898,11 +865,6 @@ public async ValueTask WriteStateAsync(CancellationToken cancellationToken = def lock (_lock) { ThrowIfStateOperationsUnavailable(); - foreach (var state in _states.Values) - { - state.ValidateWrite(); - } - var isSnapshot = _migrationSnapshotRequired || _storage.IsCompactionRequested; operation = isSnapshot ? JournalingInstruments.OperationSnapshot : JournalingInstruments.OperationAppend; pendingWrite = isSnapshot @@ -1361,11 +1323,6 @@ void IStateMachine.WritePendingEntries(JournalStreamWriter writer) { } Message = "Error processing work items.")] private static partial void LogErrorProcessingWorkItems(ILogger logger, Exception exception); - [LoggerMessage( - Level = LogLevel.Error, - Message = "Error notifying journaled state \"{Name}\" of a terminal failure.")] - private static partial void LogErrorNotifyingFaultedState(ILogger logger, Exception exception, string name); - [LoggerMessage( Level = LogLevel.Information, Message = "State \"{Name}\" was not found. I have substituted a placeholder for graceful time-based retirement.")] diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index c2cb5bd35d6..0bf70416d8d 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -306,15 +306,10 @@ public partial interface IPreservedJournalEntry public partial interface IStateMachine { - void OnDeleteStarted(); - void OnFaulted(System.Exception exception); void OnRecoveryCompleted(); void OnWriteCompleted(); void ReplayEntry(JournalEntry entry, JournalReplayContext context); void Reset(JournalStreamWriter writer); - void ValidateDelete(); - void ValidatePendingChanges(); - void ValidateWrite(); void WritePendingEntries(JournalStreamWriter writer); void WriteSnapshot(JournalStreamWriter writer); } diff --git a/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs b/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs index c887c0248d2..1888be3fdd6 100644 --- a/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs @@ -1,4 +1,5 @@ using System.Threading.Channels; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using NSubstitute; @@ -9,597 +10,215 @@ namespace Orleans.Journaling.Tests; public partial class StateManagerTests { [Fact] - public async Task StateHooks_DefaultsAcceptExistingStates() + public async Task StateMachineProtocol_WritesAndResetsAfterDeletion() { - IStateMachine state = new AlwaysWritingState(); - state.ValidatePendingChanges(); - state.ValidateWrite(); - state.ValidateDelete(); - state.OnDeleteStarted(); - state.OnFaulted(new IOException("Default notification.")); - var storage = new CapturingStorage(); await using var manager = CreateTestSystem(storage).Manager; + var state = new LifecycleState(); manager.RegisterStateMachine("state", state); await manager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, state.ResetCount); + Assert.Equal(1, state.RecoveryCompletedCount); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); - await manager.DeleteStateAsync(TestContext.Current.CancellationToken); Assert.Single(storage.Appends); + Assert.Equal(1, state.CaptureCount); + Assert.Equal(1, state.WriteCompletedCount); + + await manager.DeleteStateAsync(TestContext.Current.CancellationToken); Assert.Equal(1, storage.DeleteCount); + Assert.Equal(2, state.ResetCount); + Assert.Equal(1, state.WriteCompletedCount); } [Theory] [InlineData(false)] [InlineData(true)] - public async Task AdmissionVeto_UsesCallerContextAndLeavesManagerHealthy(bool delete) - { - var storage = new CapturingStorage(); - await using var manager = CreateTestSystem(storage).Manager; - var state = new HookState(); - manager.RegisterStateMachine("state", state); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - var callerContext = new AsyncLocal { Value = "request" }; - var insideAdmission = true; - var expected = new InvalidOperationException("Request rejected."); - var calls = 0; - void Validate() - { - Assert.True(insideAdmission); - Assert.Equal("request", callerContext.Value); - calls++; - throw expected; - } - - if (delete) state.ValidateDeleteAction = Validate; - else state.ValidateWriteAction = Validate; - var request = delete - ? manager.DeleteStateAsync(TestContext.Current.CancellationToken) - : manager.WriteStateAsync(TestContext.Current.CancellationToken); - insideAdmission = false; - Assert.Equal(1, calls); - Assert.Same(expected, await Record.ExceptionAsync(() => request.AsTask())); - Assert.Null(state.Failure); - Assert.Equal(0, state.DeleteStartedCount); - Assert.Equal(1, state.ResetCount); - Assert.Empty(storage.Appends); - Assert.Equal(0, storage.DeleteCount); - - state.ValidateWriteAction = null; - state.ValidateDeleteAction = null; - await manager.WriteStateAsync(TestContext.Current.CancellationToken); - await manager.DeleteStateAsync(TestContext.Current.CancellationToken); - Assert.Single(storage.Appends); - Assert.Equal(1, storage.DeleteCount); - Assert.Null(state.Failure); - } - - [Fact] - public async Task Delete_RechecksAllValidationBeforeAnyStart() - { - var storage = new CapturingStorage { BlockNextAppend = true }; - await using var manager = CreateTestSystem(storage).Manager; - var first = new HookState(); - var second = new HookState(); - manager.RegisterStateMachine("first", first); - manager.RegisterStateMachine("second", second); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - await WaitFor(storage.BlockedAppendStarted.Task); - - var validations = new List(); - first.ValidateDeleteAction = () => validations.Add("first"); - var reject = false; - var expected = new InvalidOperationException("Deletion became unsafe."); - second.ValidateDeleteAction = () => - { - validations.Add("second"); - if (reject) throw expected; - }; - var delete = manager.DeleteStateAsync(TestContext.Current.CancellationToken).AsTask(); - Assert.Equal(["first", "second"], validations); - reject = true; - storage.ReleaseAppend.SetResult(); - await WaitFor(write); - Assert.Same(expected, await Record.ExceptionAsync(() => WaitFor(delete))); - Assert.Equal(["first", "second", "first", "second"], validations); - Assert.Equal(0, first.DeleteStartedCount); - Assert.Equal(0, second.DeleteStartedCount); - Assert.Equal(0, storage.DeleteCount); - Assert.Same(expected, first.Failure); - Assert.Same(expected, second.Failure); - } - - [Fact] - public async Task Delete_StartsAfterAllValidationAndResetsBeforeWaiter() + public async Task Delete_ResetsStatesBeforeCompletion(bool cancelWait) { var storage = new BlockingDeleteStorage(); await using var manager = CreateTestSystem(storage).Manager; - var events = new List(); - var first = new HookState(); - var second = new HookState(); + var first = new LifecycleState(); + var second = new LifecycleState(); manager.RegisterStateMachine("first", first); manager.RegisterStateMachine("second", second); await manager.InitializeAsync(TestContext.Current.CancellationToken); - first.ValidateDeleteAction = () => events.Add("validate first"); - second.ValidateDeleteAction = () => events.Add("validate second"); - first.DeleteStartedAction = () => events.Add("start first"); - second.DeleteStartedAction = () => events.Add("start second"); - first.ResetAction = () => events.Add("reset first"); - second.ResetAction = () => events.Add("reset second"); - - var delete = manager.DeleteStateAsync(TestContext.Current.CancellationToken).AsTask(); + using var caller = new CancellationTokenSource(); + var deletion = manager.DeleteStateAsync(caller.Token).AsTask(); await WaitFor(storage.FirstDeleteStarted.Task); - Assert.Equal( - ["validate first", "validate second", "validate first", "validate second", "start first", "start second"], - events); Assert.Equal(1, first.ResetCount); Assert.Equal(1, second.ResetCount); - Assert.False(delete.IsCompleted); - storage.AllowFirstDelete.SetResult(); - await WaitFor(delete); - Assert.Equal(["reset first", "reset second"], events.TakeLast(2)); - Assert.Equal(2, first.ResetCount); - Assert.Equal(2, second.ResetCount); - } - - [Theory] - [InlineData("abc", false)] - [InlineData("acb", false)] - [InlineData("bac", false)] - [InlineData("bca", false)] - [InlineData("cab", false)] - [InlineData("cba", false)] - [InlineData("abc", true)] - [InlineData("acb", true)] - [InlineData("bac", true)] - [InlineData("bca", true)] - [InlineData("cab", true)] - [InlineData("cba", true)] - public async Task PendingValidation_ValidatesAllStatesBeforeAnyCapture(string order, bool snapshot) - { - var storage = new CapturingStorage { IsCompactionRequested = snapshot }; - var format = new TrackingJournalFormat(SessionPool); - await using var manager = CreateTestSystem(storage, journalFormat: format).Manager; - var states = new Dictionary { ['a'] = new(), ['b'] = new(), ['c'] = new() }; - var events = new List(); - foreach (var key in order) + Assert.False(deletion.IsCompleted); + if (cancelWait) { - var state = states[key]; - manager.RegisterStateMachine(key.ToString(), state); - state.CaptureAction = () => - { - Assert.All(states.Values, value => Assert.Equal(1, value.PendingValidationCount)); - events.Add($"capture {key}"); - }; + caller.Cancel(); + await Assert.ThrowsAnyAsync(() => deletion); } - await manager.InitializeAsync(TestContext.Current.CancellationToken); - var initialWriter = Assert.Single(format.Writers); - var initialEntries = initialWriter.BeganEntryIds.Count; - foreach (var key in order) + var resets = new List(); + var resetCompleted = NewSignal(); + first.ResetAction = () => { - states[key].PendingValidationAction = () => - { - Assert.Same(initialWriter, Assert.Single(format.Writers)); - Assert.Equal(initialEntries, initialWriter.BeganEntryIds.Count); - Assert.All(states.Values, value => Assert.Equal(0, value.CaptureCount)); - events.Add($"validate {key}"); - }; - } - - await manager.WriteStateAsync(TestContext.Current.CancellationToken); - - Assert.All(states.Values, state => + if (!cancelWait) Assert.False(deletion.IsCompleted); + resets.Add("first"); + }; + second.ResetAction = () => { - Assert.Equal(1, state.PendingValidationCount); - Assert.Equal(1, state.CaptureCount); - Assert.Equal(1, state.WriteCompletedCount); - }); - Assert.Equal(order.Select(key => $"validate {key}").Concat(order.Select(key => $"capture {key}")), events); - Assert.Equal(snapshot ? 1 : 0, storage.Replaces.Count); - Assert.Equal(snapshot ? 0 : 1, storage.Appends.Count); + if (!cancelWait) Assert.False(deletion.IsCompleted); + resets.Add("second"); + resetCompleted.SetResult(); + }; + storage.AllowFirstDelete.SetResult(); + await WaitFor(resetCompleted.Task); + if (!cancelWait) await WaitFor(deletion); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(["first", "second"], resets); + Assert.Equal(2, first.ResetCount); + Assert.Equal(2, second.ResetCount); + Assert.Equal(1, storage.DeleteCount); } - [Fact] - public async Task PendingValidation_AndCaptureShareSchedulerTurn() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task OperationLocalPreparation_ThenSynchronousUpdatesPersistTogether(bool snapshot) { var context = new QueuedSynchronizationContext(); await context.Run(async () => { - await using var manager = CreateTestSystem().Manager; - var state = new HookState(); - var validationTurn = -1; - state.PendingValidationAction = () => validationTurn = context.Turn; - state.CaptureAction = () => Assert.Equal(validationTurn, context.Turn); - manager.RegisterStateMachine("state", state); + var storage = new CapturingStorage { IsCompactionRequested = snapshot }; + await using var manager = CreateTestSystem(storage).Manager; + var dictionary = new DurableDictionary("items", manager, CreateDictionaryCodec()); + var total = new DurableValue("total", manager, CreateValueCodec()); await manager.InitializeAsync(TestContext.Current.CancellationToken); + dictionary.Add("existing", 1); + total.Value = 1; + + var preparing = NewSignal(); + var release = NewSignal(); + var update = PrepareAndApply(); + await WaitFor(preparing.Task); await manager.WriteStateAsync(TestContext.Current.CancellationToken); - Assert.Equal(1, state.PendingValidationCount); - Assert.Equal(1, state.CaptureCount); - Assert.Equal(1, state.WriteCompletedCount); + Assert.False(update.IsCompleted); + Assert.Single(dictionary); + Assert.Equal(1, total.Value); + await AssertRecovered(1); + + release.SetResult(); + await WaitFor(update); + Assert.Equal(2, dictionary.Count); + Assert.Equal(43, total.Value); + await AssertRecovered(43); + + async Task PrepareAndApply() + { + var proposed = (Key: "prepared", Value: 42); + preparing.SetResult(); + await WaitFor(release.Task); + Assert.False(dictionary.ContainsKey(proposed.Key)); + dictionary.Add(proposed.Key, proposed.Value); + total.Value += proposed.Value; + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + } + + async Task AssertRecovered(int expectedTotal) + { + await using var recovered = CreateTestSystem(storage).Manager; + var recoveredItems = new DurableDictionary("items", recovered, CreateDictionaryCodec()); + var recoveredTotal = new DurableValue("total", recovered, CreateValueCodec()); + await recovered.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(expectedTotal, recoveredTotal.Value); + Assert.Equal(expectedTotal == 1 ? 1 : 2, recoveredItems.Count); + Assert.Equal(1, recoveredItems["existing"]); + if (expectedTotal != 1) Assert.Equal(42, recoveredItems["prepared"]); + } }); } [Fact] - public async Task ZeroByteWrite_ValidatesPendingChangesWithoutWriteCompleted() + public async Task ZeroByteWrite_DoesNotRepeatAcknowledgement() { var storage = new CapturingStorage(); await using var manager = CreateTestSystem(storage).Manager; - var state = new HookState { EmitEntry = false }; + var state = new LifecycleState { EmitEntry = false }; manager.RegisterStateMachine("state", state); await manager.InitializeAsync(TestContext.Current.CancellationToken); await manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Equal(0, manager.PendingWriteByteCount); Assert.Equal(1, state.WriteCompletedCount); await manager.WriteStateAsync(TestContext.Current.CancellationToken); - Assert.Equal(2, state.PendingValidationCount); Assert.Equal(2, state.CaptureCount); Assert.Equal(1, state.WriteCompletedCount); Assert.Single(storage.Appends); } - [Theory] - [InlineData("append", false, false)] - [InlineData("append", false, true)] - [InlineData("append", true, false)] - [InlineData("append", true, true)] - [InlineData("snapshot", false, false)] - [InlineData("snapshot", false, true)] - [InlineData("snapshot", true, false)] - [InlineData("snapshot", true, true)] - [InlineData("empty-buffer", false, false)] - [InlineData("empty-buffer", false, true)] - [InlineData("empty-buffer", true, false)] - [InlineData("empty-buffer", true, true)] - public async Task PartialApplyFailure_FencesBeforeCapture(string path, bool previouslyAdmitted, bool stateFirst) - { - var context = new QueuedSynchronizationContext(); - await context.Run(async () => - { - var storage = new CapturingStorage(); - var format = new TrackingJournalFormat(SessionPool); - await using var manager = CreateTestSystem(storage, journalFormat: format).Manager; - var state = new HookState { EmitEntry = false }; - var other = new HookState { EmitEntry = false }; - if (stateFirst) manager.RegisterStateMachine("state", state); - var value = new DurableValue("business", manager, CreateValueCodec()); - manager.RegisterStateMachine("other", other); - if (!stateFirst) manager.RegisterStateMachine("state", state); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - if (path == "empty-buffer") - { - await manager.WriteStateAsync(TestContext.Current.CancellationToken); - Assert.Equal(0, manager.PendingWriteByteCount); - } - - storage.IsCompactionRequested = path == "snapshot"; - var previousWrites = storage.Appends.Count; - var previousCaptures = state.CaptureCount; - var previousAcks = state.WriteCompletedCount; - var initialWriter = Assert.Single(format.Writers); - var initialEntries = initialWriter.BeganEntryIds.Count; - var waiters = new List(); - var expected = new InvalidOperationException("Synchronous apply failed after changing business state."); - Exception? latchedFailure = null; - var handlerContext = new AsyncLocal(); - var admissions = 0; - state.ValidateWriteAction = () => - { - Assert.False(handlerContext.Value); - admissions++; - }; - state.PendingValidationAction = () => - { - Assert.Same(initialWriter, Assert.Single(format.Writers)); - Assert.Equal(initialEntries, initialWriter.BeganEntryIds.Count); - Assert.Same(expected, latchedFailure); - throw latchedFailure!; - }; - if (previouslyAdmitted) waiters.Add(manager.WriteStateAsync(CancellationToken.None).AsTask()); - handlerContext.Value = true; - try - { - value.Value = 42; - throw expected; - } - catch (InvalidOperationException exception) - { - latchedFailure = exception; - } - finally - { - handlerContext.Value = false; - } - - if (path == "empty-buffer") Assert.Equal(0, manager.PendingWriteByteCount); - waiters.Add(manager.WriteStateAsync(CancellationToken.None).AsTask()); - waiters.Add(manager.DeleteStateAsync(CancellationToken.None).AsTask()); - waiters.Add(manager.InitializeAsync(CancellationToken.None).AsTask()); - var notifications = new List(); - var stateOwnedWaiter = NewSignal(); - state.FaultAction = exception => - { - Assert.Same(expected, exception); - Assert.All(waiters, waiter => Assert.False(waiter.IsCompleted)); - stateOwnedWaiter.TrySetException(exception); - notifications.Add("state"); - }; - other.FaultAction = exception => - { - Assert.Same(expected, exception); - Assert.All(waiters, waiter => Assert.False(waiter.IsCompleted)); - notifications.Add("other"); - }; - Assert.Equal(previouslyAdmitted ? 2 : 1, admissions); - foreach (var waiter in waiters) - { - Assert.Same(expected, await Record.ExceptionAsync(() => WaitFor(waiter))); - } - - Assert.Same(expected, await Record.ExceptionAsync(() => WaitFor(stateOwnedWaiter.Task))); - Assert.Equal(stateFirst ? new[] { "state", "other" } : ["other", "state"], notifications); - Assert.Equal(42, value.Value); - Assert.Equal(previousCaptures, state.CaptureCount); - Assert.Equal(previousAcks, state.WriteCompletedCount); - Assert.Equal(previousWrites, storage.Appends.Count); - Assert.Empty(storage.Replaces); - Assert.Equal(0, storage.DeleteCount); - var rejected = await Assert.ThrowsAsync(() => manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask()); - Assert.Same(expected, rejected.InnerException); - }); - } - [Fact] - public async Task PendingValidation_AllowsIndependentWriteDuringLocalPreparation() + public async Task LaterStorageFailure_PreservesAlreadyCapturedWriteAcknowledgement() { - var context = new QueuedSynchronizationContext(); - await context.Run(async () => - { - var storage = new CapturingStorage(); - await using var manager = CreateTestSystem(storage).Manager; - var state = new HookState(); - manager.RegisterStateMachine("state", state); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - var handlerContext = new AsyncLocal(); - var preparing = false; - var admissionCalls = 0; - var rejected = new InvalidOperationException("Handler preparation cannot commit state."); - state.ValidateWriteAction = () => - { - admissionCalls++; - if (handlerContext.Value) throw rejected; - }; - state.PendingValidationAction = () => - { - Assert.True(preparing); - Assert.False(handlerContext.Value); - Assert.Equal(2, admissionCalls); - }; - - var independent = manager.WriteStateAsync(CancellationToken.None).AsTask(); - preparing = true; - handlerContext.Value = true; - Assert.Same(rejected, await Record.ExceptionAsync(() => manager.WriteStateAsync(CancellationToken.None).AsTask())); - await independent; - Assert.Equal(1, state.PendingValidationCount); - Assert.Equal(1, state.WriteCompletedCount); - Assert.Single(storage.Appends); - Assert.Null(state.Failure); - - handlerContext.Value = false; - preparing = false; - state.PendingValidationAction = null; - await manager.WriteStateAsync(CancellationToken.None); - Assert.Equal(3, admissionCalls); - Assert.Equal(2, state.WriteCompletedCount); - Assert.Null(state.Failure); - }); - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task CommittedOnlyWrite_ChecksLatchedStateFailure(bool emptyPrefix) - { - var storage = new CapturingStorage(); + var storage = new CapturingStorage { BlockNextAppend = true }; await using var manager = CreateTestSystem(storage).Manager; - var state = new HookState(); + var state = new LifecycleState(); manager.RegisterStateMachine("state", state); await manager.InitializeAsync(TestContext.Current.CancellationToken); - if (emptyPrefix) - { - await manager.WriteStateAsync(TestContext.Current.CancellationToken); - Assert.Equal(0, manager.PendingWriteByteCount); - } - - var previousCaptures = state.CaptureCount; - var previousWrites = storage.Appends.Count; - var expected = new IOException("Latched state failure."); - state.PendingValidationAction = () => throw expected; - var write = StartWhileEntryIsOpen(); - Assert.Same(expected, await Record.ExceptionAsync(() => WaitFor(write))); - Assert.Same(expected, state.Failure); - Assert.Equal(previousCaptures, state.CaptureCount); - Assert.Equal(previousWrites, storage.Appends.Count); - - Task StartWhileEntryIsOpen() - { - using var entry = state.Writer.BeginEntry(); - var result = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - using var completed = new ManualResetEventSlim(); - result.GetAwaiter().OnCompleted(completed.Set); - Assert.True(completed.Wait(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken), - "The committed-only write must validate pending changes while the lexical entry is open."); - return result; - } - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task Fault_NotifiesEveryStateBeforeWaitersAndPreservesOriginal(bool failValidation) - { - var context = new QueuedSynchronizationContext(); - await context.Run(async () => - { - var expected = new IOException("Original failure."); - var notificationFailure = new InvalidOperationException("Notification failed."); - var storage = new CapturingStorage { BlockNextAppend = !failValidation, NextAppendException = expected }; - var logger = Substitute.For(); - logger.IsEnabled(Arg.Any()).Returns(true); - var loggerFactory = Substitute.For(); - loggerFactory.CreateLogger(Arg.Any()).Returns(logger); - var shared = new JournaledStateManagerShared(new Logger(loggerFactory), - Options.Create(ManagerOptions), TimeProvider.System, ServiceProvider); - await using var manager = new JournaledStateManager(shared, storage); - var first = new HookState(); - var second = new HookState(); - manager.RegisterStateMachine("first", first); - manager.RegisterStateMachine("second", second); - if (failValidation) first.PendingValidationAction = () => throw expected; - await manager.InitializeAsync(TestContext.Current.CancellationToken); - var current = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - if (!failValidation) await WaitFor(storage.BlockedAppendStarted.Task); - var queued = new[] - { - current, - manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(), - manager.DeleteStateAsync(TestContext.Current.CancellationToken).AsTask(), - manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask() - }; - var notified = new List(); - first.FaultAction = exception => - { - Assert.Same(expected, exception); - Assert.All(queued, task => Assert.False(task.IsCompleted)); - notified.Add("first"); - throw notificationFailure; - }; - second.FaultAction = exception => - { - Assert.Same(expected, exception); - Assert.All(queued, task => Assert.False(task.IsCompleted)); - var rejected = Assert.Throws(() => manager.RegisterStateMachine("late", new HookState())); - Assert.Same(expected, rejected.InnerException); - notified.Add("second"); - }; - - if (!failValidation) storage.ReleaseAppend.SetResult(); - foreach (var task in queued) - { - Assert.Same(expected, await Record.ExceptionAsync(() => WaitFor(task))); - } - - Assert.Equal(["first", "second"], notified); - Assert.Equal(1, first.FaultCount); - Assert.Equal(1, second.FaultCount); - Assert.Contains(logger.ReceivedCalls(), call => - call.GetMethodInfo().Name == nameof(ILogger.Log) - && Equals(call.GetArguments()[0], LogLevel.Error) - && ReferenceEquals(call.GetArguments()[3], notificationFailure)); - var late = await Assert.ThrowsAsync(() => manager.WriteStateAsync(CancellationToken.None).AsTask()); - Assert.Same(expected, late.InnerException); - Assert.Empty(storage.Appends); - }); + var capturedWrite = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + await WaitFor(storage.BlockedAppendStarted.Task); + var expected = new IOException("Failure of the next append."); + storage.NextAppendException = expected; + var failingWrite = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + Assert.Equal(0, state.WriteCompletedCount); + storage.ReleaseAppend.SetResult(); + await WaitFor(capturedWrite); + Assert.Same(expected, await Record.ExceptionAsync(() => WaitFor(failingWrite))); + Assert.Equal(2, state.CaptureCount); + Assert.Equal(1, state.WriteCompletedCount); + Assert.Single(storage.Appends); + var rejected = await Assert.ThrowsAsync(() => manager.WriteStateAsync(CancellationToken.None).AsTask()); + Assert.Same(expected, rejected.InnerException); } [Fact] - public async Task Fault_CallbackCanCoordinateCrossThreadManagerReentry() + public async Task DeactivationFailure_PreservesOriginalStorageFailure() { var expected = new IOException("Original storage failure."); + var secondary = new InvalidOperationException("Deactivation failed."); var storage = new CapturingStorage { BlockNextAppend = true, NextAppendException = expected }; - await using var manager = CreateTestSystem(storage).Manager; - var first = new HookState(); - var second = new HookState(); - manager.RegisterStateMachine("first", first); - manager.RegisterStateMachine("second", second); + var provider = Substitute.For(); + var context = Substitute.For(); + context.GrainId.Returns(GrainId.Create("test-grain", "deactivation-failure")); + context.ActivationServices.Returns(ServiceProvider); + provider.CreateStorage(JournalId.FromGrainId(context.GrainId)).Returns(storage); + context.When(value => value.Deactivate(Arg.Any(), Arg.Any())) + .Do(_ => throw secondary); + var shared = new JournaledStateManagerShared( + ServiceProvider.GetRequiredService>(), + Options.Create(ManagerOptions), TimeProvider.System, ServiceProvider); + await using var manager = new JournaledStateManager(shared, provider, context); + manager.RegisterStateMachine("state", new LifecycleState()); await manager.InitializeAsync(TestContext.Current.CancellationToken); - var current = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); + var current = manager.WriteStateAsync(CancellationToken.None).AsTask(); await WaitFor(storage.BlockedAppendStarted.Task); - var queued = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - var callbackEntered = NewSignal(); - var workerReady = NewSignal(); - var worker = Task.Run(async () => - { - workerReady.SetResult(); - await WaitFor(callbackEntered.Task); - return (ThreadId: Environment.CurrentManagedThreadId, - Failure: Record.Exception(() => manager.RegisterStateMachine("late", new HookState()))); - }, TestContext.Current.CancellationToken); - await WaitFor(workerReady.Task); - Exception? callbackError = null; - var notifications = new List(); - first.FaultAction = exception => - { - callbackError = Record.Exception(() => - { - Assert.Same(expected, exception); - callbackEntered.SetResult(); - var result = worker.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken).GetAwaiter().GetResult(); - Assert.NotEqual(Environment.CurrentManagedThreadId, result.ThreadId); - Assert.Same(expected, Assert.IsType(result.Failure).InnerException); - Assert.False(current.IsCompleted); - Assert.False(queued.IsCompleted); - notifications.Add("first"); - }); - }; - second.FaultAction = exception => - { - Assert.Same(expected, exception); - Assert.False(current.IsCompleted); - Assert.False(queued.IsCompleted); - notifications.Add("second"); - }; - + var queued = manager.WriteStateAsync(CancellationToken.None).AsTask(); storage.ReleaseAppend.SetResult(); Assert.Same(expected, await Record.ExceptionAsync(() => WaitFor(current))); Assert.Same(expected, await Record.ExceptionAsync(() => WaitFor(queued))); - await WaitFor(worker); - Assert.Null(callbackError); - Assert.Equal(["first", "second"], notifications); - Assert.Equal(1, first.FaultCount); - Assert.Equal(1, second.FaultCount); - var lookup = Assert.Throws(() => manager.TryGetStateMachine("late", out _)); - Assert.Same(expected, lookup.InnerException); - } - - [Fact] - public async Task LatchedFailure_PreservesAlreadyCapturedWriteAcknowledgement() - { - var storage = new CapturingStorage { BlockNextAppend = true }; - await using var manager = CreateTestSystem(storage).Manager; - var state = new HookState(); - manager.RegisterStateMachine("state", state); - await manager.InitializeAsync(TestContext.Current.CancellationToken); - var capturedWrite = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - await WaitFor(storage.BlockedAppendStarted.Task); - var expected = new IOException("Failure after capture."); - state.PendingValidationAction = () => throw expected; - var failingWrite = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); - Assert.Null(state.Failure); - Assert.Equal(0, state.WriteCompletedCount); - storage.ReleaseAppend.SetResult(); - await WaitFor(capturedWrite); - Assert.Same(expected, await Record.ExceptionAsync(() => WaitFor(failingWrite))); - Assert.Equal(1, state.CaptureCount); - Assert.Equal(1, state.WriteCompletedCount); - Assert.Same(expected, state.Failure); - Assert.Single(storage.Appends); + var late = await Assert.ThrowsAsync(() => manager.InitializeAsync(CancellationToken.None).AsTask()); + Assert.Same(expected, late.InnerException); + context.Received(1).Deactivate( + Arg.Is(reason => ReferenceEquals(reason.Exception, expected)), Arg.Any()); } [Fact] - public async Task InitializationFailure_NotifiesRegisteredStatesOnce() + public async Task InitializationFailure_PreservesOriginalCause() { var expected = new IOException("Recovery failed."); var storage = new CapturingStorage { NextReadException = expected }; await using var manager = CreateTestSystem(storage).Manager; - var state = new HookState(); + var state = new LifecycleState(); manager.RegisterStateMachine("state", state); Assert.Same(expected, await Record.ExceptionAsync(() => manager.InitializeAsync(CancellationToken.None).AsTask())); - Assert.Same(expected, state.Failure); var retry = await Assert.ThrowsAsync(() => manager.InitializeAsync(CancellationToken.None).AsTask()); Assert.Same(expected, retry.InnerException); - Assert.Equal(1, state.FaultCount); + Assert.Equal(0, state.RecoveryCompletedCount); } [Fact] @@ -607,7 +226,7 @@ public async Task InitializationCallerCancellation_LeavesOwnedRecoveryRunning() { var storage = new MutableReadStorage(1, Array.Empty()); await using var manager = CreateTestSystem(storage).Manager; - var state = new HookState(); + var state = new LifecycleState(); manager.RegisterStateMachine("state", state); using var caller = new CancellationTokenSource(); var canceledWaiter = manager.InitializeAsync(caller.Token).AsTask(); @@ -617,7 +236,6 @@ public async Task InitializationCallerCancellation_LeavesOwnedRecoveryRunning() await Assert.ThrowsAnyAsync(() => WaitFor(canceledWaiter)); Assert.False(storage.ReadToken.IsCancellationRequested); Assert.False(remainingWaiter.IsCompleted); - Assert.Equal(0, state.FaultCount); Assert.Equal(0, state.RecoveryCompletedCount); storage.AllowBlockedRead.SetResult(); @@ -626,19 +244,18 @@ public async Task InitializationCallerCancellation_LeavesOwnedRecoveryRunning() await manager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Equal(["append"], storage.OperationLog); Assert.Equal(1, state.WriteCompletedCount); - Assert.Null(state.Failure); } [Theory] [InlineData(false)] [InlineData(true)] - public async Task RecoveryShutdown_CancelsInitializationWithoutFaultNotification(bool lifecycleStop) + public async Task RecoveryShutdown_CancelsInitializationWithoutCompletingRecovery(bool lifecycleStop) { var storage = new MutableReadStorage(1, Array.Empty()); var sut = CreateTestSystem(storage); await using var manager = sut.Manager; - var first = new HookState(); - var second = new HookState(); + var first = new LifecycleState(); + var second = new LifecycleState(); manager.RegisterStateMachine("first", first); manager.RegisterStateMachine("second", second); var startup = lifecycleStop @@ -663,15 +280,14 @@ await WaitFor(lifecycleStop Assert.Empty(storage.OperationLog); Assert.All(new[] { first, second }, state => { - Assert.Equal(0, state.FaultCount); Assert.Equal(0, state.RecoveryCompletedCount); - Assert.Equal(0, state.PendingValidationCount); Assert.Equal(0, state.CaptureCount); Assert.Equal(0, state.WriteCompletedCount); - Assert.Null(state.Failure); }); if (lifecycleStop) { + Assert.True(manager.TryGetStateMachine("first", out var registered)); + Assert.Same(first, registered); await Assert.ThrowsAnyAsync(() => manager.WriteStateAsync(CancellationToken.None).AsTask()); } @@ -683,7 +299,7 @@ await WaitFor(lifecycleStop [InlineData("provider-cancellation")] [InlineData("provider-io")] [InlineData("replay")] - public async Task RecoveryFailure_FencesAndNotifiesAllStates(string failure) + public async Task RecoveryFailure_FencesAndFaultsInitializationWaiters(string failure) { var context = new QueuedSynchronizationContext(); await context.Run(async () => @@ -695,7 +311,7 @@ await context.Run(async () => ? new RawReadStorage([1, 2, 3]) : new CapturingStorage { NextReadException = expected }; await using var manager = CreateTestSystem(storage).Manager; - var states = new[] { new HookState(), new HookState() }; + var states = new[] { new LifecycleState(), new LifecycleState() }; manager.RegisterStateMachine("first", states[0]); manager.RegisterStateMachine("second", states[1]); var waiters = new[] @@ -703,15 +319,6 @@ await context.Run(async () => manager.InitializeAsync(CancellationToken.None).AsTask(), manager.InitializeAsync(CancellationToken.None).AsTask() }; - var notified = new List(); - foreach (var state in states) - { - state.FaultAction = exception => - { - Assert.All(waiters, waiter => Assert.False(waiter.IsCompleted)); - notified.Add(exception); - }; - } var observed = await Record.ExceptionAsync(() => WaitFor(waiters[0])); if (failure == "replay") @@ -725,14 +332,7 @@ await context.Run(async () => } Assert.Same(observed, await Record.ExceptionAsync(() => WaitFor(waiters[1]))); - Assert.Equal(2, notified.Count); - Assert.All(notified, exception => Assert.Same(observed, exception)); - Assert.All(states, state => - { - Assert.Same(observed, state.Failure); - Assert.Equal(1, state.FaultCount); - Assert.Equal(0, state.RecoveryCompletedCount); - }); + Assert.All(states, state => Assert.Equal(0, state.RecoveryCompletedCount)); var rejected = await Assert.ThrowsAsync(() => manager.InitializeAsync(CancellationToken.None).AsTask()); Assert.Same(observed, rejected.InnerException); }); @@ -747,7 +347,7 @@ public async Task RecoveryIoFailure_DuringShutdownPreservesOriginalCause() storage.ReadAsync(Arg.Any(), Arg.Any()) .Returns(call => ReadAsync(call.Arg())); await using var manager = CreateTestSystem(storage).Manager; - var state = new HookState(); + var state = new LifecycleState(); manager.RegisterStateMachine("state", state); var initializing = manager.InitializeAsync(CancellationToken.None).AsTask(); await WaitFor(entered.Task); @@ -755,8 +355,6 @@ public async Task RecoveryIoFailure_DuringShutdownPreservesOriginalCause() await WaitFor(manager.DisposeAsync().AsTask()); Assert.Same(expected, await Record.ExceptionAsync(() => WaitFor(initializing))); Assert.Same(expected, await Record.ExceptionAsync(() => WaitFor(another))); - Assert.Same(expected, state.Failure); - Assert.Equal(1, state.FaultCount); Assert.Equal(0, state.RecoveryCompletedCount); async ValueTask ReadAsync(CancellationToken token) @@ -774,22 +372,23 @@ async ValueTask ReadAsync(CancellationToken token) } [Fact] - public async Task IdleShutdown_CompletesWithoutFaultNotification() + public async Task IdleShutdown_CompletesWithoutFencing() { var sut = CreateTestSystem(); await using var manager = sut.Manager; - var state = new HookState(); + var state = new LifecycleState(); manager.RegisterStateMachine("state", state); await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); await sut.Lifecycle.OnStop(TestContext.Current.CancellationToken); - Assert.Null(state.Failure); - Assert.Equal(0, state.FaultCount); + Assert.True(manager.TryGetStateMachine("state", out var registered)); + Assert.Same(state, registered); + await Assert.ThrowsAnyAsync(() => manager.WriteStateAsync(CancellationToken.None).AsTask()); } [Theory] [InlineData(false)] [InlineData(true)] - public async Task AdmittedShutdownCancellation_NotifiesAndFaultsWaiters(bool snapshot) + public async Task AdmittedShutdownCancellation_FencesAndFaultsWaiters(bool snapshot) { var storage = new CapturingStorage { @@ -799,7 +398,7 @@ public async Task AdmittedShutdownCancellation_NotifiesAndFaultsWaiters(bool sna }; var sut = CreateTestSystem(storage); await using var manager = sut.Manager; - var state = new HookState(); + var state = new LifecycleState(); manager.RegisterStateMachine("state", state); await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); var write = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); @@ -807,9 +406,9 @@ public async Task AdmittedShutdownCancellation_NotifiesAndFaultsWaiters(bool sna var queued = manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask(); await sut.Lifecycle.OnStop(TestContext.Current.CancellationToken); var exception = await Assert.ThrowsAnyAsync(() => WaitFor(write)); - Assert.Same(exception, state.Failure); Assert.Same(exception, await Record.ExceptionAsync(() => WaitFor(queued))); - Assert.Equal(1, state.FaultCount); + var rejected = Assert.Throws(() => manager.TryGetStateMachine("state", out _)); + Assert.Same(exception, rejected.InnerException); Assert.Equal(0, state.WriteCompletedCount); } @@ -825,7 +424,7 @@ public async Task CallerCancellation_PreservesOwnedStorageAndAck(bool snapshot) BlockNextReplace = snapshot }; await using var manager = CreateTestSystem(storage).Manager; - var state = new HookState(); + var state = new LifecycleState(); var acknowledged = NewSignal(); state.WriteCompletedAction = () => acknowledged.SetResult(); manager.RegisterStateMachine("state", state); @@ -843,40 +442,30 @@ public async Task CallerCancellation_PreservesOwnedStorageAndAck(bool snapshot) else storage.ReleaseAppend.SetResult(); await WaitFor(acknowledged.Task); await WaitFor(nextWrite); - Assert.Equal(2, state.PendingValidationCount); + Assert.Equal(2, state.CaptureCount); Assert.Equal(1, state.WriteCompletedCount); Assert.Equal(0, manager.PendingWriteByteCount); - Assert.Null(state.Failure); Assert.Equal(snapshot ? 0 : 1, storage.Appends.Count); Assert.Equal(snapshot ? 1 : 0, storage.Replaces.Count); } [Fact] - public async Task AdmittedDeleteShutdownCancellation_NotifiesAndFaultsWaiters() + public async Task AdmittedDeleteShutdownCancellation_FencesAndFaultsWaiters() { var storage = new BlockingDeleteStorage(); var sut = CreateTestSystem(storage); await using var manager = sut.Manager; - var state = new HookState(); + var state = new LifecycleState(); manager.RegisterStateMachine("state", state); await sut.Lifecycle.OnStart(TestContext.Current.CancellationToken); var deleting = manager.DeleteStateAsync(CancellationToken.None).AsTask(); await WaitFor(storage.FirstDeleteStarted.Task); var queued = manager.WriteStateAsync(CancellationToken.None).AsTask(); - var notified = false; - state.FaultAction = _ => - { - Assert.False(deleting.IsCompleted); - Assert.False(queued.IsCompleted); - notified = true; - }; await WaitFor(sut.Lifecycle.OnStop(TestContext.Current.CancellationToken)); var exception = await Assert.ThrowsAnyAsync(() => WaitFor(deleting)); - Assert.Same(exception, state.Failure); Assert.Same(exception, await Record.ExceptionAsync(() => WaitFor(queued))); - Assert.True(notified); - Assert.Equal(1, state.FaultCount); - Assert.Equal(1, state.DeleteStartedCount); + var rejected = Assert.Throws(() => manager.TryGetStateMachine("state", out _)); + Assert.Same(exception, rejected.InnerException); Assert.Equal(1, state.ResetCount); Assert.Equal(0, state.WriteCompletedCount); } @@ -889,20 +478,19 @@ await context.Run(async () => { var storage = new CapturingStorage(); await using var manager = CreateTestSystem(storage).Manager; - var state = new HookState(); + var state = new LifecycleState(); manager.RegisterStateMachine("state", state); await manager.InitializeAsync(TestContext.Current.CancellationToken); using var caller = new CancellationTokenSource(); var write = manager.WriteStateAsync(caller.Token).AsTask(); - Assert.Equal(0, state.PendingValidationCount); + Assert.Equal(0, state.CaptureCount); caller.Cancel(); var remainingWaiter = manager.WriteStateAsync(CancellationToken.None).AsTask(); await Assert.ThrowsAnyAsync(() => write); await remainingWaiter; Assert.Single(storage.Appends); - Assert.Equal(1, state.PendingValidationCount); + Assert.Equal(1, state.CaptureCount); Assert.Equal(1, state.WriteCompletedCount); - Assert.Null(state.Failure); }); } @@ -915,8 +503,6 @@ private sealed class QueuedSynchronizationContext : SynchronizationContext private readonly Channel<(SendOrPostCallback Callback, object? State)> _queue = Channel.CreateUnbounded<(SendOrPostCallback, object?)>(); - public int Turn { get; private set; } - public override void Post(SendOrPostCallback callback, object? state) => Assert.True(_queue.Writer.TryWrite((callback, state))); public async Task Run(Func action) @@ -938,7 +524,6 @@ private void Invoke(SendOrPostCallback callback, object? state) SetSynchronizationContext(this); try { - Turn++; callback(state); } finally @@ -948,52 +533,26 @@ private void Invoke(SendOrPostCallback callback, object? state) } } - private sealed class HookState : IStateMachine + private sealed class LifecycleState : IStateMachine { public bool EmitEntry { get; set; } = true; - public Action? PendingValidationAction { get; set; } - public Action? ValidateWriteAction { get; set; } - public Action? ValidateDeleteAction { get; set; } - public Action? DeleteStartedAction { get; set; } public Action? ResetAction { get; set; } - public Action? CaptureAction { get; set; } public Action? WriteCompletedAction { get; set; } - public Action? FaultAction { get; set; } - public JournalStreamWriter Writer { get; private set; } public int ResetCount { get; private set; } - public int PendingValidationCount { get; private set; } public int CaptureCount { get; private set; } - public int DeleteStartedCount { get; private set; } public int WriteCompletedCount { get; private set; } - public int FaultCount { get; private set; } public int RecoveryCompletedCount { get; private set; } - public Exception? Failure { get; private set; } - public void ValidatePendingChanges() - { - PendingValidationCount++; - PendingValidationAction?.Invoke(); - } - public void ValidateWrite() => ValidateWriteAction?.Invoke(); - public void ValidateDelete() => ValidateDeleteAction?.Invoke(); public void OnRecoveryCompleted() => RecoveryCompletedCount++; - public void OnDeleteStarted() - { - DeleteStartedCount++; - DeleteStartedAction?.Invoke(); - } - public void Reset(JournalStreamWriter writer) { - Writer = writer; ResetCount++; ResetAction?.Invoke(); } public void WritePendingEntries(JournalStreamWriter writer) { - CaptureAction?.Invoke(); CaptureCount++; if (EmitEntry) { @@ -1012,13 +571,6 @@ public void OnWriteCompleted() WriteCompletedAction?.Invoke(); } - public void OnFaulted(Exception exception) - { - FaultCount++; - Failure = exception; - FaultAction?.Invoke(exception); - } - public void ReplayEntry(JournalEntry entry, JournalReplayContext context) => throw new NotSupportedException(); } } From 149f685123f332f4f533652f47e21f9eb7260f47 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 21 Sep 2026 09:34:40 -0700 Subject: [PATCH 07/11] refactor(journaling): supply custom state codecs during construction --- .../grains/journaling/runtime-behavior.md | 10 +- .../IJournaledStateManager.cs | 14 --- .../JournaledStateManager.cs | 3 - .../Orleans.Journaling/Orleans.Journaling.cs | 1 - .../KeyedJournalingRegistrationTests.cs | 93 ++++++++++++------- 5 files changed, 63 insertions(+), 58 deletions(-) diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index f1e6a6becd1..e1aa4a5616c 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -107,10 +107,12 @@ individual initialization caller's token ends only its wait; owned recovery cont ## Custom state lifecycle Custom implementations share the manager's single logical execution thread. -Resolve write codecs through to use the -owning manager's configured format, including before recovery of an empty journal. Delegating managers forward -codec resolution to that owner. Grain-bound managers resolve codecs from the activation's services; -standalone owners use shared application services. +Supply command codecs as constructor dependencies. The registration factory selects codecs keyed by the +same write-format key used to configure the journal owner. Activation-owned state factories resolve those +dependencies from the activation's services; standalone callers supply codecs with the appropriate lifetime. +The codec is available when the state is constructed, including for an empty journal. +During replay, selects the codec +for each entry's stored format. States synchronously encode their pending changes through or their current contents through . diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index cf80ccd29ba..feff90a848b 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -43,20 +43,6 @@ public interface IJournaledStateManager : IAsyncDisposable /// if the state machine is registered; otherwise, . bool TryGetStateMachine(string name, [NotNullWhen(true)] out IStateMachine? stateMachine); - /// - /// Resolves a command codec for this manager's configured write journal format. - /// - /// The command codec service type. - /// The codec registered for this manager's write format. - /// - /// Codecs are resolved from the owning activation's services for grain-bound managers and from shared - /// application services for standalone owners. They are available before recovery, including for an empty journal. Delegating managers forward - /// this call to their owning manager. The default implementation throws . - /// - /// The manager does not support command codec resolution. - TCodec GetRequiredCommandCodec() where TCodec : notnull - => throw new NotSupportedException("This journaled state manager does not support write command codec resolution."); - /// /// Persists pending changes from the registered state machines to the journal. /// diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 676dfe4ef5e..f9811ed6ee3 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -100,9 +100,6 @@ internal static IJournalStorage CreateStorage(IJournalStorageProvider storagePro internal IServiceProvider ServiceProvider => _grainContext is { } context ? context.ActivationServices : _shared.ServiceProvider; - public TCodec GetRequiredCommandCodec() where TCodec : notnull - => JournalFormatServices.GetRequiredCommandCodec(ServiceProvider, _shared.JournalFormatKey); - public bool TryGetStateMachine(string name, [NotNullWhen(true)] out IStateMachine? stateMachine) { ArgumentException.ThrowIfNullOrEmpty(name); diff --git a/src/api/Orleans.Journaling/Orleans.Journaling.cs b/src/api/Orleans.Journaling/Orleans.Journaling.cs index 0bf70416d8d..f29f6db150e 100644 --- a/src/api/Orleans.Journaling/Orleans.Journaling.cs +++ b/src/api/Orleans.Journaling/Orleans.Journaling.cs @@ -224,7 +224,6 @@ public partial interface IJournaledStateManager : System.IAsyncDisposable long PendingWriteByteCount { get; } System.Threading.Tasks.ValueTask DeleteStateAsync(System.Threading.CancellationToken cancellationToken = default); - TCodec GetRequiredCommandCodec(); System.Threading.Tasks.ValueTask InitializeAsync(System.Threading.CancellationToken cancellationToken = default); void RegisterStateMachine(string name, IStateMachine stateMachine); System.Threading.Tasks.ValueTask System.IAsyncDisposable.DisposeAsync(); diff --git a/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs b/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs index 5281aec9408..6c994b1f2b0 100644 --- a/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs +++ b/test/Orleans.Journaling.Tests/KeyedJournalingRegistrationTests.cs @@ -383,7 +383,7 @@ public void DurableService_ResolvesCommandCodecFromJournalFormatKey() } [Fact] - public async Task ManagerCommandCodec_UsesActivationScopeBeforeRecovery() + public async Task StateConstruction_InjectsActivationScopedCodecBeforeRecovery() { var builder = CreateNamedProviderBuilder(); builder.AddVolatileJournalStorage(); @@ -401,20 +401,25 @@ public async Task ManagerCommandCodec_UsesActivationScopeBeforeRecovery() static (services, _) => new OrleansBinaryDurableValueCommandCodec( services.GetRequiredService().GetCodec(), services.GetRequiredService())); + builder.Services.AddStateMachine(static (services, _) => + new CodecState(services.GetRequiredKeyedService>(OrleansBinaryJournalFormat.JournalFormatKey))); await using var services = builder.Services.BuildServiceProvider(validateScopes: true); await using var first = services.CreateAsyncScope(); await using var second = services.CreateAsyncScope(); var owner = first.ServiceProvider.GetRequiredService(); var manager = first.ServiceProvider.GetRequiredService(); Assert.Same(owner, manager); - var codec = owner.GetRequiredCommandCodec>(); + var state = manager.GetOrAddState("value"); + var codec = state.Codec; Assert.Same(first.ServiceProvider.GetRequiredKeyedService>(OrleansBinaryJournalFormat.JournalFormatKey), codec); - Assert.NotSame(codec, second.ServiceProvider.GetRequiredService() - .GetRequiredCommandCodec>()); + Assert.NotSame(codec, second.ServiceProvider.GetRequiredService() + .GetOrAddState("value").Codec); Assert.Throws(() => services.GetRequiredKeyedService>(OrleansBinaryJournalFormat.JournalFormatKey)); - var state = new DurableValue("value", owner, codec); - Assert.Same(state, manager.GetOrAddValue("value")); + Assert.Same(state, manager.GetOrAddState("value")); + Assert.Same(state, first.ServiceProvider.GetRequiredKeyedService("value")); + Assert.True(owner.TryGetStateMachine("value", out var registered)); + Assert.Same(state, registered); var lifecycle = Assert.IsType(first.ServiceProvider.GetRequiredService().ObservableLifecycle); Assert.Equal(1, lifecycle.Subscriptions); await lifecycle.OnStart(TestContext.Current.CancellationToken); @@ -426,7 +431,7 @@ public async Task ManagerCommandCodec_UsesActivationScopeBeforeRecovery() } [Fact] - public async Task ManagerCommandCodec_UsesOwningNamedFormatOnEmptyJournal() + public async Task StateConstruction_InjectsNamedFormatCodecOnEmptyJournal() { var builder = CreateNamedProviderBuilder(); builder.AddVolatileJournalStorage(); @@ -435,7 +440,7 @@ public async Task ManagerCommandCodec_UsesOwningNamedFormatOnEmptyJournal() new NamedBinaryJournalFormat(services.GetRequiredService())); builder.Services.AddKeyedSingleton(typeof(IDurableDictionaryCommandCodec<,>), CustomFormatKey, typeof(OrleansBinaryDurableDictionaryCommandCodec<,>)); - builder.Services.AddKeyedSingleton>(CustomFormatKey, static (services, _) => + builder.Services.AddKeyedScoped>(CustomFormatKey, static (services, _) => new OrleansBinaryDurableValueCommandCodec( services.GetRequiredService().GetCodec(), services.GetRequiredService())); @@ -447,39 +452,54 @@ public async Task ManagerCommandCodec_UsesOwningNamedFormatOnEmptyJournal() TimeProvider.System, services), new TestJournalStorageProvider(customStorage))); - await using var services = builder.Services.BuildServiceProvider(); + await using var services = builder.Services.BuildServiceProvider(validateScopes: true); + await using var dependencies = services.CreateAsyncScope(); var factory = services.GetRequiredKeyedService("custom"); await using var defaultManager = services.GetRequiredService().CreateStandalone(new JournalId("default")); await using var customManager = factory.CreateStandalone(new JournalId("custom")); - IJournaledStateManager delegating = new DelegatingCodecManager(customManager); - var codec = delegating.GetRequiredCommandCodec>(); - Assert.Same(services.GetRequiredKeyedService>(CustomFormatKey), codec); - Assert.Same(services.GetRequiredKeyedService>(JsonLinesJournalFormat.JournalFormatKey), - defaultManager.GetRequiredCommandCodec>()); - Assert.NotSame(defaultManager.GetRequiredCommandCodec>(), codec); - var state = new DurableValue("value", delegating, codec); + IJournaledStateManager delegating = new DelegatingStateManager(customManager); + var codec = dependencies.ServiceProvider.GetRequiredKeyedService>(CustomFormatKey); + Assert.Throws(() => + services.GetRequiredKeyedService>(CustomFormatKey)); + var defaultCodec = services.GetRequiredKeyedService>(JsonLinesJournalFormat.JournalFormatKey); + Assert.NotSame(defaultCodec, codec); + var state = new CodecState(codec); + Assert.Same(codec, state.Codec); + delegating.RegisterStateMachine("value", state); + var defaultState = new CodecState(defaultCodec); + defaultManager.RegisterStateMachine("value", defaultState); + await defaultManager.InitializeAsync(TestContext.Current.CancellationToken); await delegating.InitializeAsync(TestContext.Current.CancellationToken); Assert.Equal(0, state.Value); Assert.Empty(customStorage.Segments); state.Value = 42; + defaultState.Value = 7; await delegating.WriteStateAsync(TestContext.Current.CancellationToken); + await defaultManager.WriteStateAsync(TestContext.Current.CancellationToken); Assert.Single(customStorage.Segments); await using var recovered = factory.CreateStandalone(new JournalId("custom")); - var recoveredState = new DurableValue("value", recovered, recovered.GetRequiredCommandCodec>()); + var recoveredState = new CodecState(codec); + recovered.RegisterStateMachine("value", recoveredState); await recovered.InitializeAsync(TestContext.Current.CancellationToken); Assert.Equal(42, recoveredState.Value); - var missing = Assert.Throws(() => customManager.GetRequiredCommandCodec>()); - Assert.Contains(CustomFormatKey, missing.Message); - Assert.Contains(nameof(IDurableQueueCommandCodec), missing.Message); + await using var recoveredDefault = services.GetRequiredService().CreateStandalone(new JournalId("default")); + var recoveredDefaultState = new CodecState(defaultCodec); + recoveredDefault.RegisterStateMachine("value", recoveredDefaultState); + await recoveredDefault.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(7, recoveredDefaultState.Value); } [Fact] - public void ManagerCommandCodec_DefaultReportsUnsupportedResolution() + public void StateConstruction_MissingFormatCodecFails() { - IJournaledStateManager manager = new UnsupportedCodecManager(); - var exception = Assert.Throws(() => manager.GetRequiredCommandCodec>()); - Assert.Contains("write command codec resolution", exception.Message); + var builder = CreateNamedProviderBuilder(); + builder.AddJournaling(); + using var services = builder.Services.BuildServiceProvider(); + Assert.NotNull(services.GetRequiredKeyedService>(JsonLinesJournalFormat.JournalFormatKey)); + var exception = Assert.Throws(() => + new CodecState(services.GetRequiredKeyedService>(CustomFormatKey))); + Assert.Contains(nameof(IDurableValueCommandCodec), exception.Message); } [Fact] @@ -529,9 +549,20 @@ private sealed class NamedBinaryJournalFormat(IJournalFormat inner) : IJournalFo public void Replay(JournalBufferReader input, JournalReplayContext context) => inner.Replay(input, context); } - private sealed class DelegatingCodecManager(IJournaledStateManager inner) : IJournaledStateManager + private sealed class CodecState(IDurableValueCommandCodec codec) : IStateMachine, IDurableValueCommandHandler + { + public IDurableValueCommandCodec Codec { get; } = codec; + public int Value { get; set; } + public void Reset(JournalStreamWriter writer) => Value = 0; + public void WritePendingEntries(JournalStreamWriter writer) => Codec.WriteSet(Value, writer); + public void WriteSnapshot(JournalStreamWriter writer) => Codec.WriteSet(Value, writer); + public void ReplayEntry(JournalEntry entry, JournalReplayContext context) => + context.GetRequiredCommandCodec(entry.FormatKey, Codec).Apply(entry.Reader, this); + public void ApplySet(int value) => Value = value; + } + + private sealed class DelegatingStateManager(IJournaledStateManager inner) : IJournaledStateManager { - public TCodec GetRequiredCommandCodec() where TCodec : notnull => inner.GetRequiredCommandCodec(); public ValueTask InitializeAsync(CancellationToken cancellationToken) => inner.InitializeAsync(cancellationToken); public void RegisterStateMachine(string name, IStateMachine state) => inner.RegisterStateMachine(name, state); public bool TryGetStateMachine(string name, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out IStateMachine? state) @@ -540,16 +571,6 @@ public bool TryGetStateMachine(string name, [System.Diagnostics.CodeAnalysis.Not public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => inner.DeleteStateAsync(cancellationToken); } - private sealed class UnsupportedCodecManager : IJournaledStateManager - { - public ValueTask InitializeAsync(CancellationToken cancellationToken) => throw new NotSupportedException(); - public void RegisterStateMachine(string name, IStateMachine state) => throw new NotSupportedException(); - public bool TryGetStateMachine(string name, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out IStateMachine? state) - => throw new NotSupportedException(); - public ValueTask WriteStateAsync(CancellationToken cancellationToken) => throw new NotSupportedException(); - public ValueTask DeleteStateAsync(CancellationToken cancellationToken) => throw new NotSupportedException(); - } - private sealed class LifecycleJournalStorageProvider : IJournalStorageProvider, IJournalStorageCatalog, ILifecycleParticipant { private readonly VolatileJournalStorageProvider _storage = new(); From f9235e20993861659faf34efb824df1e1ab87d25 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 21 Sep 2026 10:28:04 -0700 Subject: [PATCH 08/11] fix(journaling): allow explicit recovery retries --- .../grains/journaling/runtime-behavior.md | 12 +- .../IJournaledStateManager.cs | 4 +- src/Orleans.Journaling/IStateMachine.cs | 4 +- .../JournaledStateManager.cs | 18 ++- src/Orleans.Journaling/README.md | 12 +- .../StateManagerLifecycleTests.cs | 127 ++++++++++++++++-- .../StateManagerTests.cs | 66 ++++++--- 7 files changed, 203 insertions(+), 40 deletions(-) diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index e1aa4a5616c..49a6b41e3a9 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -78,7 +78,7 @@ Design commands to tolerate retries at the application boundary. Use operation i ## Storage failures -A failed append, snapshot replacement, delete, or initialization permanently fences that manager instance. +A failed append, snapshot replacement, or delete permanently fences that manager instance. Queued operations fault, and later write, delete, registration, and initialization requests fail explicitly. Existing in-memory state remains available to in-flight calls until deactivation completes. The grain runtime starts deactivation as part of handling the failure. @@ -97,8 +97,12 @@ assigned lifetimes. Cancelling a write's cancellation token stops the caller's wait. An already queued write continues to its storage outcome, so the caller reconciles that outcome before retrying the command. -An initialization failure preserves stored data for diagnosis. Restore the required format/codec registration -or repair the backing data before creating a fresh manager or retrying activation. +An initialization failure reports its error to that attempt's callers and leaves the manager uninitialized. +The caller can retry after a transient +failure or after restoring the required format, codec, or backing data. Each attempt resets recovery +bookkeeping and replays the journal from the beginning using the same registered state machines. +Concurrent callers share the active attempt; writes and deletion become available after initialization +succeeds. State registration stays closed once initialization has begun. Owner shutdown which cancels initial recovery cancels all initialization waiters and leaves the manager stopped. Disposal waits for the owned read to finish before releasing journal resources. Cancelling an @@ -123,7 +127,7 @@ The journal owner keeps feature operations quiescent through deletion's storage including when a caller cancels its wait. Successful deletion calls before completing deletion waiters. -The manager records the first capture or storage failure, fences further persistence, faults current +The manager records the first write or delete failure, fences further persistence, faults current and queued manager waiters, and requests grain deactivation. Features observe their write failures and complete their own operation waiters and resource cleanup through their operation and lifecycle ownership. Standalone callers own that cleanup explicitly. A previously captured write retains its actual storage diff --git a/src/Orleans.Journaling/IJournaledStateManager.cs b/src/Orleans.Journaling/IJournaledStateManager.cs index feff90a848b..b6363a1b4cc 100644 --- a/src/Orleans.Journaling/IJournaledStateManager.cs +++ b/src/Orleans.Journaling/IJournaledStateManager.cs @@ -19,7 +19,9 @@ public interface IJournaledStateManager : IAsyncDisposable /// Initializes the state manager by replaying its journal. /// /// - /// A recovery failure permanently fences this instance. Recover by creating a new manager and new state instances. + /// A recovery failure fails the current initialization attempt and leaves this instance uninitialized. + /// A subsequent call retries recovery from the beginning, resetting and replaying the registered state machines. + /// Writes become available after initialization succeeds. A manager fenced by a persistence failure requires a new instance. /// Owner shutdown which cancels recovery cancels initialization and leaves this instance stopped. /// Cancelling the caller's token ends only that caller's wait while owned recovery continues. /// diff --git a/src/Orleans.Journaling/IStateMachine.cs b/src/Orleans.Journaling/IStateMachine.cs index 39f824a6db7..7fd2a8c7366 100644 --- a/src/Orleans.Journaling/IStateMachine.cs +++ b/src/Orleans.Journaling/IStateMachine.cs @@ -25,8 +25,8 @@ namespace Orleans.Journaling; /// rather than treating in-memory mutations as durable. /// /// -/// A failed journal operation permanently fences the manager and requests grain deactivation. -/// A new manager initializes new state instances by calling and replaying durable entries. +/// A failed write or delete permanently fences the manager and requests grain deactivation. +/// Recovery calls before replaying durable entries, including when initialization is retried. /// /// /// diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index f9811ed6ee3..827a862b330 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -162,8 +162,9 @@ public async ValueTask InitializeAsync(CancellationToken cancellationToken = def lock (_lock) { ThrowIfFenced(); - if (_workLoop is null) + if (_workLoop is null || _state is ManagerState.RecoveryFailed) { + _state = ManagerState.Unknown; _workLoop = Start(); } @@ -197,7 +198,19 @@ private async Task WorkLoop() } catch (Exception exception) { - Fence(exception); + try + { + LogErrorProcessingWorkItems(_shared.Logger, exception); + } + finally + { + lock (_lock) + { + _state = ManagerState.RecoveryFailed; + FaultQueuedWorkItemsUnderLock(exception); + } + } + return; } @@ -1186,6 +1199,7 @@ private sealed class RegisterStateWorkItem(string name) : WorkItem(name) private enum ManagerState : byte { Unknown, + RecoveryFailed, Ready, Fenced } diff --git a/src/Orleans.Journaling/README.md b/src/Orleans.Journaling/README.md index 549abe06179..671bbda74e9 100644 --- a/src/Orleans.Journaling/README.md +++ b/src/Orleans.Journaling/README.md @@ -183,8 +183,9 @@ provider and requested state name. - `OnRecoveryCompleted`: finish reconstruction before application use. - `OnWriteCompleted`: publish effects that depend on storage acknowledgement. -The recovery model uses fresh instances and replay. `JournalReplayContext.ResolveStateMachine` -routes entries to the state machine for their stream. +Recovery resets state machines and replays durable entries. A failed initialization can be retried on the +same manager and registered states. `JournalReplayContext.ResolveStateMachine` routes entries to the +state machine for their stream. `IJournaledStateManager` is independent of the grain-facing `IDurableStateManager` and extends `IAsyncDisposable`. Its owner API provides `RegisterStateMachine`, `TryGetStateMachine`, @@ -255,13 +256,18 @@ durable state and await `WriteStateAsync`. One acknowledgement covers the manage batch, including changes staged by interleaved callers. Applications are responsible for sequencing that transition with other interleaved operations and for making uncertain-outcome retries idempotent. -A failed journal operation permanently fences the manager, faults queued operations, and requests +A failed write or delete permanently fences the manager, faults queued operations, and requests deactivation of the associated grain. In-flight calls retain their existing in-memory state while subsequent state-manager operations fail explicitly. A new activation recovers the actual durable outcome. For a manager created through `IJournaledStateManagerFactory`, dispose the failed instance and create another manager for the same `JournalId`, explicitly constructing and registering fresh state components before initialization and retiring the old components and dependencies according to their assigned lifetimes. +An initialization failure reports its error to the attempt's callers and leaves the manager uninitialized. +Call `InitializeAsync` again to retry from the beginning using the existing reset/replay contract. +Concurrent callers share the active attempt, and writes become available after recovery succeeds. +State registration stays closed after initialization first begins. + Cancelling a caller's wait leaves an already queued write running. Observe durability through write acknowledgement or a fresh activation before deciding whether to retry an application command. diff --git a/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs b/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs index 1888be3fdd6..a0eae08c706 100644 --- a/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs @@ -208,7 +208,7 @@ public async Task DeactivationFailure_PreservesOriginalStorageFailure() } [Fact] - public async Task InitializationFailure_PreservesOriginalCause() + public async Task InitializationFailure_ReportsOriginalCauseAndAllowsRetry() { var expected = new IOException("Recovery failed."); var storage = new CapturingStorage { NextReadException = expected }; @@ -216,9 +216,76 @@ public async Task InitializationFailure_PreservesOriginalCause() var state = new LifecycleState(); manager.RegisterStateMachine("state", state); Assert.Same(expected, await Record.ExceptionAsync(() => manager.InitializeAsync(CancellationToken.None).AsTask())); - var retry = await Assert.ThrowsAsync(() => manager.InitializeAsync(CancellationToken.None).AsTask()); - Assert.Same(expected, retry.InnerException); Assert.Equal(0, state.RecoveryCompletedCount); + Assert.True(manager.TryGetStateMachine("state", out var registered)); + Assert.Same(state, registered); + var write = await Assert.ThrowsAsync(() => manager.WriteStateAsync(CancellationToken.None).AsTask()); + var delete = await Assert.ThrowsAsync(() => manager.DeleteStateAsync(CancellationToken.None).AsTask()); + Assert.Contains("not been initialized", write.Message); + Assert.Contains("not been initialized", delete.Message); + var lateRegistration = Assert.Throws(() => manager.RegisterStateMachine("late", new LifecycleState())); + Assert.Contains("initialization has begun", lateRegistration.Message); + + await manager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(1, state.RecoveryCompletedCount); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Single(storage.Appends); + Assert.Equal(1, state.WriteCompletedCount); + } + + [Fact] + public async Task RecoveryRetry_CoalescesCallersAndOwnsRead() + { + var storage = new MutableReadStorage(2, [1, 2, 3], CreatePersistedValueBytes("value", 42)); + await using var manager = CreateTestSystem(storage).Manager; + var value = new DurableValue("value", manager, CreateValueCodec()); + await Assert.ThrowsAsync(() => manager.InitializeAsync(CancellationToken.None).AsTask()); + Assert.Equal(1, storage.ReadCount); + + using var caller = new CancellationTokenSource(); + var canceledWaiter = manager.InitializeAsync(caller.Token).AsTask(); + await WaitFor(storage.BlockedReadStarted.Task); + var remainingWaiter = manager.InitializeAsync(CancellationToken.None).AsTask(); + Assert.Equal(2, storage.ReadCount); + var write = await Assert.ThrowsAsync(() => manager.WriteStateAsync(CancellationToken.None).AsTask()); + Assert.Contains("not been initialized", write.Message); + var registration = Assert.Throws(() => manager.RegisterStateMachine("late", new LifecycleState())); + Assert.Contains("initialization has begun", registration.Message); + caller.Cancel(); + await Assert.ThrowsAnyAsync(() => WaitFor(canceledWaiter)); + Assert.False(storage.ReadToken.IsCancellationRequested); + Assert.False(remainingWaiter.IsCompleted); + + storage.AllowBlockedRead.SetResult(); + await WaitFor(remainingWaiter); + Assert.Equal(42, value.Value); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(2, storage.ReadCount); + value.Value = 43; + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal(["append"], storage.OperationLog); + } + + [Fact] + public async Task RecoveryCompletionFailure_RetryResetsRegisteredState() + { + var storage = new CapturingStorage(); + await using var manager = CreateTestSystem(storage).Manager; + var state = new LifecycleState(); + manager.RegisterStateMachine("state", state); + var expected = new InvalidOperationException("Recovery completion failed."); + state.RecoveryCompletedAction = () => throw expected; + Assert.Same(expected, await Record.ExceptionAsync(() => manager.InitializeAsync(CancellationToken.None).AsTask())); + Assert.Equal(1, state.ResetCount); + Assert.Equal(1, state.RecoveryCompletedCount); + Assert.Empty(storage.Appends); + + state.RecoveryCompletedAction = null; + await manager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal(2, state.ResetCount); + Assert.Equal(2, state.RecoveryCompletedCount); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Single(storage.Appends); } [Fact] @@ -299,7 +366,7 @@ await WaitFor(lifecycleStop [InlineData("provider-cancellation")] [InlineData("provider-io")] [InlineData("replay")] - public async Task RecoveryFailure_FencesAndFaultsInitializationWaiters(string failure) + public async Task RecoveryFailure_FaultsAttemptWaitersAndAllowsRetry(string failure) { var context = new QueuedSynchronizationContext(); await context.Run(async () => @@ -308,7 +375,7 @@ await context.Run(async () => ? new OperationCanceledException("Provider read cancellation.", new CancellationToken(canceled: true)) : new IOException("Provider read failure."); IJournalStorage storage = failure == "replay" - ? new RawReadStorage([1, 2, 3]) + ? new MutableReadStorage([1, 2, 3], []) : new CapturingStorage { NextReadException = expected }; await using var manager = CreateTestSystem(storage).Manager; var states = new[] { new LifecycleState(), new LifecycleState() }; @@ -333,11 +400,50 @@ await context.Run(async () => Assert.Same(observed, await Record.ExceptionAsync(() => WaitFor(waiters[1]))); Assert.All(states, state => Assert.Equal(0, state.RecoveryCompletedCount)); - var rejected = await Assert.ThrowsAsync(() => manager.InitializeAsync(CancellationToken.None).AsTask()); - Assert.Same(observed, rejected.InnerException); + var retry = manager.InitializeAsync(CancellationToken.None).AsTask(); + var concurrent = manager.InitializeAsync(CancellationToken.None).AsTask(); + await WaitFor(Task.WhenAll(retry, concurrent)); + Assert.All(states, state => Assert.Equal(1, state.RecoveryCompletedCount)); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.All(states, state => Assert.Equal(1, state.WriteCompletedCount)); }); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RecoveryRetry_ShutdownCancelsAllWaitingCallers(bool lifecycleStop) + { + var storage = new MutableReadStorage(2, [1, 2, 3], []); + var sut = CreateTestSystem(storage); + await using var manager = sut.Manager; + var state = new LifecycleState(); + manager.RegisterStateMachine("state", state); + await Assert.ThrowsAsync(() => lifecycleStop + ? sut.Lifecycle.OnStart(CancellationToken.None) + : manager.InitializeAsync(CancellationToken.None).AsTask()); + var first = manager.InitializeAsync(CancellationToken.None).AsTask(); + await WaitFor(storage.BlockedReadStarted.Task); + var second = manager.InitializeAsync(CancellationToken.None).AsTask(); + await WaitFor(lifecycleStop + ? sut.Lifecycle.OnStop(TestContext.Current.CancellationToken) + : manager.DisposeAsync().AsTask()); + + await Assert.ThrowsAnyAsync(() => WaitFor(first)); + await Assert.ThrowsAnyAsync(() => WaitFor(second)); + Assert.True(storage.ReadToken.IsCancellationRequested); + Assert.Equal(2, storage.ReadCount); + Assert.Equal(0, state.RecoveryCompletedCount); + Assert.Empty(storage.OperationLog); + if (lifecycleStop) + { + await Assert.ThrowsAnyAsync(() => manager.InitializeAsync(CancellationToken.None).AsTask()); + } + + await manager.DisposeAsync(); + await Assert.ThrowsAsync(() => manager.InitializeAsync(CancellationToken.None).AsTask()); + } + [Fact] public async Task RecoveryIoFailure_DuringShutdownPreservesOriginalCause() { @@ -538,12 +644,17 @@ private sealed class LifecycleState : IStateMachine public bool EmitEntry { get; set; } = true; public Action? ResetAction { get; set; } public Action? WriteCompletedAction { get; set; } + public Action? RecoveryCompletedAction { get; set; } public int ResetCount { get; private set; } public int CaptureCount { get; private set; } public int WriteCompletedCount { get; private set; } public int RecoveryCompletedCount { get; private set; } - public void OnRecoveryCompleted() => RecoveryCompletedCount++; + public void OnRecoveryCompleted() + { + RecoveryCompletedCount++; + RecoveryCompletedAction?.Invoke(); + } public void Reset(JournalStreamWriter writer) { diff --git a/test/Orleans.Journaling.Tests/StateManagerTests.cs b/test/Orleans.Journaling.Tests/StateManagerTests.cs index 89cc65294c1..2c9abd55015 100644 --- a/test/Orleans.Journaling.Tests/StateManagerTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerTests.cs @@ -121,8 +121,8 @@ public async Task StateManager_WriteOperations_RequireSuccessfulInitialization() var deleteException = await Assert.ThrowsAsync( () => sut.Manager.DeleteStateAsync(TestContext.Current.CancellationToken).AsTask()); - Assert.Contains("fenced", writeException.Message, StringComparison.Ordinal); - Assert.Contains("fenced", deleteException.Message, StringComparison.Ordinal); + Assert.Contains("not been initialized", writeException.Message, StringComparison.Ordinal); + Assert.Contains("not been initialized", deleteException.Message, StringComparison.Ordinal); } [Fact] @@ -617,7 +617,7 @@ await Assert.ThrowsAsync( [InlineData("append")] [InlineData("replace")] [InlineData("delete")] - public async Task StateManager_FailureDeactivatesOwningGrain(string operation) + public async Task StateManager_FailureHandling_RespectsRecoveryBoundary(string operation) { var expected = new IOException("Expected journal operation failure."); var storage = new CapturingStorage(); @@ -635,6 +635,14 @@ public async Task StateManager_FailureDeactivatesOwningGrain(string operation) if (operation == "initialize") { storage.NextReadException = expected; + Assert.Same(expected, await Assert.ThrowsAsync(() => + manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask())); + context.DidNotReceive().Deactivate(Arg.Any(), Arg.Any()); + await manager.InitializeAsync(TestContext.Current.CancellationToken); + value.Value = 42; + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Single(storage.Appends); + return; } else { @@ -648,7 +656,6 @@ public async Task StateManager_FailureDeactivatesOwningGrain(string operation) var failedOperation = operation switch { - "initialize" => manager.InitializeAsync(TestContext.Current.CancellationToken), "delete" => manager.DeleteStateAsync(TestContext.Current.CancellationToken), _ => manager.WriteStateAsync(TestContext.Current.CancellationToken) }; @@ -1064,7 +1071,7 @@ public async Task StateManager_Recovery_RejectsMalformedTrailingData() } [Fact] - public async Task StateManager_FreshRecovery_ReplaysFixedStorage() + public async Task StateManager_RecoveryRetry_ReplaysFixedStorage() { var validBytes = CreatePersistedValueBytes("value", 42); var storage = new MutableReadStorage([.. validBytes, 1, 2, 3], validBytes); @@ -1075,11 +1082,6 @@ await Assert.ThrowsAsync( () => sut.Lifecycle.OnStart(TestContext.Current.CancellationToken) .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); - await Assert.ThrowsAsync( - () => sut.Manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask()); - await sut.Manager.DisposeAsync(); - sut = CreateTestSystem(storage: storage); - value = new DurableValue("value", sut.Manager, CreateValueCodec()); await sut.Manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask() .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask() @@ -1090,7 +1092,37 @@ await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask( } [Fact] - public async Task StateManager_FreshRecovery_PreservesUnknownStreamOnce() + public async Task StateManager_RecoveryRetry_ReplaysListWithoutDuplicatingEntries() + { + var seedStorage = new CapturingStorage(); + await using (var seed = CreateTestSystem(seedStorage).Manager) + { + var source = new DurableList("items", seed, + new OrleansBinaryDurableListCommandCodec(CodecProvider.GetCodec(), SessionPool)); + await seed.InitializeAsync(TestContext.Current.CancellationToken); + source.Add(1); + source.Add(2); + await seed.WriteStateAsync(TestContext.Current.CancellationToken); + } + + var bytes = seedStorage.RecoverableBytes; + var storage = new MutableReadStorage([.. bytes, 1, 2, 3], bytes); + await using var manager = CreateTestSystem(storage).Manager; + var items = new DurableList("items", manager, + new OrleansBinaryDurableListCommandCodec(CodecProvider.GetCodec(), SessionPool)); + await Assert.ThrowsAsync(() => manager.InitializeAsync(CancellationToken.None).AsTask()); + Assert.Equal([1, 2], items); + + await manager.InitializeAsync(TestContext.Current.CancellationToken); + Assert.Equal([1, 2], items); + Assert.Equal(2, storage.ReadCount); + items.Add(3); + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Equal([1, 2, 3], items); + } + + [Fact] + public async Task StateManager_RecoveryRetry_PreservesUnknownStreamOnce() { var validBytes = CreateUnknownStreamBytes(new JournalStreamId(99), [1, 2, 3]); var storage = new MutableReadStorage([.. validBytes, 1, 2, 3], validBytes) { IsCompactionRequested = true }; @@ -1100,10 +1132,6 @@ await Assert.ThrowsAsync( () => sut.Lifecycle.OnStart(TestContext.Current.CancellationToken) .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); - await Assert.ThrowsAsync( - () => sut.Manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask()); - await sut.Manager.DisposeAsync(); - sut = CreateTestSystem(storage: storage); await sut.Manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask() .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask() @@ -1116,7 +1144,7 @@ await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask( } [Fact] - public async Task StateManager_FreshRecovery_RemovesStaleRetiredPlaceholder() + public async Task StateManager_RecoveryRetry_RemovesStaleRetiredPlaceholder() { var storage = new MutableReadStorage([.. CreateNamedUnknownStreamBytes("stale", new JournalStreamId(8), [1, 2, 3]), 1, 2, 3], []); var sut = CreateTestSystem(storage: storage); @@ -1125,10 +1153,6 @@ await Assert.ThrowsAsync( () => sut.Lifecycle.OnStart(TestContext.Current.CancellationToken) .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); - await Assert.ThrowsAsync( - () => sut.Manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask()); - await sut.Manager.DisposeAsync(); - sut = CreateTestSystem(storage: storage); await sut.Manager.InitializeAsync(TestContext.Current.CancellationToken).AsTask() .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); await sut.Manager.WriteStateAsync(TestContext.Current.CancellationToken).AsTask() @@ -2276,6 +2300,8 @@ private sealed class MutableReadStorage : IJournalStorage private byte[] _bytes; private int _readCount; + public int ReadCount => Volatile.Read(ref _readCount); + public MutableReadStorage(params byte[][] readSnapshots) : this(blockedReadNumber: 0, readSnapshots) { } From c4b9efff90a5131315b146320282d122dfc1b8f5 Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 21 Sep 2026 10:43:03 -0700 Subject: [PATCH 09/11] docs(journaling): clarify recovery instance lifetimes --- src/Orleans.Journaling/IStateMachine.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Journaling/IStateMachine.cs b/src/Orleans.Journaling/IStateMachine.cs index 7fd2a8c7366..bf3fd61204e 100644 --- a/src/Orleans.Journaling/IStateMachine.cs +++ b/src/Orleans.Journaling/IStateMachine.cs @@ -32,7 +32,8 @@ namespace Orleans.Journaling; /// /// Application code prepares fallible work in operation-local data and stages only mutations which are /// safe to commit. Staged mutations are shared by all interleaved callers using the same manager. -/// Storage acknowledgement establishes durability; recovery takes place in a fresh manager and state instances. +/// Storage acknowledgement establishes durability. A new activation creates a fresh manager and state instances; +/// retrying failed initialization resets and replays the existing instances. /// /// public interface IStateMachine From 00e35e55ebecf25ba3d3c36b3420d22cc6b4f65c Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 21 Sep 2026 13:43:33 -0700 Subject: [PATCH 10/11] fix(journaling): retain one work loop across recovery retries --- .../grains/journaling/runtime-behavior.md | 2 + .../JournaledStateManager.cs | 56 ++++++++----- .../StateManagerLifecycleTests.cs | 82 +++++++++++++++++++ 3 files changed, 121 insertions(+), 19 deletions(-) diff --git a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md index 49a6b41e3a9..a1d1d278664 100644 --- a/docs/site/src/content/docs/grains/journaling/runtime-behavior.md +++ b/docs/site/src/content/docs/grains/journaling/runtime-behavior.md @@ -103,6 +103,8 @@ failure or after restoring the required format, codec, or backing data. Each att bookkeeping and replays the journal from the beginning using the same registered state machines. Concurrent callers share the active attempt; writes and deletion become available after initialization succeeds. State registration stays closed once initialization has begun. +One work-loop task owns recovery attempts and subsequent journal work for the manager's lifetime. +After a failed attempt it waits for an explicit initialization request before retrying. Owner shutdown which cancels initial recovery cancels all initialization waiters and leaves the manager stopped. Disposal waits for the owned read to finish before releasing journal resources. Cancelling an diff --git a/src/Orleans.Journaling/JournaledStateManager.cs b/src/Orleans.Journaling/JournaledStateManager.cs index 827a862b330..cbf6be752bc 100644 --- a/src/Orleans.Journaling/JournaledStateManager.cs +++ b/src/Orleans.Journaling/JournaledStateManager.cs @@ -156,15 +156,14 @@ public void RegisterStateMachine(string name, IStateMachine stateMachine) public async ValueTask InitializeAsync(CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); - _shutdownCancellation.Token.ThrowIfCancellationRequested(); Task task; bool didEnqueue; lock (_lock) { + _shutdownCancellation.Token.ThrowIfCancellationRequested(); ThrowIfFenced(); - if (_workLoop is null || _state is ManagerState.RecoveryFailed) + if (_workLoop is null) { - _state = ManagerState.Unknown; _workLoop = Start(); } @@ -188,30 +187,50 @@ private Task Start() private async Task WorkLoop() { await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ContinueOnCapturedContext | ConfigureAwaitOptions.ForceYielding); - try - { - await RecoverAsync(_shutdownCancellation.Token).ConfigureAwait(true); - } - catch (OperationCanceledException) when (_shutdownCancellation.IsCancellationRequested) - { - return; - } - catch (Exception exception) + while (!_shutdownCancellation.IsCancellationRequested) { try { - LogErrorProcessingWorkItems(_shared.Logger, exception); + await RecoverAsync(_shutdownCancellation.Token).ConfigureAwait(true); + _workSignal.Signal(); + break; } - finally + catch (OperationCanceledException) when (_shutdownCancellation.IsCancellationRequested) { - lock (_lock) + return; + } + catch (Exception exception) + { + try + { + LogErrorProcessingWorkItems(_shared.Logger, exception); + } + finally { - _state = ManagerState.RecoveryFailed; - FaultQueuedWorkItemsUnderLock(exception); + lock (_lock) + { + FaultQueuedWorkItemsUnderLock(exception); + } } } - return; + // Signals can remain from the failed attempt. Retry only for newly queued initialization work. + while (true) + { + await _workSignal.WaitAsync().ConfigureAwait(true); + if (_shutdownCancellation.IsCancellationRequested) + { + return; + } + + lock (_lock) + { + if (_workQueue.Count > 0) + { + break; + } + } + } } while (!_shutdownCancellation.Token.IsCancellationRequested) @@ -1199,7 +1218,6 @@ private sealed class RegisterStateWorkItem(string name) : WorkItem(name) private enum ManagerState : byte { Unknown, - RecoveryFailed, Ready, Fenced } diff --git a/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs b/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs index a0eae08c706..173d48b9bc4 100644 --- a/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs @@ -1,3 +1,4 @@ +using System.Reflection; using System.Threading.Channels; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -233,6 +234,87 @@ public async Task InitializationFailure_ReportsOriginalCauseAndAllowsRetry() Assert.Equal(1, state.WriteCompletedCount); } + [Fact] + public async Task RecoveryRetries_ReuseSingleWorkLoopTask() + { + var storage = new MutableReadStorage(3, [1, 2, 3], [1, 2, 3], CreatePersistedValueBytes("value", 42)); + await using var manager = CreateTestSystem(storage).Manager; + var value = new DurableValue("value", manager, CreateValueCodec()); + var initial = manager.InitializeAsync(CancellationToken.None).AsTask(); + var workLoop = GetWorkLoop(); + await Assert.ThrowsAsync(() => WaitFor(initial)); + Assert.Equal(1, storage.ReadCount); + + var retry = manager.InitializeAsync(CancellationToken.None).AsTask(); + await Assert.ThrowsAsync(() => WaitFor(retry)); + Assert.Same(workLoop, GetWorkLoop()); + Assert.False(workLoop.IsCompleted); + Assert.Equal(2, storage.ReadCount); + + const int callerCount = 8; + var start = NewSignal(); + var allEnqueued = NewSignal(); + var enqueued = 0; + var callers = Enumerable.Range(0, callerCount).Select(_ => Task.Run(async () => + { + await WaitFor(start.Task); + var initialization = manager.InitializeAsync(CancellationToken.None).AsTask(); + if (Interlocked.Increment(ref enqueued) == callerCount) allEnqueued.SetResult(); + await initialization; + }, TestContext.Current.CancellationToken)).ToArray(); + start.SetResult(); + await WaitFor(allEnqueued.Task); + await WaitFor(storage.BlockedReadStarted.Task); + Assert.Same(workLoop, GetWorkLoop()); + Assert.False(workLoop.IsCompleted); + Assert.Equal(3, storage.ReadCount); + Assert.All(callers, caller => Assert.False(caller.IsCompleted)); + + storage.AllowBlockedRead.SetResult(); + await WaitFor(Task.WhenAll(callers)); + Assert.Equal(42, value.Value); + value.Value = 43; + await manager.WriteStateAsync(TestContext.Current.CancellationToken); + Assert.Same(workLoop, GetWorkLoop()); + Assert.False(workLoop.IsCompleted); + Assert.Equal(3, storage.ReadCount); + + await WaitFor(manager.DisposeAsync().AsTask()); + Assert.Same(workLoop, GetWorkLoop()); + Assert.True(workLoop.IsCompletedSuccessfully); + + Task GetWorkLoop() => Assert.IsAssignableFrom( + typeof(JournaledStateManager).GetField("_workLoop", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(manager)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RecoveryFailure_WaitsForExplicitRetryOrShutdown(bool lifecycleStop) + { + var context = new QueuedSynchronizationContext(); + await context.Run(async () => + { + var storage = new MutableReadStorage([1, 2, 3], []); + var sut = CreateTestSystem(storage); + await using var manager = sut.Manager; + manager.RegisterStateMachine("state", new LifecycleState()); + await Assert.ThrowsAsync(() => lifecycleStop + ? sut.Lifecycle.OnStart(CancellationToken.None) + : manager.InitializeAsync(CancellationToken.None).AsTask()); + await Task.Yield(); + Assert.Equal(1, storage.ReadCount); + Assert.Empty(storage.OperationLog); + Assert.True(manager.TryGetStateMachine("state", out _)); + Assert.Throws(() => manager.RegisterStateMachine("late", new LifecycleState())); + + await WaitFor(lifecycleStop + ? sut.Lifecycle.OnStop(TestContext.Current.CancellationToken) + : manager.DisposeAsync().AsTask()); + Assert.Equal(1, storage.ReadCount); + }); + } + [Fact] public async Task RecoveryRetry_CoalescesCallersAndOwnsRead() { From 901d42432a8e7daa1308aa7a232f02e3f1adc67a Mon Sep 17 00:00:00 2001 From: Reuben Bond Date: Mon, 21 Sep 2026 14:05:55 -0700 Subject: [PATCH 11/11] test(journaling): clarify directory-only initialization flush --- test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs b/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs index 173d48b9bc4..abe57f4be4d 100644 --- a/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs +++ b/test/Orleans.Journaling.Tests/StateManagerLifecycleTests.cs @@ -144,7 +144,11 @@ public async Task ZeroByteWrite_DoesNotRepeatAcknowledgement() var state = new LifecycleState { EmitEntry = false }; manager.RegisterStateMachine("state", state); await manager.InitializeAsync(TestContext.Current.CancellationToken); + // Initialization records the state name even when the state emits no entries. + Assert.True(manager.PendingWriteByteCount > 0); await manager.WriteStateAsync(TestContext.Current.CancellationToken); + var directoryEntry = Assert.Single(ReadBinaryEntries(Assert.Single(storage.Appends))); + Assert.Equal(0u, directoryEntry.StreamId.Value); Assert.Equal(0, manager.PendingWriteByteCount); Assert.Equal(1, state.WriteCompletedCount); await manager.WriteStateAsync(TestContext.Current.CancellationToken);