From 02b4147889ee1d0f2ef84076a90c64fdbeb8f700 Mon Sep 17 00:00:00 2001 From: Elle Mouton Date: Mon, 13 Jul 2026 09:01:14 -0700 Subject: [PATCH 1/8] chainsource: reorg-aware conf/spend lifecycle + synthesized finality Add the reorg-aware chain-observation substrate the rest of the reorg-safety stack consumes: a seq-ordered conf/spend watch lifecycle (Confirmed -> Reorged -> re-Confirmed -> Done) with height-based finality synthesis for backends that cannot deliver a native Done (gRPC lndclient and lwwallet). Finality synthesis is armed off the select loop, tied to the sub-actor's long-lived context (a per-attempt timeout would tear the in-process block-epoch forwarder down the instant it armed), retried with a capped exponential backoff until it succeeds or the watch's context is cancelled (a single-confirmation tx has no later event to piggy-back a retry on), and evaluated against the best height captured at arm time so a tx already buried past FinalityDepth when the watch arms finalizes immediately instead of hanging for the next block. --- chainsource/backend.go | 91 +++- chainsource/chainsource.go | 28 +- chainsource/chainsource_test.go | 48 ++- chainsource/conf_actor.go | 419 ++++++++++++++++-- chainsource/epoch_drain_test.go | 76 ++++ chainsource/finality.go | 124 ++++++ chainsource/finality_test.go | 107 +++++ chainsource/messages.go | 99 +++++ chainsource/reorg_test.go | 735 ++++++++++++++++++++++++++++++++ chainsource/spend_actor.go | 335 ++++++++++++++- chainsource/transform.go | 38 ++ 11 files changed, 2043 insertions(+), 57 deletions(-) create mode 100644 chainsource/epoch_drain_test.go create mode 100644 chainsource/finality.go create mode 100644 chainsource/finality_test.go create mode 100644 chainsource/reorg_test.go diff --git a/chainsource/backend.go b/chainsource/backend.go index 3ef13b54e..7a6acfcaa 100644 --- a/chainsource/backend.go +++ b/chainsource/backend.go @@ -137,12 +137,41 @@ type ChainBackend interface { // ConfRegistration encapsulates the channels and control functions for a // confirmation registration. This mirrors lnd's chainntnfs.ConfirmationEvent // structure but provides a backend-agnostic interface. +// +// The registration is reorg-aware: after a confirmation has been delivered +// on Confirmed, a subsequent reorg that buries the original block can cause +// a fresh send on Reorged. Once the transaction re-confirms on the new +// canonical chain another event arrives on Confirmed. The lifecycle is +// therefore Confirmed -> Reorged -> Confirmed -> ... terminated by either a +// send on Done (the confirmation is past the backend's reorg-safety depth) +// or a caller-driven Cancel. Backends that cannot observe reorgs leave +// Reorged and Done as never-firing channels; callers must not assume either +// will ever fire. type ConfRegistration struct { - // Confirmed is a channel that fires once when the transaction reaches - // the target number of confirmations. The channel is buffered and will - // only send a single event. + // Confirmed fires every time the transaction reaches the target + // number of confirmations on the canonical chain. After a Reorged + // event it may fire again when the transaction re-confirms. The + // channel is buffered. Confirmed <-chan *TxConfirmation + // Reorged fires when a previously delivered confirmation is reorged + // out of the canonical chain. The payload is the backend forwarder's + // monotonic sequence number (see TxConfirmation.Seq) rather than + // reorg depth or block identity, which the lndclient gRPC transport + // cannot preserve; the sequence lets the consumer order this signal + // against confirmations sharing the same sequence space even though + // they arrive on a different channel. Backends that cannot observe + // reorgs leave this channel nil-equivalent (allocated but never + // written to); reading from it is always safe. + Reorged <-chan uint64 + + // Done fires once when the confirmation watch is past the backend's + // reorg-safety depth and will receive no further events. Callers + // must still invoke Cancel to release client-side resources. + // Backends that cannot synthesize a safety-depth signal leave this + // channel nil-equivalent (allocated but never written to). + Done <-chan struct{} + // Cancel is a function that can be called to cancel this registration // and clean up resources. After calling Cancel, no more events will be // sent on any channels. @@ -168,17 +197,54 @@ type TxConfirmation struct { // when the confirmation was registered with IncludeBlock=true. This // matches lnd's chainntnfs behavior. Block *wire.MsgBlock + + // Seq is a per-registration monotonic sequence number stamped by the + // backend forwarder in the order it observed lifecycle events. + // Confirmed and Reorged share one sequence space so the consumer can + // order them even though they arrive on separate channels (a select + // over two ready channels picks at random and cannot recover order). + // The consumer applies an event only when its Seq exceeds the highest + // Seq seen so far, discarding a stale event that lost a cross-channel + // race. Zero means the backend does not stamp sequences (it never + // reorgs, so ordering is moot); such events are always applied. + Seq uint64 } // SpendRegistration encapsulates the channels and control functions for a // spend registration. This mirrors lnd's chainntnfs.SpendEvent structure. +// +// The registration is reorg-aware: after a spend has been delivered on +// Spend, a subsequent reorg that buries the spending block can cause a +// fresh send on Reorged. If the outpoint is then re-spent on the new +// canonical chain (by the same or a different transaction) another event +// arrives on Spend. The lifecycle is therefore Spend -> Reorged -> Spend +// -> ... terminated by either a send on Done (the spend is past the +// backend's reorg-safety depth) or a caller-driven Cancel. Backends that +// cannot observe reorgs leave Reorged and Done as never-firing channels. type SpendRegistration struct { - // Spend is a channel that fires when the monitored outpoint is spent. - // The spending transaction must have at least one confirmation. The - // channel is buffered and will send an event for each spend (though - // typically only one unless reorgs occur). + // Spend fires every time the monitored outpoint is spent on the + // canonical chain. After a Reorged event it may fire again when a + // new spender confirms. The channel is buffered. Spend <-chan *SpendDetail + // Reorged fires when a previously delivered spend is reorged out of + // the canonical chain. The payload is the backend forwarder's + // monotonic sequence number (see SpendDetail.Seq) rather than reorg + // depth or block identity, which the lndclient gRPC transport cannot + // preserve; the sequence lets the consumer order this signal against + // spends sharing the same sequence space even though they arrive on a + // different channel. Backends that cannot observe spend reorgs leave + // this channel nil-equivalent (allocated but never written to); + // reading from it is always safe. + Reorged <-chan uint64 + + // Done fires once when the spend watch is past the backend's + // reorg-safety depth and will receive no further events. Callers + // must still invoke Cancel to release client-side resources. + // Backends that cannot synthesize a safety-depth signal leave this + // channel nil-equivalent (allocated but never written to). + Done <-chan struct{} + // Cancel is a function that can be called to cancel this registration // and clean up resources. Cancel func() @@ -203,6 +269,17 @@ type SpendDetail struct { // SpendingHeight is the block height where the spending transaction // was confirmed. SpendingHeight int32 + + // Seq is a per-registration monotonic sequence number stamped by the + // backend forwarder in the order it observed lifecycle events. Spend + // and Reorged share one sequence space so the consumer can order them + // even though they arrive on separate channels (a select over two + // ready channels picks at random and cannot recover order). The + // consumer applies an event only when its Seq exceeds the highest Seq + // seen so far, discarding a stale event that lost a cross-channel + // race. Zero means the backend does not stamp sequences (it never + // reorgs, so ordering is moot); such events are always applied. + Seq uint64 } // BlockRegistration encapsulates the channels and control functions for a diff --git a/chainsource/chainsource.go b/chainsource/chainsource.go index 44c6e7ce6..42f0cdde4 100644 --- a/chainsource/chainsource.go +++ b/chainsource/chainsource.go @@ -18,6 +18,16 @@ const ( // allows for buffering up to 10 blocks in transit, which should cover // normal block arrival patterns without blocking the backend. epochChannelSize = 10 + + // DefaultFinalityDepth is the conventional Bitcoin reorg-safety + // depth. After this many inclusive confirmations the ConfActor + // and SpendActor synthesize their Done events when the backend + // transport (notably lndclient over gRPC) does not surface one + // of its own. Six is the value the wider Lightning stack treats + // as final for the purposes of channel funding / settlement, so + // using it here keeps the unroll subsystem's finality threshold + // aligned with the rest of the daemon's chain assumptions. + DefaultFinalityDepth uint32 = 6 ) // ChainSourceConfig holds configuration for ChainSourceActor. @@ -32,6 +42,14 @@ type ChainSourceConfig struct { // falls back to extracting a logger from context via LoggerFromContext, // or uses btclog.Disabled if no logger is found. Log fn.Option[btclog.Logger] + + // FinalityDepth is forwarded to each spawned sub-actor as the + // number of confirmations past the first observed positive event + // that the actor uses to synthesize a Done signal when the backend + // transport (notably lndclient over gRPC) cannot deliver one. Zero + // disables height-based finality synthesis. See + // ConfActorConfig.FinalityDepth / SpendActorConfig.FinalityDepth. + FinalityDepth uint32 } // WithLogger returns a new config with the given logger set. @@ -318,8 +336,9 @@ func (a *ChainSourceActor) handleRegisterConf(ctx context.Context, ) confCfg := ConfActorConfig{ - Backend: a.cfg.Backend, - Log: fn.Some(a.logger(ctx)), + Backend: a.cfg.Backend, + Log: fn.Some(a.logger(ctx)), + FinalityDepth: a.cfg.FinalityDepth, } confActor := NewConfActor(confCfg) actorRef := serviceKey.Spawn(a.cfg.System, actorID, confActor) @@ -353,8 +372,9 @@ func (a *ChainSourceActor) handleRegisterSpend(ctx context.Context, ) spendCfg := SpendActorConfig{ - Backend: a.cfg.Backend, - Log: fn.Some(a.logger(ctx)), + Backend: a.cfg.Backend, + Log: fn.Some(a.logger(ctx)), + FinalityDepth: a.cfg.FinalityDepth, } spendActor := NewSpendActor(spendCfg) actorRef := serviceKey.Spawn(a.cfg.System, actorID, spendActor) diff --git a/chainsource/chainsource_test.go b/chainsource/chainsource_test.go index 822f4cb92..92b0d89d5 100644 --- a/chainsource/chainsource_test.go +++ b/chainsource/chainsource_test.go @@ -3,6 +3,7 @@ package chainsource import ( "context" "errors" + "sync/atomic" "testing" "time" @@ -17,12 +18,23 @@ import ( // mockBackend implements ChainBackend for testing. type mockBackend struct { - confChan chan *TxConfirmation - spendChan chan *SpendDetail - epochChan chan *BlockEpoch + confChan chan *TxConfirmation + confReorgedChan chan uint64 + confDoneChan chan struct{} + spendChan chan *SpendDetail + spendReorgedChan chan uint64 + spendDoneChan chan struct{} + epochChan chan *BlockEpoch epochCancel chan struct{} + // confCancelled / spendCancelled count Cancel invocations on the + // most recently issued registration. The reorg-aware lifecycle + // tests rely on these to assert that the actor released the + // registration after a Done event. + confCancelled atomic.Int32 + spendCancelled atomic.Int32 + feeRate btcutil.Amount bestHeight int32 bestHash chainhash.Hash @@ -42,12 +54,16 @@ type reconnectBlockBackend struct { // newMockBackend creates a new mock backend for testing. func newMockBackend() *mockBackend { return &mockBackend{ - confChan: make(chan *TxConfirmation, 1), - spendChan: make(chan *SpendDetail, 1), - epochChan: make(chan *BlockEpoch, 10), - epochCancel: make(chan struct{}, 10), - feeRate: 1000, - bestHeight: 100, + confChan: make(chan *TxConfirmation, 1), + confReorgedChan: make(chan uint64, 1), + confDoneChan: make(chan struct{}, 1), + spendChan: make(chan *SpendDetail, 1), + spendReorgedChan: make(chan uint64, 1), + spendDoneChan: make(chan struct{}, 1), + epochChan: make(chan *BlockEpoch, 10), + epochCancel: make(chan struct{}, 10), + feeRate: 1000, + bestHeight: 100, } } @@ -208,7 +224,11 @@ func (m *mockBackend) RegisterConf(ctx context.Context, txid *chainhash.Hash, return &ConfRegistration{ Confirmed: m.confChan, - Cancel: func() {}, + Reorged: m.confReorgedChan, + Done: m.confDoneChan, + Cancel: func() { + m.confCancelled.Add(1) + }, }, nil } @@ -217,8 +237,12 @@ func (m *mockBackend) RegisterSpend(ctx context.Context, *SpendRegistration, error) { return &SpendRegistration{ - Spend: m.spendChan, - Cancel: func() {}, + Spend: m.spendChan, + Reorged: m.spendReorgedChan, + Done: m.spendDoneChan, + Cancel: func() { + m.spendCancelled.Add(1) + }, }, nil } diff --git a/chainsource/conf_actor.go b/chainsource/conf_actor.go index d5d1ecfaa..e9e1b6852 100644 --- a/chainsource/conf_actor.go +++ b/chainsource/conf_actor.go @@ -22,6 +22,20 @@ type ConfActorConfig struct { // falls back to extracting a logger from context via LoggerFromContext, // or uses btclog.Disabled if no logger is found. Log fn.Option[btclog.Logger] + + // FinalityDepth is the number of confirmations past the first + // observed Confirmed event that the actor uses to synthesize a Done + // signal when the backend cannot deliver one. Zero disables + // height-based finality synthesis entirely; in that case the actor + // only fires ConfDoneEvent when the backend's own Done channel + // fires (e.g. an in-process lnd notifier). Non-zero values close + // the lndclient transport gap, where the gRPC layer does not + // surface lnd's internal "past reorg-safety depth" signal. + // + // The depth is counted inclusively (a tx confirmed at height H is + // at depth 1; height-based finality fires once the actor observes + // a block at H + FinalityDepth - 1). The conventional choice is 6. + FinalityDepth uint32 } // WithLogger returns a new config with the given logger set. @@ -67,9 +81,28 @@ type ConfActor struct { // mode. notifyActor fn.Option[actor.TellOnlyRef[ConfirmationEvent]] + // notifyReorged receives negative confirmation events in Actor mode. + notifyReorged fn.Option[actor.TellOnlyRef[ConfReorgedEvent]] + + // notifyDone receives finality events in Actor mode. + notifyDone fn.Option[actor.TellOnlyRef[ConfDoneEvent]] + // registration is the backend registration for this watch. registration *ConfRegistration + // blockReg is the block-epoch subscription used by height-based + // finality synthesis. Allocated lazily after the first Confirmed + // event when FinalityDepth > 0 and the registration is reorg-aware, + // torn down when the actor exits. + blockReg *BlockRegistration + + // confirmHeight records the block height the most recent + // Confirmed event arrived at. Used by the height-based finality + // synthesizer to compute current depth. Zero means there is no + // active confirmation to count from (either we have not yet seen + // one, or the last one was reorged out). + confirmHeight int32 + // ctx is the actor's internal context for cancellation, created from // context.Background() to ensure it outlives any request context. //nolint:containedctx @@ -143,6 +176,13 @@ func (a *ConfActor) handleRegisterConf(actorCtx context.Context, "than zero"), ) } + if req.NotifyActor.IsNone() && + (req.NotifyReorged.IsSome() || req.NotifyDone.IsSome()) { + return fn.Err[ConfResp]( + fmt.Errorf("confirmation reorg/done notifications " + + "require actor-mode NotifyActor"), + ) + } a.txid = req.Txid a.pkScript = req.PkScript @@ -150,6 +190,8 @@ func (a *ConfActor) handleRegisterConf(actorCtx context.Context, a.heightHint = req.HeightHint a.includeBlock = req.IncludeBlock a.notifyActor = req.NotifyActor + a.notifyReorged = req.NotifyReorged + a.notifyDone = req.NotifyDone // We're either in future or iterator mode, set the promise // accordingly. @@ -197,8 +239,8 @@ func (a *ConfActor) handleRegisterConf(actorCtx context.Context, } // monitorConfirmation runs in a background goroutine and waits for the target -// confirmation count to be reached. When reached, it delivers the event and -// exits. +// confirmation count to be reached. Legacy watches exit after confirmation, +// while reorg-aware actor-mode watches remain alive for reorg and done events. func (a *ConfActor) monitorConfirmation() { defer a.wg.Done() defer a.cancel() @@ -215,50 +257,324 @@ func (a *ConfActor) monitorConfirmation() { if a.registration != nil { a.registration.Cancel() } + if a.blockReg != nil { + a.blockReg.Cancel() + } }() - select { - case confDetails, ok := <-a.registration.Confirmed: - if !ok || confDetails == nil { - log.WarnS(a.ctx, "Confirmation subscription closed", - fmt.Errorf("channel closed or nil details"), + var lastEvent *ConfirmationEvent + reorgAware := a.notifyReorged.IsSome() || a.notifyDone.IsSome() + + // lastSeq is the highest backend forwarder sequence applied so far. + // Confirmed and Reorged signals arrive on separate channels and a + // select cannot order two ready channels, so we order them by the + // shared sequence instead: an event whose Seq does not exceed lastSeq + // lost a cross-channel race to a newer signal and is discarded. This + // makes the actor's view correct regardless of delivery interleaving + // — both reorg-then-reconfirm and confirm-then-reorg resolve to the + // highest-sequence outcome. Seq 0 means the backend does not stamp + // sequences (it never reorgs); those events are always applied. + var lastSeq uint64 + + // blockEpochs is rebound when height-based finality synthesis + // arms a block subscription. Until then a nil channel keeps the + // select arm parked. + var blockEpochs <-chan *BlockEpoch + + // blockRegCh hands a finality block subscription from the off-loop + // arming goroutine back to this loop; arming guards against launching + // more than one armer at a time. See armFinalityAsync for why arming + // runs off the select loop. + blockRegCh := make(chan finalityArmResult) + var arming bool + + for { + select { + case confDetails, ok := <-a.registration.Confirmed: + if !ok || confDetails == nil { + log.WarnS( + a.ctx, + "Confirmation subscription closed", + fmt.Errorf("channel closed or nil "+ + "details"), + ) + a.failConfirmation( + fmt.Errorf("confirmation " + + "subscription closed"), + ) + + return + } + + // Discard a confirmation that lost a cross-channel race + // to a newer reorg: an event whose sequence does not + // exceed the highest applied is stale. + if confDetails.Seq != 0 && confDetails.Seq <= lastSeq { + continue + } + if confDetails.Seq > lastSeq { + lastSeq = confDetails.Seq + } + + log.InfoS(a.ctx, "Received confirmation from backend", + "block_height", confDetails.BlockHeight, + "block_hash", confDetails.BlockHash, + ) + + event, err := buildConfirmationEvent(confDetails, a) + if err != nil { + log.WarnS( + a.ctx, + "Failed to build confirmation event", + err, + ) + a.failConfirmation(err) + + return + } + + log.InfoS(a.ctx, "Delivering confirmation event", + "txid", event.Txid, + "block_height", event.BlockHeight, ) - a.failConfirmation( - fmt.Errorf("confirmation subscription closed"), + a.deliverConfirmation(event) + lastEvent = &event + a.confirmHeight = event.BlockHeight + if !reorgAware || a.promise.IsSome() { + return + } + + // Arm height-based finality synthesis on the first + // confirmation if requested, off the select loop so + // the bounded RegisterBlocks retries cannot stall + // delivery of Reorged/Done/ctx.Done on this watch. A + // nil blockEpochs channel keeps the synthesis arm + // parked until the registration is handed back on + // blockRegCh. + if a.cfg.FinalityDepth > 0 && a.blockReg == nil && + !arming { + + arming = true + a.armFinalityAsync(blockRegCh, log) + } + + case armed := <-blockRegCh: + // Finality arming completed. Clear the flag; a nil reg + // only happens when the watch context was cancelled + // (arming otherwise retries until it succeeds). + arming = false + if armed.reg == nil { + continue + } + a.blockReg = armed.reg + blockEpochs = armed.reg.Epochs + + // The block-epoch subscription only delivers FUTURE + // epochs, but the confirmation that armed it may + // already be buried past FinalityDepth (it confirmed + // several blocks ago, or we re-armed after a restart). + // Use the tip observed at arm time to synthesize Done + // at once rather than hang until a fresh block is + // mined. The confirmHeight==0 / FinalityDepth==0 guards + // mirror the epoch handler below. + if a.confirmHeight == 0 || a.cfg.FinalityDepth == 0 { + continue + } + if armed.height-a.confirmHeight+1 < + int32(a.cfg.FinalityDepth) { + + continue + } + + log.InfoS(a.ctx, "Synthesizing confirmation done on arm "+ + "from height-based safety depth", + "confirm_height", a.confirmHeight, + "current_height", armed.height, + "finality_depth", int(a.cfg.FinalityDepth), ) + a.deliverConfDone(lastEvent) return - } - log.InfoS(a.ctx, "Received confirmation from backend", - "block_height", confDetails.BlockHeight, - "block_hash", confDetails.BlockHash, - ) + case seq, ok := <-a.registration.Reorged: + if !ok { + a.registration.Reorged = nil + continue + } + + // Discard a stale reorg that lost a cross-channel race + // to a newer confirmation. + if seq != 0 && seq <= lastSeq { + continue + } + if seq > lastSeq { + lastSeq = seq + } + + a.deliverConfReorged(lastEvent) + + // The previous confirmation is no longer on the + // canonical chain. Clear the cached event so a later + // Done cannot report the reorged-out txid, and reset + // the depth counter so the next re-confirmation starts + // a fresh window. + lastEvent = nil + a.confirmHeight = 0 + + case _, ok := <-a.registration.Done: + if !ok { + a.registration.Done = nil + continue + } + + a.deliverConfDone(lastEvent) - event, err := buildConfirmationEvent(confDetails, a) - if err != nil { - log.WarnS(a.ctx, "Failed to build confirmation event", - err, + return + + case epoch, ok := <-blockEpochs: + if !ok || epoch == nil { + blockEpochs = nil + continue + } + + // Coalesce any epochs already queued behind this one + // and evaluate finality against the most recent + // height only. With rapid-fire blocks (or a backend + // that re-delivers historical epochs) the channel can + // hold several epochs at once; processing them one per + // loop iteration would re-check the same monotonic + // Done condition repeatedly and risk synthesizing + // against a stale height. If the channel closed during + // the drain, park it so we stop selecting on it. + var closed bool + epoch, closed = drainToLatestEpoch(blockEpochs, epoch) + if closed { + blockEpochs = nil + } + + // The confirmHeight==0 guard is load-bearing: a reorg + // resets confirmHeight to 0 (see the Reorged case + // above), and a fresh epoch on the new tip would + // otherwise produce a negative depth (epoch.Height - 0 + // wraps to a large value); the depth comparison is + // meaningful only after a confirmation arms + // confirmHeight. FinalityDepth==0 disables synthesis + // entirely. + if a.confirmHeight == 0 || + a.cfg.FinalityDepth == 0 { + + continue + } + + depth := epoch.Height - a.confirmHeight + 1 + if depth < int32(a.cfg.FinalityDepth) { + continue + } + + log.InfoS(a.ctx, "Synthesizing confirmation done "+ + "from height-based safety depth", + "confirm_height", a.confirmHeight, + "current_height", epoch.Height, + "finality_depth", int(a.cfg.FinalityDepth), ) - a.failConfirmation(err) + + // Finality is terminal for the whole watch: deliver + // Done (a no-op when only NotifyReorged was set) and + // stop. A reorg-only watch therefore intentionally + // stops receiving reorg events once the confirmation is + // buried FinalityDepth deep — past that depth a reorg + // is beyond the safety threshold the watch was created + // to cover, so there is nothing left to observe. + a.deliverConfDone(lastEvent) return - } - log.InfoS(a.ctx, "Delivering confirmation event", - "txid", event.Txid, - "block_height", event.BlockHeight, - ) - a.deliverConfirmation(event) + case <-a.ctx.Done(): + log.InfoS(a.ctx, "ConfActor context cancelled") + a.failConfirmation(a.ctx.Err()) - case <-a.ctx.Done(): - log.InfoS(a.ctx, "ConfActor context cancelled") - a.failConfirmation(a.ctx.Err()) + return + } + } +} + +// drainToLatestEpoch non-blockingly consumes any block epochs already +// queued on ch and returns the most recent non-nil epoch, starting from +// cur. With rapid-fire blocks (or a backend that re-delivers historical +// epochs) several epochs can sit in the channel at once; height-based +// finality synthesis depends only on the highest observed height, so +// collapsing the backlog to the newest epoch avoids re-evaluating the +// same monotonic Done condition once per stale epoch. The returned bool +// reports whether the channel was observed closed during the drain so the +// caller can park its receive on a nil channel. +func drainToLatestEpoch(ch <-chan *BlockEpoch, + cur *BlockEpoch) (*BlockEpoch, bool) { + + latest := cur + for { + select { + case e, ok := <-ch: + if !ok { + return latest, true + } + if e != nil { + latest = e + } + + default: + return latest, false + } } } // deliverConfirmation sends the confirmation event to the subscriber and // completes the promise (Future mode) or sends to the actor (Actor mode). +// armFinalityAsync registers a block-epoch subscription for height-based +// finality synthesis off the actor's select loop. registerBlocksForFinality +// retries with a bounded backoff that can run for tens of seconds; doing it +// inline would block delivery of Reorged/Done/ctx.Done on this watch for the +// whole window. The registration (or nil on failure) is handed back on regCh, +// or cancelled if the actor exits before the loop reads it. The goroutine is +// tracked by the actor's wait group so Stop drains it. +func (a *ConfActor) armFinalityAsync(regCh chan<- finalityArmResult, + log btclog.Logger) { + + a.wg.Go(func() { + reg, err := registerBlocksForFinality(a.ctx, a.cfg.Backend, log) + if err != nil { + log.WarnS(a.ctx, "Giving up on height-based finality "+ + "synthesis; conf sub-actor will rely on "+ + "backend Done", err) + reg = nil + } + + // Capture the tip at arm time so the loop can finalize + // immediately when the arming confirmation is already buried + // past FinalityDepth (the block-epoch sub only delivers future + // epochs). A read failure is non-fatal: height stays zero and + // the loop falls back to waiting for the next epoch. + var height int32 + if reg != nil { + h, _, hErr := a.cfg.Backend.BestBlock(a.ctx) + if hErr != nil { + log.WarnS(a.ctx, "Failed to read best height "+ + "for on-arm finality check; will wait "+ + "for next epoch", hErr) + } else { + height = h + } + } + + select { + case regCh <- finalityArmResult{reg: reg, height: height}: + case <-a.ctx.Done(): + if reg != nil { + reg.Cancel() + } + } + }) +} + func (a *ConfActor) deliverConfirmation(event ConfirmationEvent) { a.promise.WhenSome(func(p actor.Promise[ConfirmationEvent]) { p.Complete(fn.Ok(event)) @@ -272,6 +588,53 @@ func (a *ConfActor) deliverConfirmation(event ConfirmationEvent) { }) } +// deliverConfReorged sends a reorg event to actor-mode subscribers. The +// correlation Txid is the registration's configured txid when set, since +// that is the identifier the caller asked us to watch; pkScript-only +// watches fall back to the txid carried on the most recent positive +// ConfirmationEvent. +func (a *ConfActor) deliverConfReorged(lastEvent *ConfirmationEvent) { + var event ConfReorgedEvent + switch { + case a.txid != nil: + event.Txid = *a.txid + + case lastEvent != nil: + event.Txid = lastEvent.Txid + } + + a.notifyReorged.WhenSome(func(ref actor.TellOnlyRef[ConfReorgedEvent]) { + log := a.logger(a.ctx) + if err := ref.Tell(a.ctx, event); err != nil { + log.WarnS(a.ctx, "Failed to deliver confirmation reorg", + err, + ) + } + }) +} + +// deliverConfDone sends a confirmation finality event to actor-mode +// subscribers. Txid follows the same precedence as deliverConfReorged. +func (a *ConfActor) deliverConfDone(lastEvent *ConfirmationEvent) { + var event ConfDoneEvent + switch { + case a.txid != nil: + event.Txid = *a.txid + + case lastEvent != nil: + event.Txid = lastEvent.Txid + } + + a.notifyDone.WhenSome(func(ref actor.TellOnlyRef[ConfDoneEvent]) { + log := a.logger(a.ctx) + if err := ref.Tell(a.ctx, event); err != nil { + log.WarnS(a.ctx, "Failed to deliver confirmation done", + err, + ) + } + }) +} + // failConfirmation completes the promise with an error (Future mode) or does // nothing (Actor mode - errors are not delivered in async mode). func (a *ConfActor) failConfirmation(err error) { diff --git a/chainsource/epoch_drain_test.go b/chainsource/epoch_drain_test.go new file mode 100644 index 000000000..9a439f84c --- /dev/null +++ b/chainsource/epoch_drain_test.go @@ -0,0 +1,76 @@ +package chainsource + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestDrainToLatestEpochCoalesces verifies that a backlog of queued block +// epochs collapses to the most recent one, so finality synthesis evaluates +// the highest observed height rather than re-checking once per stale epoch. +func TestDrainToLatestEpochCoalesces(t *testing.T) { + t.Parallel() + + ch := make(chan *BlockEpoch, 8) + cur := &BlockEpoch{Height: 100} + + // Queue several newer epochs behind the one already dequeued. + for h := int32(101); h <= 105; h++ { + ch <- &BlockEpoch{Height: h} + } + + got, closed := drainToLatestEpoch(ch, cur) + require.False(t, closed) + require.NotNil(t, got) + require.Equal(t, int32(105), got.Height) + + // The channel must be fully drained afterwards. + require.Len(t, ch, 0) +} + +// TestDrainToLatestEpochEmpty verifies that with nothing queued the helper +// returns the current epoch unchanged and reports the channel open. +func TestDrainToLatestEpochEmpty(t *testing.T) { + t.Parallel() + + ch := make(chan *BlockEpoch, 4) + cur := &BlockEpoch{Height: 42} + + got, closed := drainToLatestEpoch(ch, cur) + require.False(t, closed) + require.Same(t, cur, got) +} + +// TestDrainToLatestEpochSkipsNil verifies that nil epochs in the backlog are +// ignored: the most recent non-nil epoch wins. +func TestDrainToLatestEpochSkipsNil(t *testing.T) { + t.Parallel() + + ch := make(chan *BlockEpoch, 4) + cur := &BlockEpoch{Height: 10} + ch <- &BlockEpoch{Height: 11} + ch <- nil + + got, closed := drainToLatestEpoch(ch, cur) + require.False(t, closed) + require.NotNil(t, got) + require.Equal(t, int32(11), got.Height) +} + +// TestDrainToLatestEpochClosed verifies that a closed channel is reported so +// the caller can park its receive, while still returning the latest epoch +// observed before the close. +func TestDrainToLatestEpochClosed(t *testing.T) { + t.Parallel() + + ch := make(chan *BlockEpoch, 4) + cur := &BlockEpoch{Height: 7} + ch <- &BlockEpoch{Height: 8} + close(ch) + + got, closed := drainToLatestEpoch(ch, cur) + require.True(t, closed) + require.NotNil(t, got) + require.Equal(t, int32(8), got.Height) +} diff --git a/chainsource/finality.go b/chainsource/finality.go new file mode 100644 index 000000000..debed8eb5 --- /dev/null +++ b/chainsource/finality.go @@ -0,0 +1,124 @@ +package chainsource + +import ( + "context" + "time" + + "github.com/btcsuite/btclog/v2" +) + +const ( + // finalityArmInitialBackoff is the first delay between RegisterBlocks + // attempts when arming the block-epoch subscription that drives + // height-based finality synthesis. It doubles on each failure up to + // finalityArmMaxBackoff. + finalityArmInitialBackoff = 100 * time.Millisecond + + // finalityArmMaxBackoff caps the retry delay. Arming is retried + // indefinitely (until the watch's context is cancelled) rather than + // abandoned after a fixed count: for gRPC lndclient and lwwallet, + // height synthesis is the ONLY source of the terminal Done, so giving + // up would strand the round/exit in ProvisionallyConfirmed forever + // (a single-confirmation tx has no later event to trigger a re-arm). + // A brief backend outage at the arming moment therefore only delays + // finality, never permanently disables it. 30s stops a genuinely-down + // backend from spinning while bounding recovery latency once it heals. + finalityArmMaxBackoff = 30 * time.Second + + // finalityArmEscalateAfter is the consecutive-failure count past which + // the retry log escalates to an operator-visible "finality stalled" + // warning, so a persistently unarmable watch is detectable rather than + // silently stuck in provisional. + finalityArmEscalateAfter = 5 +) + +// finalityArmResult is handed from a conf/spend sub-actor's off-loop finality +// arming goroutine back to its select loop. reg is the block-epoch +// subscription (nil only when the watch context was cancelled). height is the +// best chain height observed at arm time (zero if it could not be read); the +// loop uses it to synthesize Done immediately when the arming +// confirmation/spend is already buried past FinalityDepth, rather than +// hanging until a fresh block epoch arrives (the subscription only delivers +// FUTURE epochs). +type finalityArmResult struct { + reg *BlockRegistration + height int32 +} + +// registerBlocksForFinality registers a block-epoch subscription used +// to synthesize a Done signal at FinalityDepth past an observed +// confirmation or spend. It retries RegisterBlocks indefinitely with a +// capped exponential backoff (until the passed context is cancelled) +// because finality synthesis is the only Done source for backends that +// do not write the upstream Done channel (notably lndclient over gRPC and +// lwwallet). Abandoning the arm after a fixed number of attempts would +// leak the per-watch sub-actor AND strand the round/exit in +// ProvisionallyConfirmed forever if the backend merely hiccups at the +// arming moment: a single-confirmation tx has no later event to trigger a +// re-arm, so the watch would only recover on a daemon restart. +// +// The retries run in a dedicated arming goroutine (not the sub-actor's +// select loop), so blocking here is safe: more confirmation/spend events +// on this specific watch are not expected during the retry window (we +// already consumed the one that triggered the arm), and ctx cancellation +// breaks out promptly. The goroutine is bounded by the watch's lifetime. +// +// The passed ctx MUST be the sub-actor's long-lived context, and it is +// handed to RegisterBlocks unwrapped: for in-process backends the +// block-epoch forwarder goroutine is tied to the ctx it receives, so +// bounding each attempt with a cancellable child ctx (and cancelling it +// once the call returns) would tear the subscription down the instant it +// was armed — starving finality synthesis of the very epochs it needs. +// +// Returns the registration on success, or a non-nil error only when the +// context is cancelled (the watch is shutting down). +func registerBlocksForFinality(ctx context.Context, backend ChainBackend, + log btclog.Logger) (*BlockRegistration, error) { + + backoff := finalityArmInitialBackoff + for attempt := 1; ; attempt++ { + reg, err := backend.RegisterBlocks(ctx) + if err == nil { + if attempt > 1 { + log.InfoS(ctx, "Finality block subscription "+ + "armed after retries", + "attempts", attempt, + ) + } + + return reg, nil + } + + // Height synthesis is the only finality source for gRPC + // lndclient / lwwallet, so a persistent arming failure stalls + // the round/exit in provisional. Escalate the log once retries + // pass finalityArmEscalateAfter so the stall is + // operator-visible rather than silent; the capped backoff + // throttles it. + if attempt >= finalityArmEscalateAfter { + log.WarnS(ctx, "Finality block subscription arming "+ + "persistently failing; round/exit finality is "+ + "stalled until the backend recovers", err, + "attempts", attempt, + "backoff", backoff, + ) + } else { + log.WarnS(ctx, "RegisterBlocks for finality synthesis "+ + "failed; retrying", err, + "attempt", attempt, + "backoff", backoff, + ) + } + + select { + case <-time.After(backoff): + case <-ctx.Done(): + return nil, ctx.Err() + } + + backoff *= 2 + if backoff > finalityArmMaxBackoff { + backoff = finalityArmMaxBackoff + } + } +} diff --git a/chainsource/finality_test.go b/chainsource/finality_test.go new file mode 100644 index 000000000..f3101089b --- /dev/null +++ b/chainsource/finality_test.go @@ -0,0 +1,107 @@ +package chainsource + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/btcsuite/btclog/v2" + "github.com/stretchr/testify/require" +) + +// flakyArmBackend fails RegisterBlocks the first failCount times, then +// succeeds. It models a backend that is briefly unavailable at the exact +// moment finality arming is attempted (the failure mode that previously +// stranded a watch once the bounded retry schedule was exhausted). +type flakyArmBackend struct { + *mockBackend + + mu sync.Mutex + failLeft int + attempts int +} + +// RegisterBlocks fails until failLeft is drained, then returns a live +// registration. +func (b *flakyArmBackend) RegisterBlocks(ctx context.Context) ( + *BlockRegistration, error) { + + b.mu.Lock() + defer b.mu.Unlock() + + b.attempts++ + if b.failLeft > 0 { + b.failLeft-- + + return nil, errors.New("backend temporarily unavailable") + } + + return &BlockRegistration{ + Epochs: make(chan *BlockEpoch, 1), + Cancel: func() {}, + }, nil +} + +// attemptCount returns how many RegisterBlocks calls have been made. +func (b *flakyArmBackend) attemptCount() int { + b.mu.Lock() + defer b.mu.Unlock() + + return b.attempts +} + +// TestRegisterBlocksForFinalityRetriesUntilArmed asserts that finality arming +// retries past the old fixed three-attempt cap and eventually arms once a +// transiently-unavailable backend recovers. This is the reorg-safety +// robustness property: for gRPC lndclient / lwwallet, height synthesis is the +// only Done source, so abandoning the arm would strand the round/exit in +// provisional until a daemon restart. failLeft is five (past both the old cap +// and finalityArmEscalateAfter) so the escalation-warning branch is exercised. +func TestRegisterBlocksForFinalityRetriesUntilArmed(t *testing.T) { + t.Parallel() + + backend := &flakyArmBackend{ + mockBackend: newMockBackend(), + failLeft: 5, + } + + reg, err := registerBlocksForFinality( + context.Background(), backend, btclog.Disabled, + ) + require.NoError(t, err, "arming must succeed once the backend recovers") + require.NotNil(t, reg, "a live block registration must be returned") + require.NotNil(t, reg.Epochs) + require.GreaterOrEqual( + t, backend.attemptCount(), 6, + "arming must retry past the old three-attempt cap", + ) +} + +// TestRegisterBlocksForFinalityStopsOnContextCancel asserts that the otherwise +// unbounded arming retry exits promptly when the watch's context is cancelled +// (daemon shutdown), returning the context error rather than looping forever. +func TestRegisterBlocksForFinalityStopsOnContextCancel(t *testing.T) { + t.Parallel() + + // A backend that never recovers, so arming would loop indefinitely + // were it not bounded by the context. + backend := &flakyArmBackend{ + mockBackend: newMockBackend(), + failLeft: 1 << 30, + } + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(150 * time.Millisecond) + cancel() + }() + + reg, err := registerBlocksForFinality(ctx, backend, btclog.Disabled) + require.Error( + t, err, "arming must return once the context is cancelled", + ) + require.ErrorIs(t, err, context.Canceled) + require.Nil(t, reg) +} diff --git a/chainsource/messages.go b/chainsource/messages.go index cd5eefba3..98e15ea91 100644 --- a/chainsource/messages.go +++ b/chainsource/messages.go @@ -261,6 +261,15 @@ type RegisterConfRequest struct { // events will be sent to this actor asynchronously. If None, a Future // is returned in the response for blocking await. NotifyActor fn.Option[actor.TellOnlyRef[ConfirmationEvent]] + + // NotifyReorged is an optional actor reference for negative + // confirmation events. It is only used in async actor mode. + NotifyReorged fn.Option[actor.TellOnlyRef[ConfReorgedEvent]] + + // NotifyDone is an optional actor reference notified when the + // confirmation watch is beyond the backend's reorg tracking horizon. + // It is only used in async actor mode. + NotifyDone fn.Option[actor.TellOnlyRef[ConfDoneEvent]] } // MessageType returns the message type identifier for logging and debugging. @@ -332,6 +341,48 @@ func (m ConfirmationEvent) MessageType() string { return "ConfirmationEvent" } +// ConfReorgedEvent is sent when a previously reported confirmation is +// reorged out of the canonical chain. After receiving this event a consumer +// should consider the prior confirmation no longer valid; if the transaction +// re-confirms on the new canonical chain a fresh ConfirmationEvent will +// follow on the same registration. +// +// No block hash, height, or reorg depth is carried because the lndclient +// gRPC transport does not preserve that information and we do not want to +// expose fields that are always zero in production. Consumers that need to +// invalidate cached block metadata should match on Txid and use the data +// from the most recent ConfirmationEvent they observed on this watch. +type ConfReorgedEvent struct { + actor.BaseMessage + + // Txid is the transaction ID whose confirmation was reorged. This + // matches the Txid carried on the originating ConfirmationEvent. + Txid chainhash.Hash +} + +// MessageType returns the message type identifier for logging and debugging. +func (m ConfReorgedEvent) MessageType() string { + return "ConfReorgedEvent" +} + +// ConfDoneEvent is sent when a confirmation watch has matured past the +// backend's reorg-safety depth and will receive no further events. +// Consumers may use this signal to drop any reorg-recovery bookkeeping they +// were holding for the registration. Not all backends synthesize this +// event; consumers must treat its absence as a normal operating condition +// rather than an error. +type ConfDoneEvent struct { + actor.BaseMessage + + // Txid is the transaction ID whose registration matured. + Txid chainhash.Hash +} + +// MessageType returns the message type identifier for logging and debugging. +func (m ConfDoneEvent) MessageType() string { + return "ConfDoneEvent" +} + // UnregisterConfRequest requests cancellation of a confirmation subscription. // The ChainSource actor uses the fields to construct the service key and // cancel the dedicated actor. @@ -417,6 +468,15 @@ type RegisterSpendRequest struct { // will be sent to this actor asynchronously. If None, a Future is // returned for blocking await. NotifyActor fn.Option[actor.TellOnlyRef[SpendEvent]] + + // NotifyReorged is an optional actor reference for spend reorg events. + // It is only used in async actor mode. + NotifyReorged fn.Option[actor.TellOnlyRef[SpendReorgedEvent]] + + // NotifyDone is an optional actor reference notified when the spend + // watch is beyond the backend's reorg tracking horizon. It is only used + // in async actor mode. + NotifyDone fn.Option[actor.TellOnlyRef[SpendDoneEvent]] } // MessageType returns the message type identifier for logging and debugging. @@ -481,6 +541,45 @@ func (m SpendEvent) MessageType() string { return "SpendEvent" } +// SpendReorgedEvent is sent when a previously reported spend is reorged out +// of the canonical chain. After receiving this event a consumer should +// consider the prior spend no longer valid; if the outpoint is re-spent on +// the new canonical chain a fresh SpendEvent will follow on the same +// registration. +// +// No spending txid, height, or block hash is carried because the lndclient +// gRPC transport does not preserve that information and we do not want to +// expose fields that are always zero in production. Consumers that need to +// invalidate cached spending metadata should match on Outpoint and use the +// data from the most recent SpendEvent they observed on this watch. +type SpendReorgedEvent struct { + actor.BaseMessage + + // Outpoint is the output whose spend was reorged. + Outpoint wire.OutPoint +} + +// MessageType returns the message type identifier for logging and debugging. +func (m SpendReorgedEvent) MessageType() string { + return "SpendReorgedEvent" +} + +// SpendDoneEvent is sent when a spend watch has matured past the backend's +// reorg-safety depth and will receive no further events. Not all backends +// synthesize this event; consumers must treat its absence as a normal +// operating condition rather than an error. +type SpendDoneEvent struct { + actor.BaseMessage + + // Outpoint is the output whose spend registration matured. + Outpoint wire.OutPoint +} + +// MessageType returns the message type identifier for logging and debugging. +func (m SpendDoneEvent) MessageType() string { + return "SpendDoneEvent" +} + // UnregisterSpendRequest requests cancellation of a spend subscription. // The ChainSource actor uses the fields to construct the service key and // cancel the dedicated actor. diff --git a/chainsource/reorg_test.go b/chainsource/reorg_test.go new file mode 100644 index 000000000..d045b8d49 --- /dev/null +++ b/chainsource/reorg_test.go @@ -0,0 +1,735 @@ +package chainsource + +import ( + "testing" + "time" + + "github.com/btcsuite/btcd/chainhash/v2" + "github.com/btcsuite/btcd/wire/v2" + "github.com/lightninglabs/darepo-client/baselib/actor" + "github.com/lightningnetwork/lnd/fn/v2" + "github.com/stretchr/testify/require" +) + +// awaitTimeout is the per-step wait used by the reorg lifecycle tests. It +// is generous enough to absorb scheduling jitter on overloaded CI machines +// but still short enough that a hung actor surfaces as a fast failure. +const awaitTimeout = 5 * time.Second + +// TestConfActorReorgAwareForwardsFullLifecycle drives the full +// Confirmed -> Reorged -> Confirmed -> Done sequence through a reorg-aware +// ConfActor and asserts each event is forwarded to the correct notify ref +// in order, and that the actor releases the backend registration after +// Done. +func TestConfActorReorgAwareForwardsFullLifecycle(t *testing.T) { + t.Parallel() + + backend := newMockBackend() + ctx := t.Context() + confActor := NewConfActor(ConfActorConfig{Backend: backend}) + defer confActor.Stop() + + txHash := chainhash.Hash{0x01} + confNotifier := actor.NewChannelTellOnlyRef[ConfirmationEvent]( + "conf-notify", 10, + ) + reorgNotifier := actor.NewChannelTellOnlyRef[ConfReorgedEvent]( + "conf-reorged", 10, + ) + doneNotifier := actor.NewChannelTellOnlyRef[ConfDoneEvent]( + "conf-done", 10, + ) + + var confRef actor.TellOnlyRef[ConfirmationEvent] = confNotifier + var reorgRef actor.TellOnlyRef[ConfReorgedEvent] = reorgNotifier + var doneRef actor.TellOnlyRef[ConfDoneEvent] = doneNotifier + + result := confActor.Receive(ctx, &RegisterConfRequest{ + CallerID: "test-conf-reorg-lifecycle", + Txid: &txHash, + PkScript: []byte{0x00, 0x14}, + TargetConfs: 1, + NotifyActor: fn.Some(confRef), + NotifyReorged: fn.Some(reorgRef), + NotifyDone: fn.Some(doneRef), + }) + require.True(t, result.IsOk()) + resp, err := result.Unpack() + require.NoError(t, err) + confResp, ok := resp.(*RegisterConfResponse) + require.True(t, ok) + + // Reorg-aware mode is actor-only, so the response must not carry a + // Future. + require.Nil(t, confResp.Future) + + // 1. First confirmation on the canonical chain. + blockHash1 := chainhash.Hash{0xaa} + backend.confChan <- &TxConfirmation{ + BlockHash: &blockHash1, + BlockHeight: 100, + Tx: wire.NewMsgTx(2), + } + + event1, ok := confNotifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for first ConfirmationEvent") + require.Equal(t, int32(100), event1.BlockHeight) + require.Equal(t, blockHash1, event1.BlockHash) + + // 2. Reorg evicts that confirmation. + backend.confReorgedChan <- uint64(0) + + reorgEvt, ok := reorgNotifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for ConfReorgedEvent") + require.Equal(t, txHash, reorgEvt.Txid) + + // 3. Transaction re-confirms in a different block on the new tip. + blockHash2 := chainhash.Hash{0xbb} + backend.confChan <- &TxConfirmation{ + BlockHash: &blockHash2, + BlockHeight: 101, + Tx: wire.NewMsgTx(2), + } + + event2, ok := confNotifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for re-ConfirmationEvent") + require.Equal(t, int32(101), event2.BlockHeight) + require.Equal(t, blockHash2, event2.BlockHash) + + // 4. Registration matures past reorg safety; backend fires Done. + backend.confDoneChan <- struct{}{} + + doneEvt, ok := doneNotifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for ConfDoneEvent") + require.Equal(t, txHash, doneEvt.Txid) + + // After Done the actor must release the registration. + require.Eventually(t, func() bool { + return backend.confCancelled.Load() >= 1 + }, awaitTimeout, 10*time.Millisecond, + "registration Cancel was never invoked after Done") +} + +// TestConfActorSynthesizesDoneFromFinalityDepth verifies that a +// reorg-aware ConfActor with a non-zero FinalityDepth fires +// ConfDoneEvent on its own once enough blocks have been observed past +// the first Confirmed event, even when the backend's Done channel +// never fires. This closes the lndclient gRPC gap, where lnd's +// internal "past reorg-safety depth" signal does not survive the +// transport. +func TestConfActorSynthesizesDoneFromFinalityDepth(t *testing.T) { + t.Parallel() + + backend := newMockBackend() + ctx := t.Context() + const finalityDepth = 6 + confActor := NewConfActor(ConfActorConfig{ + Backend: backend, + FinalityDepth: finalityDepth, + }) + defer confActor.Stop() + + txHash := chainhash.Hash{0x01} + confNotifier := actor.NewChannelTellOnlyRef[ConfirmationEvent]( + "conf-notify", 10, + ) + reorgNotifier := actor.NewChannelTellOnlyRef[ConfReorgedEvent]( + "conf-reorged", 10, + ) + doneNotifier := actor.NewChannelTellOnlyRef[ConfDoneEvent]( + "conf-done", 10, + ) + + var confRef actor.TellOnlyRef[ConfirmationEvent] = confNotifier + var reorgRef actor.TellOnlyRef[ConfReorgedEvent] = reorgNotifier + var doneRef actor.TellOnlyRef[ConfDoneEvent] = doneNotifier + + result := confActor.Receive(ctx, &RegisterConfRequest{ + CallerID: "test-conf-finality-depth", + Txid: &txHash, + PkScript: []byte{0x00, 0x14}, + TargetConfs: 1, + NotifyActor: fn.Some(confRef), + NotifyReorged: fn.Some(reorgRef), + NotifyDone: fn.Some(doneRef), + }) + require.True(t, result.IsOk()) + + // 1. First confirmation at height 100. This arms the height-based + // finality synthesizer. + blockHash1 := chainhash.Hash{0xaa} + backend.confChan <- &TxConfirmation{ + BlockHash: &blockHash1, + BlockHeight: 100, + Tx: wire.NewMsgTx(2), + } + _, ok := confNotifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for ConfirmationEvent") + + // 2. Push blocks up to height 104. Inclusive depth at that point + // is 5 (heights 100..104), one short of the finality threshold, + // so ConfDoneEvent MUST NOT have fired yet. + for height := int32(101); height <= + int32(100+finalityDepth-2); height++ { + + backend.epochChan <- &BlockEpoch{Height: height} + } + _, ok = doneNotifier.AwaitMessage(50 * time.Millisecond) + require.False(t, ok, "ConfDoneEvent fired before finality depth") + + // 3. One more block brings the inclusive depth to exactly + // FinalityDepth (heights 100..105). Done must fire now. + backend.epochChan <- &BlockEpoch{ + Height: 100 + int32(finalityDepth) - 1, + } + doneEvt, ok := doneNotifier.AwaitMessage(awaitTimeout) + require.True( + t, ok, + "ConfDoneEvent never fired despite reaching finality depth", + ) + require.Equal(t, txHash, doneEvt.Txid) + + // 4. After Done the actor exits and releases the registration. + require.Eventually(t, func() bool { + return backend.confCancelled.Load() >= 1 + }, awaitTimeout, 10*time.Millisecond, + "registration Cancel was never invoked after synthesized "+ + "Done") +} + +// TestConfActorDiscardsStaleReorgBySeq verifies that a reorg signal which +// lost a cross-channel race to a newer re-confirmation is discarded by +// sequence number. The re-confirmation (seq 3) is observed before the +// older reorg (seq 2); because Confirmed and Reorged arrive on separate +// channels the actor cannot order them by arrival, so it must order them +// by Seq and ignore the stale reorg. The proof is that height-based +// finality still fires against the re-confirmation height — if the stale +// reorg had been applied it would have reset confirmHeight to 0 and Done +// would never synthesize. +func TestConfActorDiscardsStaleReorgBySeq(t *testing.T) { + t.Parallel() + + backend := newMockBackend() + ctx := t.Context() + const finalityDepth = 6 + confActor := NewConfActor(ConfActorConfig{ + Backend: backend, + FinalityDepth: finalityDepth, + }) + defer confActor.Stop() + + txHash := chainhash.Hash{0x04} + confNotifier := actor.NewChannelTellOnlyRef[ConfirmationEvent]( + "conf-notify", 10, + ) + reorgNotifier := actor.NewChannelTellOnlyRef[ConfReorgedEvent]( + "conf-reorged", 10, + ) + doneNotifier := actor.NewChannelTellOnlyRef[ConfDoneEvent]( + "conf-done", 10, + ) + + var confRef actor.TellOnlyRef[ConfirmationEvent] = confNotifier + var reorgRef actor.TellOnlyRef[ConfReorgedEvent] = reorgNotifier + var doneRef actor.TellOnlyRef[ConfDoneEvent] = doneNotifier + + result := confActor.Receive(ctx, &RegisterConfRequest{ + CallerID: "test-conf-stale-reorg-seq", + Txid: &txHash, + PkScript: []byte{0x00, 0x14}, + TargetConfs: 1, + NotifyActor: fn.Some(confRef), + NotifyReorged: fn.Some(reorgRef), + NotifyDone: fn.Some(doneRef), + }) + require.True(t, result.IsOk()) + + // 1. First confirmation at height 100, seq 1. + blockHash1 := chainhash.Hash{0xaa} + backend.confChan <- &TxConfirmation{ + BlockHash: &blockHash1, + BlockHeight: 100, + Tx: wire.NewMsgTx(2), + Seq: 1, + } + _, ok := confNotifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for first ConfirmationEvent") + + // 2. The re-confirmation (seq 3) is delivered before the older reorg + // (seq 2) — the cross-channel race. It arms finality at height 101. + blockHash2 := chainhash.Hash{0xbb} + backend.confChan <- &TxConfirmation{ + BlockHash: &blockHash2, + BlockHeight: 101, + Tx: wire.NewMsgTx(2), + Seq: 3, + } + _, ok = confNotifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for re-ConfirmationEvent") + + // 3. The stale reorg (seq 2 <= 3) arrives late and must be discarded: + // no ConfReorgedEvent is delivered and confirmHeight is untouched. + backend.confReorgedChan <- uint64(2) + _, ok = reorgNotifier.AwaitMessage(50 * time.Millisecond) + require.False(t, ok, "stale reorg (seq 2) was not discarded") + + // 4. Drive blocks to the finality depth past height 101. Done must + // fire, proving confirmHeight survived the stale reorg. + for height := int32(102); height <= + int32(101+finalityDepth)-1; height++ { + + backend.epochChan <- &BlockEpoch{Height: height} + } + doneEvt, ok := doneNotifier.AwaitMessage(awaitTimeout) + require.True( + t, ok, "ConfDoneEvent never fired; stale reorg wrongly "+ + "reset confirmHeight", + ) + require.Equal(t, txHash, doneEvt.Txid) +} + +// TestConfActorDiscardsStaleConfirmBySeq verifies the opposite race: a +// re-confirmation that lost a cross-channel race to a newer reorg is +// discarded by sequence number. The reorg (seq 3) is observed before the +// stale confirmation (seq 2); the actor must ignore the confirmation and +// leave the watch unconfirmed, so height-based finality must NOT fire. +// Without sequence ordering the stale confirmation would set a non-zero +// confirmHeight and synthesize a false Done for a tx that is gone. +func TestConfActorDiscardsStaleConfirmBySeq(t *testing.T) { + t.Parallel() + + backend := newMockBackend() + ctx := t.Context() + const finalityDepth = 6 + confActor := NewConfActor(ConfActorConfig{ + Backend: backend, + FinalityDepth: finalityDepth, + }) + defer confActor.Stop() + + txHash := chainhash.Hash{0x05} + confNotifier := actor.NewChannelTellOnlyRef[ConfirmationEvent]( + "conf-notify", 10, + ) + reorgNotifier := actor.NewChannelTellOnlyRef[ConfReorgedEvent]( + "conf-reorged", 10, + ) + doneNotifier := actor.NewChannelTellOnlyRef[ConfDoneEvent]( + "conf-done", 10, + ) + + var confRef actor.TellOnlyRef[ConfirmationEvent] = confNotifier + var reorgRef actor.TellOnlyRef[ConfReorgedEvent] = reorgNotifier + var doneRef actor.TellOnlyRef[ConfDoneEvent] = doneNotifier + + result := confActor.Receive(ctx, &RegisterConfRequest{ + CallerID: "test-conf-stale-confirm-seq", + Txid: &txHash, + PkScript: []byte{0x00, 0x14}, + TargetConfs: 1, + NotifyActor: fn.Some(confRef), + NotifyReorged: fn.Some(reorgRef), + NotifyDone: fn.Some(doneRef), + }) + require.True(t, result.IsOk()) + + // 1. First confirmation at height 100, seq 1. + blockHash1 := chainhash.Hash{0xaa} + backend.confChan <- &TxConfirmation{ + BlockHash: &blockHash1, + BlockHeight: 100, + Tx: wire.NewMsgTx(2), + Seq: 1, + } + _, ok := confNotifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for first ConfirmationEvent") + + // 2. The newer reorg (seq 3) is observed first and resets the watch. + backend.confReorgedChan <- uint64(3) + _, ok = reorgNotifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for ConfReorgedEvent") + + // 3. The stale re-confirmation (seq 2 <= 3) arrives late and must be + // discarded: no ConfirmationEvent is delivered and the watch stays + // unconfirmed. + blockHash2 := chainhash.Hash{0xbb} + backend.confChan <- &TxConfirmation{ + BlockHash: &blockHash2, + BlockHeight: 101, + Tx: wire.NewMsgTx(2), + Seq: 2, + } + _, ok = confNotifier.AwaitMessage(50 * time.Millisecond) + require.False(t, ok, "stale confirmation (seq 2) was not discarded") + + // 4. Drive many blocks well past any finality window. Done must NOT + // fire because confirmHeight was never re-armed. + for height := int32(101); height <= int32(120); height++ { + backend.epochChan <- &BlockEpoch{Height: height} + } + _, ok = doneNotifier.AwaitMessage(100 * time.Millisecond) + require.False( + t, ok, "ConfDoneEvent fired for a reorged-out tx; stale "+ + "confirmation was wrongly applied", + ) +} + +// TestConfActorReorgAwareRejectsWithoutNotifyActor verifies that opting in +// to reorg-aware mode without an actor-mode confirmation ref is rejected at +// admission. Allowing it would silently drop every re-confirmation after +// the first, since a Future can only complete once. +func TestConfActorReorgAwareRejectsWithoutNotifyActor(t *testing.T) { + t.Parallel() + + backend := newMockBackend() + ctx := t.Context() + confActor := NewConfActor(ConfActorConfig{Backend: backend}) + defer confActor.Stop() + + txHash := chainhash.Hash{0x02} + reorgNotifier := actor.NewChannelTellOnlyRef[ConfReorgedEvent]( + "conf-reorged", 1, + ) + var reorgRef actor.TellOnlyRef[ConfReorgedEvent] = reorgNotifier + + result := confActor.Receive(ctx, &RegisterConfRequest{ + CallerID: "test-conf-reorg-no-notify", + Txid: &txHash, + PkScript: []byte{0x00, 0x14}, + TargetConfs: 1, + NotifyReorged: fn.Some(reorgRef), + }) + require.True(t, result.IsErr()) + _, err := result.Unpack() + require.ErrorContains( + t, err, + "reorg/done notifications require actor-mode NotifyActor", + ) +} + +// TestConfActorLegacyExitsAfterFirstConfirm ensures legacy Actor-mode +// subscribers (no NotifyReorged / NotifyDone) keep their historical +// single-shot contract even when the backend would later emit Reorged or +// Done. The actor must cancel the registration after the first +// confirmation. +func TestConfActorLegacyExitsAfterFirstConfirm(t *testing.T) { + t.Parallel() + + backend := newMockBackend() + ctx := t.Context() + confActor := NewConfActor(ConfActorConfig{Backend: backend}) + defer confActor.Stop() + + txHash := chainhash.Hash{0x03} + notifier := actor.NewChannelTellOnlyRef[ConfirmationEvent]( + "conf-notify", 10, + ) + var confRef actor.TellOnlyRef[ConfirmationEvent] = notifier + + result := confActor.Receive(ctx, &RegisterConfRequest{ + CallerID: "test-conf-legacy-single-shot", + Txid: &txHash, + PkScript: []byte{0x00, 0x14}, + TargetConfs: 1, + NotifyActor: fn.Some(confRef), + }) + require.True(t, result.IsOk()) + + // First confirmation goes through. + blockHash := chainhash.Hash{0x10} + backend.confChan <- &TxConfirmation{ + BlockHash: &blockHash, + BlockHeight: 200, + Tx: wire.NewMsgTx(2), + } + + event, ok := notifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for first ConfirmationEvent") + require.Equal(t, int32(200), event.BlockHeight) + + // Actor must have exited after the first confirmation, releasing the + // registration. + require.Eventually(t, func() bool { + return backend.confCancelled.Load() >= 1 + }, awaitTimeout, 10*time.Millisecond, + "legacy ConfActor did not cancel registration after first "+ + "confirmation") + + // No further events should reach the notifier. + _, ok = notifier.AwaitMessage(50 * time.Millisecond) + require.False( + t, ok, "legacy ConfActor delivered an unexpected second event", + ) +} + +// TestSpendActorReorgAwareForwardsFullLifecycle drives the spend lifecycle +// (Spend -> Reorged -> Spend -> Done) through a reorg-aware SpendActor and +// asserts every event is forwarded to the correct notify ref in order. +func TestSpendActorReorgAwareForwardsFullLifecycle(t *testing.T) { + t.Parallel() + + backend := newMockBackend() + ctx := t.Context() + spendActor := NewSpendActor(SpendActorConfig{Backend: backend}) + defer spendActor.Stop() + + outpoint := wire.OutPoint{Hash: chainhash.Hash{0x11}, Index: 0} + + spendNotifier := actor.NewChannelTellOnlyRef[SpendEvent]( + "spend-notify", 10, + ) + reorgNotifier := actor.NewChannelTellOnlyRef[SpendReorgedEvent]( + "spend-reorged", 10, + ) + doneNotifier := actor.NewChannelTellOnlyRef[SpendDoneEvent]( + "spend-done", 10, + ) + + var spendRef actor.TellOnlyRef[SpendEvent] = spendNotifier + var reorgRef actor.TellOnlyRef[SpendReorgedEvent] = reorgNotifier + var doneRef actor.TellOnlyRef[SpendDoneEvent] = doneNotifier + + result := spendActor.Receive(ctx, &RegisterSpendRequest{ + CallerID: "test-spend-reorg-lifecycle", + Outpoint: &outpoint, + PkScript: []byte{0x00, 0x14}, + NotifyActor: fn.Some(spendRef), + NotifyReorged: fn.Some(reorgRef), + NotifyDone: fn.Some(doneRef), + }) + require.True(t, result.IsOk()) + resp, err := result.Unpack() + require.NoError(t, err) + spendResp, ok := resp.(*RegisterSpendResponse) + require.True(t, ok) + require.Nil(t, spendResp.Future) + + // 1. First spend confirms. + spendingTx1 := wire.NewMsgTx(2) + hash1 := spendingTx1.TxHash() + backend.spendChan <- &SpendDetail{ + SpentOutPoint: &outpoint, + SpenderTxHash: &hash1, + SpendingTx: spendingTx1, + SpendingHeight: 150, + } + + spend1, ok := spendNotifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for first SpendEvent") + require.Equal(t, outpoint, spend1.Outpoint) + require.Equal(t, int32(150), spend1.SpendingHeight) + require.Equal(t, hash1, spend1.SpendingTxid) + + // 2. Reorg evicts that spend. + backend.spendReorgedChan <- uint64(0) + + reorgEvt, ok := reorgNotifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for SpendReorgedEvent") + require.Equal(t, outpoint, reorgEvt.Outpoint) + + // 3. A different spender wins the new chain. + spendingTx2 := wire.NewMsgTx(2) + spendingTx2.AddTxIn(&wire.TxIn{Sequence: 1}) + hash2 := spendingTx2.TxHash() + require.NotEqual(t, hash1, hash2) + backend.spendChan <- &SpendDetail{ + SpentOutPoint: &outpoint, + SpenderTxHash: &hash2, + SpendingTx: spendingTx2, + SpendingHeight: 151, + } + + spend2, ok := spendNotifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for re-SpendEvent") + require.Equal(t, outpoint, spend2.Outpoint) + require.Equal(t, int32(151), spend2.SpendingHeight) + require.Equal(t, hash2, spend2.SpendingTxid) + + // 4. Registration matures past reorg safety. + backend.spendDoneChan <- struct{}{} + + doneEvt, ok := doneNotifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for SpendDoneEvent") + require.Equal(t, outpoint, doneEvt.Outpoint) + + require.Eventually(t, func() bool { + return backend.spendCancelled.Load() >= 1 + }, awaitTimeout, 10*time.Millisecond, + "registration Cancel was never invoked after Done") +} + +// TestSpendActorSynthesizesDoneFromFinalityDepth mirrors the conf-side +// height-based finality test for the spend watch. A reorg-aware +// SpendActor with non-zero FinalityDepth fires SpendDoneEvent on its +// own once enough blocks have been observed past the first Spend, +// closing the same lndclient gRPC gap that ConfActor closes for +// confirmations. +func TestSpendActorSynthesizesDoneFromFinalityDepth(t *testing.T) { + t.Parallel() + + backend := newMockBackend() + ctx := t.Context() + const finalityDepth = 6 + spendActor := NewSpendActor(SpendActorConfig{ + Backend: backend, + FinalityDepth: finalityDepth, + }) + defer spendActor.Stop() + + outpoint := wire.OutPoint{Hash: chainhash.Hash{0x11}, Index: 0} + + spendNotifier := actor.NewChannelTellOnlyRef[SpendEvent]( + "spend-notify", 10, + ) + reorgNotifier := actor.NewChannelTellOnlyRef[SpendReorgedEvent]( + "spend-reorged", 10, + ) + doneNotifier := actor.NewChannelTellOnlyRef[SpendDoneEvent]( + "spend-done", 10, + ) + + var spendRef actor.TellOnlyRef[SpendEvent] = spendNotifier + var reorgRef actor.TellOnlyRef[SpendReorgedEvent] = reorgNotifier + var doneRef actor.TellOnlyRef[SpendDoneEvent] = doneNotifier + + result := spendActor.Receive(ctx, &RegisterSpendRequest{ + CallerID: "test-spend-finality-depth", + Outpoint: &outpoint, + PkScript: []byte{0x00, 0x14}, + NotifyActor: fn.Some(spendRef), + NotifyReorged: fn.Some(reorgRef), + NotifyDone: fn.Some(doneRef), + }) + require.True(t, result.IsOk()) + + // 1. First spend confirms at height 150. This arms the + // height-based finality synthesizer. + spendingTx := wire.NewMsgTx(2) + hash := spendingTx.TxHash() + backend.spendChan <- &SpendDetail{ + SpentOutPoint: &outpoint, + SpenderTxHash: &hash, + SpendingTx: spendingTx, + SpendingHeight: 150, + } + _, ok := spendNotifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for SpendEvent") + + // 2. Push blocks up to height 154. Inclusive depth at that point + // is 5 (heights 150..154), one short of the finality threshold. + // SpendDoneEvent MUST NOT have fired yet. + for height := int32(151); height <= + int32(150+finalityDepth-2); height++ { + + backend.epochChan <- &BlockEpoch{Height: height} + } + _, ok = doneNotifier.AwaitMessage(50 * time.Millisecond) + require.False(t, ok, "SpendDoneEvent fired before finality depth") + + // 3. One more block brings the inclusive depth to exactly + // FinalityDepth (heights 150..155). Done must fire now. + backend.epochChan <- &BlockEpoch{ + Height: 150 + int32(finalityDepth) - 1, + } + doneEvt, ok := doneNotifier.AwaitMessage(awaitTimeout) + require.True( + t, ok, + "SpendDoneEvent never fired despite reaching finality depth", + ) + require.Equal(t, outpoint, doneEvt.Outpoint) + + require.Eventually(t, func() bool { + return backend.spendCancelled.Load() >= 1 + }, awaitTimeout, 10*time.Millisecond, + "registration Cancel was never invoked after synthesized "+ + "Done") +} + +// TestSpendActorReorgAwareRejectsWithoutNotifyActor mirrors the conf-side +// admission check: opting in to reorg-aware spend forwarding without a +// NotifyActor must be rejected. +func TestSpendActorReorgAwareRejectsWithoutNotifyActor(t *testing.T) { + t.Parallel() + + backend := newMockBackend() + ctx := t.Context() + spendActor := NewSpendActor(SpendActorConfig{Backend: backend}) + defer spendActor.Stop() + + outpoint := wire.OutPoint{Hash: chainhash.Hash{0x21}, Index: 0} + reorgNotifier := actor.NewChannelTellOnlyRef[SpendReorgedEvent]( + "spend-reorged", 1, + ) + var reorgRef actor.TellOnlyRef[SpendReorgedEvent] = reorgNotifier + + result := spendActor.Receive(ctx, &RegisterSpendRequest{ + CallerID: "test-spend-reorg-no-notify", + Outpoint: &outpoint, + PkScript: []byte{0x00, 0x14}, + NotifyReorged: fn.Some(reorgRef), + }) + require.True(t, result.IsErr()) + _, err := result.Unpack() + require.ErrorContains( + t, err, + "reorg/done notifications require actor-mode NotifyActor", + ) +} + +// TestSpendActorLegacyExitsAfterFirstSpend exercises the legacy Actor-mode +// path: a watch that did not opt into the reorg lifecycle +// (NotifyReorged/NotifyDone both unset) is single-shot, exiting and +// releasing its backend registration after the first spend. This mirrors +// ConfActor's legacy contract and the documented invariant in CLAUDE.md; +// without the reorgAware gate such a watch would run forever and, with +// FinalityDepth > 0, arm a block subscription it never requested. +func TestSpendActorLegacyExitsAfterFirstSpend(t *testing.T) { + t.Parallel() + + backend := newMockBackend() + ctx := t.Context() + spendActor := NewSpendActor(SpendActorConfig{Backend: backend}) + defer spendActor.Stop() + + outpoint := wire.OutPoint{Hash: chainhash.Hash{0x31}, Index: 0} + notifier := actor.NewChannelTellOnlyRef[SpendEvent]( + "spend-notify", 10, + ) + var spendRef actor.TellOnlyRef[SpendEvent] = notifier + + result := spendActor.Receive(ctx, &RegisterSpendRequest{ + CallerID: "test-spend-legacy-single-shot", + Outpoint: &outpoint, + PkScript: []byte{0x00, 0x14}, + NotifyActor: fn.Some(spendRef), + }) + require.True(t, result.IsOk()) + + // First spend goes through. + spendingTx := wire.NewMsgTx(2) + hash := spendingTx.TxHash() + backend.spendChan <- &SpendDetail{ + SpentOutPoint: &outpoint, + SpenderTxHash: &hash, + SpendingTx: spendingTx, + SpendingHeight: 300, + } + + first, ok := notifier.AwaitMessage(awaitTimeout) + require.True(t, ok) + require.Equal(t, int32(300), first.SpendingHeight) + + // The actor must have exited after the first spend, releasing the + // registration. + require.Eventually(t, func() bool { + return backend.spendCancelled.Load() >= 1 + }, awaitTimeout, 10*time.Millisecond, + "legacy SpendActor did not cancel registration after first "+ + "spend") + + // No further events should reach the notifier. + _, ok = notifier.AwaitMessage(50 * time.Millisecond) + require.False( + t, ok, "legacy SpendActor delivered an unexpected second event", + ) +} diff --git a/chainsource/spend_actor.go b/chainsource/spend_actor.go index 287d1b3e3..4dbaa267a 100644 --- a/chainsource/spend_actor.go +++ b/chainsource/spend_actor.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "sync" + "time" "github.com/btcsuite/btcd/chainhash/v2" "github.com/btcsuite/btcd/wire/v2" @@ -23,6 +24,15 @@ type SpendActorConfig struct { // falls back to extracting a logger from context via LoggerFromContext, // or uses btclog.Disabled if no logger is found. Log fn.Option[btclog.Logger] + + // FinalityDepth is the number of confirmations past the first + // observed Spend event that the actor uses to synthesize a Done + // signal when the backend cannot deliver one. See the matching + // field on ConfActorConfig for the rationale; the same constraint + // applies on the spend watch — lnd's chainntnfs.SpendEvent.Done + // does not survive the lndclient gRPC transport, so consumers + // that gate eviction on Done would otherwise leak per-spend state. + FinalityDepth uint32 } // WithLogger returns a new config with the given logger set. @@ -61,9 +71,26 @@ type SpendActor struct { // mode. notifyActor fn.Option[actor.TellOnlyRef[SpendEvent]] + // notifyReorged receives spend reorg events in Actor mode. + notifyReorged fn.Option[actor.TellOnlyRef[SpendReorgedEvent]] + + // notifyDone receives spend finality events in Actor mode. + notifyDone fn.Option[actor.TellOnlyRef[SpendDoneEvent]] + // registration is the backend registration for this watch. registration *SpendRegistration + // blockReg is the block-epoch subscription used by height-based + // finality synthesis. Allocated lazily after the first Spend + // event when FinalityDepth > 0, torn down when the actor exits. + blockReg *BlockRegistration + + // spendHeight records the block height the most recent Spend + // event arrived at. Zero means there is no active spend to count + // from (either we have not yet seen one, or the last one was + // reorged out). + spendHeight int32 + // ctx is the actor's internal context for cancellation, created from // context.Background() to ensure it outlives any request context. //nolint:containedctx @@ -132,12 +159,21 @@ func (a *SpendActor) handleRegisterSpend(actorCtx context.Context, "provided"), ) } + if req.NotifyActor.IsNone() && + (req.NotifyReorged.IsSome() || req.NotifyDone.IsSome()) { + return fn.Err[SpendResp]( + fmt.Errorf("spend reorg/done notifications require " + + "actor-mode NotifyActor"), + ) + } // Configure the actor with request parameters. a.outpoint = req.Outpoint a.pkScript = req.PkScript a.heightHint = req.HeightHint a.notifyActor = req.NotifyActor + a.notifyReorged = req.NotifyReorged + a.notifyDone = req.NotifyDone // Create promise for Future mode. var promise fn.Option[actor.Promise[SpendEvent]] @@ -152,10 +188,17 @@ func (a *SpendActor) handleRegisterSpend(actorCtx context.Context, // Register with the backend to receive spend notifications. We do this // before starting the goroutine so we can return an error to the - // caller if registration fails. + // caller if registration fails. The bounded timeout mirrors + // ConfActor.handleRegisterConf: a backend (LND) that is slow under + // heavy block processing load must not pin the parent Receive call + // indefinitely, since that would back-pressure the chainsource + // factory actor onto every other in-flight registration. + regCtx, regCancel := context.WithTimeout(a.ctx, 10*time.Second) + defer regCancel() + //nolint:contextcheck // actor root context owns registration lifetime registration, err := a.cfg.Backend.RegisterSpend( - a.ctx, a.outpoint, a.pkScript, a.heightHint, + regCtx, a.outpoint, a.pkScript, a.heightHint, ) if err != nil { return fn.Err[SpendResp]( @@ -191,10 +234,47 @@ func (a *SpendActor) monitorSpend() { if a.registration != nil { a.registration.Cancel() } + if a.blockReg != nil { + a.blockReg.Cancel() + } }() + log := a.logger(a.ctx) + // Monitor for spends indefinitely until cancelled or shutdown. // This allows us to catch re-org events where a spend is replaced. + var lastEvent *SpendEvent + + // reorgAware reports whether the caller opted into the multi-shot + // reorg lifecycle (at least one of NotifyReorged/NotifyDone). When + // false the watch is single-shot for backwards compatibility: the + // actor exits after the first spend, mirroring ConfActor. Without + // this gate a plain actor-mode spend watch would run forever and, with + // FinalityDepth > 0, arm a block subscription it never asked for. + reorgAware := a.notifyReorged.IsSome() || a.notifyDone.IsSome() + + // lastSeq is the highest backend forwarder sequence applied so far. + // Spend and Reorged signals arrive on separate channels and a select + // cannot order two ready channels, so we order them by the shared + // sequence instead: an event whose Seq does not exceed lastSeq lost a + // cross-channel race to a newer signal and is discarded. This makes + // the actor's view correct regardless of delivery interleaving. Seq 0 + // means the backend does not stamp sequences (it never reorgs); those + // events are always applied. + var lastSeq uint64 + + // blockEpochs is rebound when height-based finality synthesis + // arms a block subscription. Until then a nil channel keeps the + // select arm parked. + var blockEpochs <-chan *BlockEpoch + + // blockRegCh hands a finality block subscription from the off-loop + // arming goroutine back to this loop; arming guards against launching + // more than one armer at a time. See armFinalityAsync for why arming + // runs off the select loop. + blockRegCh := make(chan finalityArmResult) + var arming bool + for { select { case spend, ok := <-a.registration.Spend: @@ -206,6 +286,16 @@ func (a *SpendActor) monitorSpend() { return } + // Discard a spend that lost a cross-channel race to a + // newer reorg: an event whose sequence does not exceed + // the highest applied is stale. + if spend.Seq != 0 && spend.Seq <= lastSeq { + continue + } + if spend.Seq > lastSeq { + lastSeq = spend.Seq + } + event, err := buildSpendEvent(spend, a) if err != nil { a.failSpend(err) @@ -215,13 +305,151 @@ func (a *SpendActor) monitorSpend() { // Deliver the event. a.deliverSpend(event) - - // In Future mode, exit after first event. In Actor - // mode, continue monitoring for re-org events. - if a.promise.IsSome() { + lastEvent = &event + a.spendHeight = event.SpendingHeight + + // Exit after the first event in Future mode, and in + // actor mode that did not opt into the reorg lifecycle + // (single-shot backwards-compatible contract). Only a + // reorg-aware actor watch keeps monitoring. + if !reorgAware || a.promise.IsSome() { return } + // Arm height-based finality synthesis on the first + // spend if requested, off the select loop so the + // bounded RegisterBlocks retries cannot stall delivery + // of Reorged/Done/ctx.Done on this watch. A nil + // blockEpochs channel keeps the synthesis arm parked + // until the registration is handed back on blockRegCh. + if a.cfg.FinalityDepth > 0 && a.blockReg == nil && + !arming { + + arming = true + a.armFinalityAsync(blockRegCh, log) + } + + case armed := <-blockRegCh: + // Finality arming completed. Clear the flag; a nil reg + // only happens when the watch context was cancelled + // (arming otherwise retries until it succeeds). + arming = false + if armed.reg == nil { + continue + } + a.blockReg = armed.reg + blockEpochs = armed.reg.Epochs + + // The block-epoch subscription only delivers FUTURE + // epochs, but the spend that armed it may already be + // buried past FinalityDepth (it confirmed several + // blocks ago, or we re-armed after a restart). Use the + // tip observed at arm time to synthesize Done at once + // rather than hang until a fresh block is mined. The + // spendHeight==0 / FinalityDepth==0 guards mirror the + // epoch handler below. + if a.spendHeight == 0 || a.cfg.FinalityDepth == 0 { + continue + } + if armed.height-a.spendHeight+1 < + int32(a.cfg.FinalityDepth) { + + continue + } + + log.InfoS(a.ctx, "Synthesizing spend done on arm from "+ + "height-based safety depth", + "spend_height", a.spendHeight, + "current_height", armed.height, + "finality_depth", int(a.cfg.FinalityDepth), + ) + a.deliverSpendDone(lastEvent) + + return + + case seq, ok := <-a.registration.Reorged: + if !ok { + a.registration.Reorged = nil + continue + } + + // Discard a stale reorg that lost a cross-channel race + // to a newer spend. + if seq != 0 && seq <= lastSeq { + continue + } + if seq > lastSeq { + lastSeq = seq + } + + a.deliverSpendReorged(lastEvent) + + // The previous spend is no longer on the canonical + // chain. Clear the cached event so a later Done cannot + // report the reorged-out outpoint, and reset the depth + // counter so the next re-spend starts a fresh window. + lastEvent = nil + a.spendHeight = 0 + + case _, ok := <-a.registration.Done: + if !ok { + a.registration.Done = nil + continue + } + + a.deliverSpendDone(lastEvent) + + return + + case epoch, ok := <-blockEpochs: + if !ok || epoch == nil { + blockEpochs = nil + continue + } + + // Coalesce any epochs already queued behind this one + // and evaluate finality against the most recent height + // only. With rapid-fire blocks the channel can hold + // several epochs at once; processing them one per loop + // iteration would re-check the same monotonic Done + // condition repeatedly and risk synthesizing against a + // stale height. If the channel closed during the drain, + // park it so we stop selecting on it. + var closed bool + epoch, closed = drainToLatestEpoch(blockEpochs, epoch) + if closed { + blockEpochs = nil + } + + // The spendHeight==0 guard is load-bearing: a reorg + // resets spendHeight to 0 (the Reorged arm above), so + // a fresh epoch arriving before the re-spend would + // otherwise compute depth against a zero base and + // could synthesize Done prematurely. While + // spendHeight==0 there is no active spend to count + // from, so the depth comparison is meaningless. + // FinalityDepth==0 disables synthesis entirely. + if a.spendHeight == 0 || + a.cfg.FinalityDepth == 0 { + + continue + } + + depth := epoch.Height - a.spendHeight + 1 + if depth < int32(a.cfg.FinalityDepth) { + continue + } + + log.InfoS(a.ctx, "Synthesizing spend done from "+ + "height-based safety depth", + "spend_height", a.spendHeight, + "current_height", epoch.Height, + "finality_depth", int(a.cfg.FinalityDepth), + ) + a.deliverSpendDone(lastEvent) + + return + case <-a.ctx.Done(): // Actor was cancelled. a.failSpend(a.ctx.Err()) @@ -231,6 +459,52 @@ func (a *SpendActor) monitorSpend() { } } +// armFinalityAsync registers a block-epoch subscription for height-based +// finality synthesis off the actor's select loop. registerBlocksForFinality +// retries with a bounded backoff that can run for tens of seconds; doing it +// inline would block delivery of Reorged/Done/ctx.Done on this watch for the +// whole window. The registration (or nil on failure) is handed back on regCh, +// or cancelled if the actor exits before the loop reads it. The goroutine is +// tracked by the actor's wait group so Stop drains it. +func (a *SpendActor) armFinalityAsync(regCh chan<- finalityArmResult, + log btclog.Logger) { + + a.wg.Go(func() { + reg, err := registerBlocksForFinality(a.ctx, a.cfg.Backend, log) + if err != nil { + log.WarnS(a.ctx, "Giving up on height-based finality "+ + "synthesis; spend sub-actor will rely on "+ + "backend Done", err) + reg = nil + } + + // Capture the tip at arm time so the loop can finalize + // immediately when the arming spend is already buried past + // FinalityDepth (the block-epoch sub only delivers future + // epochs). A read failure is non-fatal: height stays zero and + // the loop falls back to waiting for the next epoch. + var height int32 + if reg != nil { + h, _, hErr := a.cfg.Backend.BestBlock(a.ctx) + if hErr != nil { + log.WarnS(a.ctx, "Failed to read best height "+ + "for on-arm finality check; will wait "+ + "for next epoch", hErr) + } else { + height = h + } + } + + select { + case regCh <- finalityArmResult{reg: reg, height: height}: + case <-a.ctx.Done(): + if reg != nil { + reg.Cancel() + } + } + }) +} + // deliverSpend delivers a spend event to the subscriber. In Future mode, it // completes the promise. In Actor mode, it sends to the registered actor. func (a *SpendActor) deliverSpend(event SpendEvent) { @@ -249,6 +523,55 @@ func (a *SpendActor) deliverSpend(event SpendEvent) { }) } +// deliverSpendReorged delivers a spend reorg event to actor-mode +// subscribers. The correlation Outpoint is the registration's configured +// outpoint when set, since that is the identifier the caller asked us to +// watch; pkScript-only watches fall back to the outpoint carried on the +// most recent positive SpendEvent. +func (a *SpendActor) deliverSpendReorged(lastEvent *SpendEvent) { + var event SpendReorgedEvent + switch { + case a.outpoint != nil: + event.Outpoint = *a.outpoint + + case lastEvent != nil: + event.Outpoint = lastEvent.Outpoint + } + + a.notifyReorged.WhenSome( + func(ref actor.TellOnlyRef[SpendReorgedEvent]) { + log := a.logger(a.ctx) + if err := ref.Tell(a.ctx, event); err != nil { + log.WarnS( + a.ctx, + "Failed to deliver spend reorg", + err, + ) + } + }, + ) +} + +// deliverSpendDone delivers a spend finality event to actor-mode subscribers. +// Outpoint follows the same precedence as deliverSpendReorged. +func (a *SpendActor) deliverSpendDone(lastEvent *SpendEvent) { + var event SpendDoneEvent + switch { + case a.outpoint != nil: + event.Outpoint = *a.outpoint + + case lastEvent != nil: + event.Outpoint = lastEvent.Outpoint + } + + a.notifyDone.WhenSome(func(ref actor.TellOnlyRef[SpendDoneEvent]) { + log := a.logger(a.ctx) + if err := ref.Tell(a.ctx, event); err != nil { + log.WarnS(a.ctx, "Failed to deliver spend done", err) + } + }) +} + // failSpend completes the promise with an error (Future mode) or does nothing // (Actor mode - errors are not delivered in async mode). func (a *SpendActor) failSpend(err error) { diff --git a/chainsource/transform.go b/chainsource/transform.go index 16f9f164a..9e392ae62 100644 --- a/chainsource/transform.go +++ b/chainsource/transform.go @@ -33,6 +33,25 @@ func MapConfirmationEvent[Out actor.Message]( return actor.NewMapInputRef(targetRef, mapFn) } +// MapConfReorgedEvent creates a transformed TellOnlyRef that accepts +// ConfReorgedEvent and transforms it to the caller's desired output message +// type. +func MapConfReorgedEvent[Out actor.Message]( + targetRef actor.TellOnlyRef[Out], mapFn func(ConfReorgedEvent) Out, +) actor.TellOnlyRef[ConfReorgedEvent] { + + return actor.NewMapInputRef(targetRef, mapFn) +} + +// MapConfDoneEvent creates a transformed TellOnlyRef that accepts +// ConfDoneEvent and transforms it to the caller's desired output message type. +func MapConfDoneEvent[Out actor.Message]( + targetRef actor.TellOnlyRef[Out], mapFn func(ConfDoneEvent) Out, +) actor.TellOnlyRef[ConfDoneEvent] { + + return actor.NewMapInputRef(targetRef, mapFn) +} + // MapSpendEvent creates a transformed TellOnlyRef that accepts SpendEvent and // transforms it to the caller's desired output message type. This is a // convenience wrapper for the common pattern of adapting chainsource spend @@ -64,6 +83,25 @@ func MapSpendEvent[Out actor.Message]( return actor.NewMapInputRef(targetRef, mapFn) } +// MapSpendReorgedEvent creates a transformed TellOnlyRef that accepts +// SpendReorgedEvent and transforms it to the caller's desired output message +// type. +func MapSpendReorgedEvent[Out actor.Message]( + targetRef actor.TellOnlyRef[Out], mapFn func(SpendReorgedEvent) Out, +) actor.TellOnlyRef[SpendReorgedEvent] { + + return actor.NewMapInputRef(targetRef, mapFn) +} + +// MapSpendDoneEvent creates a transformed TellOnlyRef that accepts +// SpendDoneEvent and transforms it to the caller's desired output message type. +func MapSpendDoneEvent[Out actor.Message]( + targetRef actor.TellOnlyRef[Out], mapFn func(SpendDoneEvent) Out, +) actor.TellOnlyRef[SpendDoneEvent] { + + return actor.NewMapInputRef(targetRef, mapFn) +} + // MapBlockEpoch creates a transformed TellOnlyRef that accepts BlockEpoch // and transforms it to the caller's desired output message type. This is a // convenience wrapper for the common pattern of adapting chainsource block From a2b96177ad0b144b01e30527c7b6ed39edc041cb Mon Sep 17 00:00:00 2001 From: Elle Mouton Date: Mon, 13 Jul 2026 09:01:14 -0700 Subject: [PATCH 2/8] txconfirm: reorg-aware confirmation watches with terminal seal Deliver the full TxConfirmed/TxReorged/re-TxConfirmed/TxFinalized/ TxFailed cycle on confirmation watches, with a terminal seal guarding the reversible fire-and-forget deliveries and a catch-up TxReorged when a reorg lands while a subscriber's initial TxConfirmed is still parked on the async notify path. --- txconfirm/actor.go | 711 +++++++++++++++++++++++++++++--- txconfirm/actor_test.go | 342 +++++++++++++-- txconfirm/broadcaster_test.go | 10 +- txconfirm/fsm_types.go | 33 +- txconfirm/funded_anchor_test.go | 24 +- txconfirm/messages.go | 83 +++- txconfirm/reorg_test.go | 147 +++++++ txconfirm/states.go | 66 ++- 8 files changed, 1296 insertions(+), 120 deletions(-) create mode 100644 txconfirm/reorg_test.go diff --git a/txconfirm/actor.go b/txconfirm/actor.go index a2d88a491..0fff91500 100644 --- a/txconfirm/actor.go +++ b/txconfirm/actor.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "log/slog" + "sync/atomic" "time" "github.com/btcsuite/btcd/chainhash/v2" @@ -52,6 +53,14 @@ var ( // waiting longer only risks blocking unrelated confirmation work behind // a durable subscriber's DB writer. terminalNotifyTimeout = time.Second + + // reversibleNotifyTimeout bounds how long a fire-and-forget reversible + // notification goroutine (TxConfirmed, TxReorged) is willing to wait + // on a slow subscriber's mailbox before logging the drop and returning. + // Reversible deliveries are best-effort: the next state transition on + // the same tracked tx supersedes the missed event, so we trade a stale + // notification for keeping txconfirm's actor loop unblocked. + reversibleNotifyTimeout = time.Second ) // ErrEnsureParamsMismatch is returned by EnsureConfirmedReq when a second @@ -185,6 +194,24 @@ type TxBroadcasterActor struct { blockSubscriptionActive bool } +// trackedSubscriber is one attached subscriber to a tracked tx. Every +// subscriber receives the full reorg-aware lifecycle (TxConfirmed, +// TxReorged, re-TxConfirmed, TxFinalized, TxFailed) and remains +// attached until TxFinalized or TxFailed is acknowledged. +// +// pendingConfirmed reports whether the INITIAL TxConfirmed delivery +// is still owed to this subscriber. It is true at admission, flipped +// false the first time notifyOneConfirmed lands successfully, and +// drives retryConfirmedRedelivery's per-tick retry loop so the +// at-least-once initial-TxConfirmed contract holds even when the +// subscriber's mailbox is briefly slow. Re-confirmations after a +// reorg are delivered best-effort because the eventual TxFinalized +// reliably carries the final height/numConfs. +type trackedSubscriber struct { + Ref actor.TellOnlyRef[Notification] + pendingConfirmed bool +} + // trackedTx stores the actor-owned handle for one tracked txid. // // The struct is the actor's single source of truth about a tracked @@ -195,7 +222,7 @@ type trackedTx struct { data trackedTxData fsm *trackedTxStateMachine - subscribers map[string]actor.TellOnlyRef[Notification] + subscribers map[string]trackedSubscriber // escalateLog rate-limits the operator-facing escalation that fires // once a tx has failed to reach any mempool repeatedly. It @@ -219,6 +246,17 @@ type trackedTx struct { // subsequent interval-paced bumps fall back to the estimator. Zero // means "no override pending". pendingTargetFeeRate int64 + + // sealed is set true the moment terminal delivery (TxFinalized / + // TxFailed) begins for this entry. Reversible deliveries + // (TxReorged / re-TxConfirmed) run on detached fire-and-forget + // goroutines, so a reversible spawned just before finality could + // otherwise Tell its subscriber after the terminal notification and + // resurrect reorg-recovery bookkeeping the consumer already dropped. + // Each reversible goroutine checks this flag immediately before its + // Tell and skips if the entry has sealed. It is atomic because it is + // read off the actor goroutine; only the actor goroutine writes it. + sealed atomic.Bool } // confirmationObservedMsg routes a chainsource confirmation callback back into @@ -239,14 +277,53 @@ func (m *confirmationObservedMsg) MessageType() string { // set. func (m *confirmationObservedMsg) txConfirmMsgSealed() {} -// terminalNotifyResultMsg returns the result of a terminal notification that -// outlived the actor-path wait budget. +// confirmationReorgedMsg routes a chainsource ConfReorgedEvent back into +// the actor mailbox. +type confirmationReorgedMsg struct { + actor.BaseMessage + txid chainhash.Hash +} + +// MessageType returns the stable message type identifier. +func (m *confirmationReorgedMsg) MessageType() string { + return "confirmationReorgedMsg" +} + +// txConfirmMsgSealed seals confirmationReorgedMsg into the package message +// set. +func (m *confirmationReorgedMsg) txConfirmMsgSealed() {} + +// confirmationDoneMsg routes a chainsource ConfDoneEvent back into the +// actor mailbox. +type confirmationDoneMsg struct { + actor.BaseMessage + txid chainhash.Hash +} + +// MessageType returns the stable message type identifier. +func (m *confirmationDoneMsg) MessageType() string { + return "confirmationDoneMsg" +} + +// txConfirmMsgSealed seals confirmationDoneMsg into the package message +// set. +func (m *confirmationDoneMsg) txConfirmMsgSealed() {} + +// terminalNotifyResultMsg returns the result of a terminal-shape +// notification that outlived the actor-path wait budget. kind carries the +// original delivery kind ("confirmed" / "finalized" / "failed") so +// handleTerminalNotifyResult can distinguish a mid-lifecycle +// initial-TxConfirmed redelivery (where the subscriber must stay attached +// to receive later TxReorged / TxFinalized) from a truly terminal +// notification (where the subscriber should be removed and the entry can +// evict once empty). type terminalNotifyResultMsg struct { actor.BaseMessage txid chainhash.Hash subscriberID string inflightKey string + kind string err error } @@ -354,6 +431,22 @@ func (a *TxBroadcasterActor) Receive(ctx context.Context, State: TxStateConfirmed, }) + case *confirmationReorgedMsg: + a.handleConfirmationReorged(ctx, req) + + return fn.Ok[Resp](&EnsureConfirmedResp{ + Txid: req.txid, + State: TxStateAwaitingConfirmation, + }) + + case *confirmationDoneMsg: + a.handleConfirmationDone(ctx, req) + + return fn.Ok[Resp](&EnsureConfirmedResp{ + Txid: req.txid, + State: TxStateFinalized, + }) + case *blockEpochObservedMsg: a.handleBlockObserved(ctx, req) @@ -401,7 +494,23 @@ func (a *TxBroadcasterActor) OnStop(ctx context.Context) error { continue } - if state == TxStateConfirmed || state == TxStateFailed { + if isTerminalTxState(state) { + // A terminal entry can still hold a registered conf + // watch: a Failed entry never received Done, and a + // Finalized entry whose Done-driven release raced + // OnStop may not have torn it down yet. Release it so + // the chainsource sub-actor does not leak for the + // daemon's lifetime. confWatchRegistered guards the + // common case where Done already released the watch. + if entry.confWatchRegistered { + if err := a.unregisterConfWatch( + ctx, entry, + ); err != nil && firstErr == nil { + + firstErr = err + } + } + if entry.fsm != nil { entry.fsm.Stop() } @@ -476,7 +585,10 @@ func (a *TxBroadcasterActor) handleEnsure(ctx context.Context, } return a.attachExistingSubscriber( - ctx, existing, req.Subscriber, + ctx, existing, trackedSubscriber{ + Ref: req.Subscriber, + pendingConfirmed: true, + }, ), nil } @@ -700,7 +812,7 @@ func (a *TxBroadcasterActor) handleCancel(ctx context.Context, return nil, err } - if state == TxStateConfirmed || state == TxStateFailed { + if isTerminalTxState(state) { a.evictTerminal(ctx, entry) return resp, nil @@ -920,8 +1032,13 @@ func (a *TxBroadcasterActor) handleBumpNow(ctx context.Context, }, nil } -// handleConfirmationObserved marks a tracked txid as confirmed and fans the -// result out to all subscribers. +// handleConfirmationObserved advances a tracked txid into the reversible +// Confirmed state and fans TxConfirmed out to all subscribers. The +// confirmation watch is intentionally kept alive: a subsequent reorg +// would arrive on the same registration, and the entry must stay +// observable so the chainsource sub-actor's reorg/done channels reach +// this layer. The terminal Finalized state is reached separately when +// the backend emits a Done signal. func (a *TxBroadcasterActor) handleConfirmationObserved(ctx context.Context, msg *confirmationObservedMsg) { @@ -942,7 +1059,9 @@ func (a *TxBroadcasterActor) handleConfirmationObserved(ctx context.Context, return } - if state == TxStateConfirmed || state == TxStateFailed { + // Terminal entries (Failed / Finalized) are sticky; if delivery to + // any subscriber was deferred, retry it and evict on success. + if isTerminalTxState(state) { if a.retryTerminalNotifications(ctx, entry) { a.evictTerminal(ctx, entry) } @@ -950,6 +1069,17 @@ func (a *TxBroadcasterActor) handleConfirmationObserved(ctx context.Context, return } + // An already-Confirmed entry receiving another Confirmed event + // without an intervening Reorged is unexpected on a well-behaved + // backend (the chainsource sub-actor only re-fires Confirmed after + // a reorg), but we tolerate it by re-delivering TxConfirmed rather + // than failing the FSM. + if state == TxStateConfirmed { + a.notifyConfirmed(ctx, entry, msg.blockHeight, msg.numConfs) + + return + } + if err := a.advanceTrackedTxFSM(ctx, entry, &trackedTxConfirmed{ BlockHeight: msg.blockHeight, }); err != nil { @@ -960,13 +1090,153 @@ func (a *TxBroadcasterActor) handleConfirmationObserved(ctx context.Context, return } + a.notifyConfirmed(ctx, entry, msg.blockHeight, msg.numConfs) +} + +// handleConfirmationReorged moves a Confirmed tracked txid back into +// AwaitingConfirmation and fans TxReorged out to all subscribers. The +// confirmation watch stays alive on the chainsource side, so a +// subsequent re-confirmation arrives on the same registration and drives +// another handleConfirmationObserved. +func (a *TxBroadcasterActor) handleConfirmationReorged(ctx context.Context, + msg *confirmationReorgedMsg) { + + entry, ok := a.tracked[msg.txid] + if !ok { + return + } + + state, err := entry.currentTxState() + if err != nil { + a.log.WarnS(ctx, "Failed to read tracked tx state", + err, "txid", entry.data.Txid) + + return + } + + // Only Confirmed entries can be reorged. A reorg ping in any other + // state is benign and dropped: it may be a late notification for an + // entry the actor has already evicted or terminally failed. + if state != TxStateConfirmed { + return + } + + if err := a.advanceTrackedTxFSM( + ctx, entry, &trackedTxReorged{}, + ); err != nil { + + a.log.WarnS(ctx, "Failed to apply reorg to tracked tx", + err, "txid", entry.data.Txid) + + return + } + + a.notifyReorged(ctx, entry) +} + +// handleConfirmationDone moves a Confirmed tracked txid into the terminal +// Finalized state, fans TxFinalized out to all subscribers, releases the +// confirmation watch, and evicts the entry. +func (a *TxBroadcasterActor) handleConfirmationDone(ctx context.Context, + msg *confirmationDoneMsg) { + + entry, ok := a.tracked[msg.txid] + if !ok { + return + } + + state, err := entry.currentTxState() + if err != nil { + a.log.WarnS(ctx, "Failed to read tracked tx state", + err, "txid", entry.data.Txid) + + return + } + + // Only Confirmed entries can be finalized. A Done ping for an entry + // that is not Confirmed has three possible origins: + // + // - Idempotent re-delivery for a tx that is already Finalized: a + // benign no-op. + // + // - Late arrival while the entry is in AwaitingConfirmation + // after a reorg: structurally anomalous for the current + // backends — chainntnfs (in-process lnd) only writes Done + // after the tx has matured past the safety depth, and the + // lndclient adapter never writes the channel at all and + // relies on height-based synthesis (which is gated on + // confirmHeight, reset to 0 on reorg). A future backend that + // DID fire Done during the reorg gap would land here, the + // chainsource sub-actor would already have exited (Done is + // one-shot), and this txid would stop receiving new events. + // The watch is unrecoverable without a fresh RegisterConf + // from this layer. + // + // We log + drop rather than failing the entry: the realistic + // backends do not produce this state, and failing on a hypothetical + // edge case would break callers that rely on the FSM staying live + // for re-confirmation after a reorg. If the log starts firing in + // the wild, the right follow-up is to re-register the conf watch + // from here (see registerConfWatch) rather than relax the guard. + if state != TxStateConfirmed { + a.log.WarnS(ctx, "Dropping confirmation Done for non-Confirmed "+ + "entry; chainsource watch is gone, entry will not "+ + "receive further reorg/finality events", + fmt.Errorf("state %s", state), + "txid", entry.data.Txid) + + return + } + + // Snapshot the confirmation height BEFORE advancing the FSM so we + // can attach it to the outgoing TxFinalized event; the new state + // preserves it but reading from the FSM state directly avoids a + // second state-lookup roundtrip. + fsmState, err := entry.currentFSMState() + if err != nil { + a.log.WarnS(ctx, "Failed to read tracked tx FSM state", + err, "txid", entry.data.Txid) + + return + } + confirmHeight, _ := trackedTxConfirmHeight(fsmState) + + if err := a.advanceTrackedTxFSM( + ctx, entry, &trackedTxFinalized{}, + ); err != nil { + + a.log.WarnS(ctx, "Failed to finalize tracked tx", + err, "txid", entry.data.Txid) + + return + } + + // Only release the conf watch and evict once every subscriber has + // acknowledged the terminal TxFinalized notification. Failed + // deliveries leave the entry in place so retryTerminalNotifications + // can resend on a later actor tick. + if !a.notifyFinalized(ctx, entry, confirmHeight) { + return + } + if err := a.unregisterConfWatch(ctx, entry); err != nil { a.log.WarnS(ctx, "Failed to unregister confirmation watch", err, "txid", entry.data.Txid) } - if a.notifyConfirmed(ctx, entry, msg.blockHeight, msg.numConfs) { - a.evictTerminal(ctx, entry) + a.evictTerminal(ctx, entry) +} + +// isTerminalTxState reports whether a public TxState value represents a +// terminal lifecycle stage that the actor will not advance further on +// its own. +func isTerminalTxState(state TxState) bool { + switch state { + case TxStateFinalized, TxStateFailed: + return true + + default: + return false } } @@ -1003,7 +1273,7 @@ func (a *TxBroadcasterActor) handleBlockObserved(ctx context.Context, continue } - if state == TxStateConfirmed || state == TxStateFailed { + if isTerminalTxState(state) { if a.retryTerminalNotifications(ctx, entry) { a.evictTerminal(ctx, entry) } @@ -1057,13 +1327,13 @@ func (a *TxBroadcasterActor) handleBlockObserved(ctx context.Context, // txid or immediately replays a terminal result. func (a *TxBroadcasterActor) attachExistingSubscriber( ctx context.Context, entry *trackedTx, - subscriber actor.TellOnlyRef[Notification], + subscriber trackedSubscriber, ) *EnsureConfirmedResp { state, err := entry.currentFSMState() if err != nil { a.notifyOneFailed( - ctx, subscriber, entry.data.Txid, + ctx, subscriber.Ref, entry.data.Txid, fmt.Sprintf("tracked tx state: %v", err), ) @@ -1073,27 +1343,57 @@ func (a *TxBroadcasterActor) attachExistingSubscriber( } } + subID := subscriber.Ref.ID() switch state := state.(type) { case *trackedTxStateConfirmed: confirmHeight, _ := trackedTxConfirmHeight(state) - if !a.notifyOneConfirmed( - ctx, subscriber, entry.data.Txid, confirmHeight, - entry.data.TargetConfs, + txConfirmed := &TxConfirmed{ + Txid: entry.data.Txid, + BlockHeight: confirmHeight, + NumConfs: entry.data.TargetConfs, + } + + // Reliable replay of the at-least-once TxConfirmed contract. + // The subscriber is retained in the map regardless of + // delivery outcome so it also receives any later TxReorged + // or TxFinalized on this entry, and on timeout the per-tick + // retryTerminalNotifications path re-attempts delivery via + // the still-true pendingConfirmed flag until it lands. + if a.notifyOneConfirmed( + ctx, subscriber.Ref, entry.data.Txid, txConfirmed, + ) { + + subscriber.pendingConfirmed = false + } + entry.subscribers[subID] = subscriber + + case *trackedTxStateFinalized: + // Finalized is terminal. Deliver TxFinalized reliably and + // retain on timeout so the per-tick retry path can finish + // the handoff. Note that a late-attaching subscriber has + // not yet received TxConfirmed; TxFinalized carries the + // authoritative confirmation height so consumers that need + // it (sweep-finality gates etc.) can recover without an + // out-of-band lookup. + if !a.notifyOneFinalized( + ctx, subscriber.Ref, entry.data.Txid, + state.ConfirmHeight, entry.data.TargetConfs, ) { - entry.subscribers[subscriber.ID()] = subscriber + entry.subscribers[subID] = subscriber } case *trackedTxStateFailed: reason, _ := trackedTxFailureReason(state) - if !a.notifyOneFailed(ctx, subscriber, entry.data.Txid, - reason) { + if !a.notifyOneFailed( + ctx, subscriber.Ref, entry.data.Txid, reason, + ) { - entry.subscribers[subscriber.ID()] = subscriber + entry.subscribers[subID] = subscriber } default: - entry.subscribers[subscriber.ID()] = subscriber + entry.subscribers[subID] = subscriber } return a.ensureResp(entry, false) @@ -1150,8 +1450,11 @@ func (a *TxBroadcasterActor) newTrackedTx(ctx context.Context, return &trackedTx{ data: data, fsm: fsm, - subscribers: map[string]actor.TellOnlyRef[Notification]{ - req.Subscriber.ID(): req.Subscriber, + subscribers: map[string]trackedSubscriber{ + req.Subscriber.ID(): { + Ref: req.Subscriber, + pendingConfirmed: true, + }, }, escalateLog: rate.Sometimes{ First: 1, @@ -1289,6 +1592,9 @@ func (a *TxBroadcasterActor) ensureBlockSubscription( } // registerConfWatch registers a confirmation watch for one tracked txid. +// The watch is registered in reorg-aware mode so the tracked entry can +// observe a confirmation being reorged out and a confirmation maturing +// past the backend's reorg-safety depth. func (a *TxBroadcasterActor) registerConfWatch(ctx context.Context, entry *trackedTx) error { @@ -1303,6 +1609,18 @@ func (a *TxBroadcasterActor) registerConfWatch(ctx context.Context, } }, ) + reorgRef := chainsource.MapConfReorgedEvent( + a.selfRef, + func(event chainsource.ConfReorgedEvent) Msg { + return &confirmationReorgedMsg{txid: event.Txid} + }, + ) + doneRef := chainsource.MapConfDoneEvent( + a.selfRef, + func(event chainsource.ConfDoneEvent) Msg { + return &confirmationDoneMsg{txid: event.Txid} + }, + ) _, err := a.cfg.ChainSource.Ask( ctx, &chainsource.RegisterConfRequest{ @@ -1311,9 +1629,11 @@ func (a *TxBroadcasterActor) registerConfWatch(ctx context.Context, PkScript: append( []byte(nil), entry.data.ConfirmationPkScript..., ), - TargetConfs: entry.data.TargetConfs, - HeightHint: entry.data.HeightHint, - NotifyActor: fn.Some(notifyRef), + TargetConfs: entry.data.TargetConfs, + HeightHint: entry.data.HeightHint, + NotifyActor: fn.Some(notifyRef), + NotifyReorged: fn.Some(reorgRef), + NotifyDone: fn.Some(doneRef), }, ).Await(ctx).Unpack() if err != nil { @@ -1570,11 +1890,18 @@ func (a *TxBroadcasterActor) retryTerminalNotifications(ctx context.Context, switch state := state.(type) { case *trackedTxStateConfirmed: - confirmHeight, _ := trackedTxConfirmHeight(state) + // Retry deferred TxConfirmed deliveries to legacy + // subscribers whose first attempt timed out. Reorg-aware + // subscribers receive TxConfirmed fire-and-forget and have + // no retry tracking; they are skipped here. Returning false + // keeps the entry alive — the caller never evicts on + // Confirmed, only on terminal states. + a.retryConfirmedRedelivery(ctx, entry, state.ConfirmHeight) - return a.notifyConfirmed( - ctx, entry, confirmHeight, entry.data.TargetConfs, - ) + return false + + case *trackedTxStateFinalized: + return a.notifyFinalized(ctx, entry, state.ConfirmHeight) case *trackedTxStateFailed: reason, _ := trackedTxFailureReason(state) @@ -1586,8 +1913,57 @@ func (a *TxBroadcasterActor) retryTerminalNotifications(ctx context.Context, } } -// handleTerminalNotifyResult applies the result of a terminal subscriber -// notification that continued after txconfirm returned to its actor mailbox. +// retryConfirmedRedelivery retries TxConfirmed delivery to every +// subscriber whose initial delivery timed out. The at-least-once +// initial-TxConfirmed contract means a missed delivery must be +// landed before the lifecycle moves on, so this helper runs every +// actor tick from retryTerminalNotifications until every +// pendingConfirmed subscriber has been notified. On successful +// delivery the subscriber's pendingConfirmed flag is cleared and +// the subscriber stays in the map so it continues to receive any +// later TxReorged / TxFinalized on this entry — eviction is +// reserved for the truly terminal Finalized / Failed states. +func (a *TxBroadcasterActor) retryConfirmedRedelivery(ctx context.Context, + entry *trackedTx, confirmHeight int32) { + + for id, subscriber := range entry.subscribers { + if !subscriber.pendingConfirmed { + continue + } + + txConfirmed := &TxConfirmed{ + Txid: entry.data.Txid, + BlockHeight: confirmHeight, + NumConfs: entry.data.TargetConfs, + } + if a.notifyOneConfirmed( + ctx, subscriber.Ref, entry.data.Txid, txConfirmed, + ) { + + subscriber.pendingConfirmed = false + entry.subscribers[id] = subscriber + } + } +} + +// handleTerminalNotifyResult applies the result of a terminal-shape +// subscriber notification that continued after txconfirm returned to its +// actor mailbox. +// +// The kind field distinguishes mid-lifecycle from truly terminal +// deliveries: +// +// - kind == "confirmed": the late-landing delivery is the INITIAL +// TxConfirmed for this subscriber. Clear pendingConfirmed and KEEP +// the subscriber attached to the entry so later TxReorged / +// TxFinalized still reach it. Removing the subscriber here would +// silently drop the entire reorg-aware tail of its lifecycle — +// exactly the kind of failure the unified reliable-delivery path is +// supposed to prevent. +// +// - kind == "finalized" or "failed": the lifecycle is genuinely over +// for this subscriber. Remove it, and evict the entry once every +// subscriber has been notified. func (a *TxBroadcasterActor) handleTerminalNotifyResult(ctx context.Context, msg *terminalNotifyResultMsg) { @@ -1596,7 +1972,8 @@ func (a *TxBroadcasterActor) handleTerminalNotifyResult(ctx context.Context, if msg.err != nil { a.log.WarnS(ctx, "Terminal notification failed after "+ "actor-path timeout", msg.err, "txid", msg.txid, - "subscriber_id", msg.subscriberID) + "subscriber_id", msg.subscriberID, + "notification_kind", msg.kind) return } @@ -1606,6 +1983,40 @@ func (a *TxBroadcasterActor) handleTerminalNotifyResult(ctx context.Context, return } + if msg.kind == "confirmed" { + subscriber, ok := entry.subscribers[msg.subscriberID] + if !ok { + return + } + subscriber.pendingConfirmed = false + entry.subscribers[msg.subscriberID] = subscriber + + // If the tx reorged out of its confirmation while this + // subscriber's initial TxConfirmed was still parked on the + // async notify path, notifyReorged skipped it: a TxReorged must + // never precede the TxConfirmed it reverses. Now that the + // initial delivery has completed and pendingConfirmed is + // cleared, deliver the catch-up TxReorged so the subscriber is + // not left believing a reorged-out tx is confirmed. A tx that + // reorged and then re-confirmed during the window is back in + // Confirmed, so the delivered TxConfirmed is already accurate + // and no catch-up is owed; terminal states own their own + // notifications. + state, err := entry.currentTxState() + if err == nil && state != TxStateConfirmed && + !isTerminalTxState(state) { + + a.notifyReversibleAsync( + ctx, &entry.sealed, subscriber.Ref, + entry.data.Txid, "TxReorged", &TxReorged{ + Txid: entry.data.Txid, + }, + ) + } + + return + } + delete(entry.subscribers, msg.subscriberID) if len(entry.subscribers) != 0 { return @@ -1619,21 +2030,205 @@ func (a *TxBroadcasterActor) handleTerminalNotifyResult(ctx context.Context, return } - if state == TxStateConfirmed || state == TxStateFailed { + if isTerminalTxState(state) { a.evictTerminal(ctx, entry) } } -// notifyConfirmed fans a confirmation result out to all current subscribers. -// It returns true only after every subscriber accepted the terminal -// notification. Failed deliveries are left in the subscriber map so a later -// actor tick can retry instead of permanently losing the confirmation. +// notifyConfirmed fans a TxConfirmed notification out to every current +// subscriber. The initial TxConfirmed (subscriber.pendingConfirmed == +// true) goes through the reliable terminal-delivery path: on success +// pendingConfirmed flips false and the subscriber is retained so it +// keeps receiving the reorg-aware lifecycle (TxReorged / TxFinalized); +// on timeout pendingConfirmed stays true and retryConfirmedRedelivery +// re-attempts on the next actor tick, so the at-least-once initial- +// TxConfirmed guarantee holds even for slow durable subscribers. +// +// Re-confirmations (pendingConfirmed == false, i.e. this subscriber +// has already received the initial TxConfirmed and the chain reorged +// then re-confirmed) are best-effort. The subscriber already knows +// the tx confirmed at least once; the eventual TxFinalized carries +// the authoritative height / numConfs so a missed re-confirmation is +// not load-bearing. +// +// Subscribers are never deleted from the map by this function — that +// happens only on TxFinalized / TxFailed acknowledgment, which is +// when the reorg-aware lifecycle reaches a truly terminal state. func (a *TxBroadcasterActor) notifyConfirmed(ctx context.Context, - entry *trackedTx, blockHeight int32, numConfs uint32) bool { + entry *trackedTx, blockHeight int32, numConfs uint32) { + + for id, subscriber := range entry.subscribers { + txConfirmed := &TxConfirmed{ + Txid: entry.data.Txid, + BlockHeight: blockHeight, + NumConfs: numConfs, + } + + if !subscriber.pendingConfirmed { + a.notifyReversibleAsync( + ctx, &entry.sealed, subscriber.Ref, + entry.data.Txid, "TxConfirmed", txConfirmed, + ) + + continue + } + + if a.notifyOneConfirmed( + ctx, subscriber.Ref, entry.data.Txid, txConfirmed, + ) { + + subscriber.pendingConfirmed = false + entry.subscribers[id] = subscriber + } + } +} + +// notifyOneConfirmed delivers one TxConfirmed notification to a legacy +// (non-opt-in) subscriber via the reliable terminal-delivery path so a +// slow durable subscriber cannot block the actor while still +// preserving the pre-reorg-aware contract of guaranteed-at-least-once +// TxConfirmed delivery. +func (a *TxBroadcasterActor) notifyOneConfirmed(ctx context.Context, + subscriber actor.TellOnlyRef[Notification], txid chainhash.Hash, + notification *TxConfirmed) bool { + + return a.notifyOneTerminal( + ctx, subscriber, txid, "confirmed", + func(notifyCtx context.Context) error { + return subscriber.Tell(notifyCtx, notification) + }, + ) +} + +// notifyReorged fans a TxReorged notification out to every retained +// subscriber whose initial TxConfirmed has already landed. Delivery is +// fire-and-forget: a slow subscriber that misses the reorg is recovered +// by the next lifecycle event (re-TxConfirmed, TxFinalized, or TxFailed) +// since the actor stays attached until one of those terminal events lands. +// +// Subscribers still owed their initial TxConfirmed (pendingConfirmed) are +// skipped: that delivery is deferred to the reliable retry path while a +// reorg's TxReorged goes out fire-and-forget, so notifying them here would +// race the two and could surface TxReorged before — or without — the +// initial TxConfirmed, violating the at-least-once contract and leaving a +// consumer believing a reorged-out tx is confirmed. They observe the live +// state on their eventual initial delivery instead. +func (a *TxBroadcasterActor) notifyReorged(ctx context.Context, + entry *trackedTx) { + + for _, subscriber := range entry.subscribers { + if subscriber.pendingConfirmed { + continue + } + + a.notifyReversibleAsync( + ctx, &entry.sealed, subscriber.Ref, entry.data.Txid, + "TxReorged", + &TxReorged{ + Txid: entry.data.Txid, + }, + ) + } +} + +// notifyReversibleAsync delivers one reversible (non-terminal) +// Notification to a subscriber on a fresh goroutine so txconfirm's +// actor loop does not block on a slow durable subscriber's mailbox. +// Failures are logged but not retried: the next event in the tracked +// tx's lifecycle (re-TxConfirmed after a TxReorged, TxFinalized, or +// eventual TxFailed) carries the live state and supersedes any +// dropped reversible delivery. +// +// The delivery context is detached from the actor transaction (so the +// txconfirm goroutine's DB tx is not held open behind a subscriber +// roundtrip) and from ctx cancellation (so a per-message context +// expiring mid-flight does not race the actor releasing its mailbox). +// Each goroutine is bounded by reversibleNotifyTimeout so a stuck +// subscriber does not leak unbounded goroutines on every block. +func (a *TxBroadcasterActor) notifyReversibleAsync(ctx context.Context, + sealed *atomic.Bool, subscriber actor.TellOnlyRef[Notification], + txid chainhash.Hash, kind string, notification Notification) { + + notifyCtx := actor.WithoutTx(context.WithoutCancel(ctx)) + notifyCtx, cancel := context.WithTimeout( + notifyCtx, reversibleNotifyTimeout, + ) + + subscriberID := subscriber.ID() + go func() { + defer cancel() + + // Drop the reversible delivery if the entry has sealed (a + // terminal TxFinalized/TxFailed has begun). Checked here, just + // before the Tell, so a reversible spawned before finality + // cannot trail the terminal notification into the subscriber's + // mailbox and resurrect bookkeeping it already released. + if sealed != nil && sealed.Load() { + return + } + + if err := subscriber.Tell(notifyCtx, notification); err != nil { + a.log.WarnS(notifyCtx, + "Failed to deliver reversible notification", + err, "txid", txid, + "subscriber_id", subscriberID, + "notification_kind", kind) + } + }() +} + +// notifyFinalized fans a TxFinalized notification out to every +// retained subscriber. The map may still contain subscribers whose +// initial reliable TxConfirmed delivery timed out +// (pendingConfirmed=true) — those still need TxConfirmed before they +// see TxFinalized, otherwise the at-least-once TxConfirmed contract +// is violated. For each pending subscriber we make one TxConfirmed +// attempt, flip pendingConfirmed on success, and either way leave +// them attached for the next finalize retry (the per-tick retry path +// will keep cycling through this function until both deliveries +// land). +// +// On successful TxFinalized delivery the subscriber is removed and +// the caller may evict the tracked entry once every subscriber has +// acknowledged. Failed deliveries are left in the subscriber map for +// retry from the next actor tick. +// +// confirmHeight is the height observed at finalization time, replayed +// onto TxFinalized so consumers that dropped the fire-and-forget +// re-TxConfirmed can recover the authoritative confirmation height +// without an out-of-band lookup. +func (a *TxBroadcasterActor) notifyFinalized(ctx context.Context, + entry *trackedTx, confirmHeight int32) bool { + + // Seal the entry so any in-flight reversible delivery skips its Tell + // rather than trailing this terminal notification into a subscriber's + // mailbox. Idempotent across the per-tick finalize retries. + entry.sealed.Store(true) for id, subscriber := range entry.subscribers { - ok := a.notifyOneConfirmed( - ctx, subscriber, entry.data.Txid, blockHeight, numConfs, + if subscriber.pendingConfirmed { + if a.notifyOneConfirmed( + ctx, subscriber.Ref, entry.data.Txid, + &TxConfirmed{ + Txid: entry.data.Txid, + BlockHeight: confirmHeight, + NumConfs: entry.data.TargetConfs, + }, + ) { + + subscriber.pendingConfirmed = false + entry.subscribers[id] = subscriber + } else { + // Hold until the next tick — we must not + // deliver TxFinalized before the documented + // initial TxConfirmed has landed. + continue + } + } + + ok := a.notifyOneFinalized( + ctx, subscriber.Ref, entry.data.Txid, confirmHeight, + entry.data.TargetConfs, ) if !ok { continue @@ -1652,9 +2247,14 @@ func (a *TxBroadcasterActor) notifyConfirmed(ctx context.Context, func (a *TxBroadcasterActor) notifyFailed(ctx context.Context, entry *trackedTx, reason string) bool { + // Seal the entry so any in-flight reversible delivery skips its Tell + // rather than trailing this terminal failure into a subscriber's + // mailbox. Idempotent across the per-tick retries. + entry.sealed.Store(true) + for id, subscriber := range entry.subscribers { ok := a.notifyOneFailed( - ctx, subscriber, entry.data.Txid, reason, + ctx, subscriber.Ref, entry.data.Txid, reason, ) if !ok { continue @@ -1666,15 +2266,20 @@ func (a *TxBroadcasterActor) notifyFailed(ctx context.Context, entry *trackedTx, return len(entry.subscribers) == 0 } -// notifyOneConfirmed delivers one confirmation notification. -func (a *TxBroadcasterActor) notifyOneConfirmed(ctx context.Context, +// notifyOneFinalized delivers one TxFinalized notification through the +// terminal-delivery path so a slow subscriber cannot block the actor +// loop. blockHeight and numConfs are replayed onto the notification +// from the last observed confirmation so opt-in subscribers that +// dropped the (fire-and-forget) TxConfirmed event can still recover +// the authoritative confirmation height. +func (a *TxBroadcasterActor) notifyOneFinalized(ctx context.Context, subscriber actor.TellOnlyRef[Notification], txid chainhash.Hash, blockHeight int32, numConfs uint32) bool { return a.notifyOneTerminal( - ctx, subscriber, txid, "confirmed", + ctx, subscriber, txid, "finalized", func(notifyCtx context.Context) error { - return subscriber.Tell(notifyCtx, &TxConfirmed{ + return subscriber.Tell(notifyCtx, &TxFinalized{ Txid: txid, BlockHeight: blockHeight, NumConfs: numConfs, @@ -1735,7 +2340,7 @@ func (a *TxBroadcasterActor) notifyOneTerminal(ctx context.Context, a.terminalNotifyInflight[inflightKey] = struct{}{} //nolint:contextcheck // async result outlives ctx a.completeTerminalNotifyAsync( - inflightKey, txid, subscriberID, errChan, cancel, + inflightKey, txid, subscriberID, kind, errChan, cancel, ) a.log.DebugS(ctx, "Terminal tx notification deferred", @@ -1748,10 +2353,13 @@ func (a *TxBroadcasterActor) notifyOneTerminal(ctx context.Context, } } -// completeTerminalNotifyAsync reports a timed-out terminal delivery back to the -// txconfirm actor once the underlying Tell returns. +// completeTerminalNotifyAsync reports a timed-out terminal-shape delivery +// back to the txconfirm actor once the underlying Tell returns. kind +// records the notification variant ("confirmed" / "finalized" / "failed") +// so the actor-side handler can apply the correct post-delivery state +// transition. func (a *TxBroadcasterActor) completeTerminalNotifyAsync(inflightKey string, - txid chainhash.Hash, subscriberID string, errChan <-chan error, + txid chainhash.Hash, subscriberID, kind string, errChan <-chan error, cancel context.CancelFunc) { if a.selfRef == nil { @@ -1768,6 +2376,7 @@ func (a *TxBroadcasterActor) completeTerminalNotifyAsync(inflightKey string, txid: txid, subscriberID: subscriberID, inflightKey: inflightKey, + kind: kind, err: err, } bgCtx := context.Background() diff --git a/txconfirm/actor_test.go b/txconfirm/actor_test.go index 688dcb64f..eebe1d0b4 100644 --- a/txconfirm/actor_test.go +++ b/txconfirm/actor_test.go @@ -29,6 +29,11 @@ import ( // testTimeout is the default timeout used by txconfirm actor tests. const testTimeout = time.Second +// confReorgedRef / confDoneRef are the reorg / done notification ref +// types used in the fake chain-source ref. +type confReorgedRef = actor.TellOnlyRef[chainsource.ConfReorgedEvent] +type confDoneRef = actor.TellOnlyRef[chainsource.ConfDoneEvent] + // confNotifyRef is the confirmation-event notification ref type used in // the fake chainsource test double. type confNotifyRef = actor.TellOnlyRef[chainsource.ConfirmationEvent] @@ -54,6 +59,8 @@ type fakeChainSourceRef struct { blockNotify actor.TellOnlyRef[chainsource.BlockEpoch] confNotify map[chainhash.Hash]confNotifyRef + confReorged map[chainhash.Hash]confReorgedRef + confDone map[chainhash.Hash]confDoneRef confConfs map[chainhash.Hash]uint32 alreadyConfirmed map[chainhash.Hash]chainsource.ConfirmationEvent @@ -148,6 +155,93 @@ func (b *blockingNotifyRef) attemptsCount() int { return b.attempts } +// deferringNotifyRef blocks Tell until released, then completes +// successfully (returns nil regardless of ctx state). Used to drive the +// async-completion path of notifyOneTerminal: Tell exceeds the actor-path +// budget, so the caller observes a timeout and parks the result on the +// completeTerminalNotifyAsync goroutine; the test then releases the Tell +// to simulate the underlying mailbox eventually accepting the +// notification, producing a terminalNotifyResultMsg{err: nil} that the +// actor mailbox processes via handleTerminalNotifyResult. +type deferringNotifyRef struct { + id string + + started chan struct{} + release chan struct{} + once sync.Once + + mu sync.Mutex + attempts int + msgs []Notification +} + +// newDeferringNotifyRef creates a subscriber that blocks on the first Tell +// until release is closed. +func newDeferringNotifyRef(id string) *deferringNotifyRef { + return &deferringNotifyRef{ + id: id, + started: make(chan struct{}), + release: make(chan struct{}), + } +} + +// ID returns the fake subscriber ID. +func (d *deferringNotifyRef) ID() string { + return d.id +} + +// Tell blocks until release is closed, records the notification, and +// returns nil. ctx cancellation is intentionally ignored so the test can +// drive the "Tell eventually succeeded" path even after the actor-side +// notifyCtx timed out. +func (d *deferringNotifyRef) Tell(_ context.Context, n Notification) error { + d.mu.Lock() + d.attempts++ + d.mu.Unlock() + + d.once.Do(func() { + close(d.started) + }) + + <-d.release + + d.mu.Lock() + d.msgs = append(d.msgs, n) + d.mu.Unlock() + + return nil +} + +// waitStarted blocks until the first Tell has begun, so the test can +// release after the actor-side timeout has fired. +func (d *deferringNotifyRef) waitStarted(t *testing.T) { + t.Helper() + + select { + case <-d.started: + case <-time.After(testTimeout): + t.Fatal("deferringNotifyRef Tell never started") + } +} + +// releaseTell unblocks every parked Tell call so the deferred deliveries +// complete and the test observes the async-completion path. +func (d *deferringNotifyRef) releaseTell() { + close(d.release) +} + +// snapshotMessages returns a defensive copy of every notification that +// has landed in the subscriber's mailbox so far. +func (d *deferringNotifyRef) snapshotMessages() []Notification { + d.mu.Lock() + defer d.mu.Unlock() + + out := make([]Notification, len(d.msgs)) + copy(out, d.msgs) + + return out +} + // ID returns the fake subscriber ID. func (r *contextInspectNotifyRef) ID() string { return r.id @@ -232,10 +326,12 @@ func (r *retryNotifyRef) awaitMessage(timeout time.Duration) (Notification, // newFakeChainSourceRef creates a new controllable chainsource test double. func newFakeChainSourceRef(bestHeight int32) *fakeChainSourceRef { return &fakeChainSourceRef{ - bestHeight: bestHeight, - feeRate: 5, - confNotify: make(map[chainhash.Hash]confNotifyRef), - confConfs: make(map[chainhash.Hash]uint32), + bestHeight: bestHeight, + feeRate: 5, + confNotify: make(map[chainhash.Hash]confNotifyRef), + confReorged: make(map[chainhash.Hash]confReorgedRef), + confDone: make(map[chainhash.Hash]confDoneRef), + confConfs: make(map[chainhash.Hash]uint32), alreadyConfirmed: make( map[chainhash.Hash]chainsource.ConfirmationEvent, ), @@ -344,6 +440,12 @@ func (f *fakeChainSourceRef) handleAsk(_ context.Context, if req.Txid != nil && req.NotifyActor.IsSome() { f.confNotify[*req.Txid] = req.NotifyActor.UnwrapOr(nil) f.confConfs[*req.Txid] = req.TargetConfs + req.NotifyReorged.WhenSome(func(r confReorgedRef) { + f.confReorged[*req.Txid] = r + }) + req.NotifyDone.WhenSome(func(r confDoneRef) { + f.confDone[*req.Txid] = r + }) if event, ok := f.alreadyConfirmed[*req.Txid]; ok { notifyRef := req.NotifyActor.UnwrapOr(nil) //nolint:contextcheck // fake backend @@ -358,6 +460,8 @@ func (f *fakeChainSourceRef) handleAsk(_ context.Context, if req.Txid != nil { delete(f.confNotify, *req.Txid) delete(f.confConfs, *req.Txid) + delete(f.confReorged, *req.Txid) + delete(f.confDone, *req.Txid) } return &chainsource.UnregisterConfResponse{}, nil @@ -417,6 +521,34 @@ func (f *fakeChainSourceRef) emitConfirmation(t *testing.T, txid chainhash.Hash, require.NoError(t, err) } +// emitConfReorged delivers a reorg event for one tracked txid. +func (f *fakeChainSourceRef) emitConfReorged(t *testing.T, + txid chainhash.Hash) { + + t.Helper() + + f.mu.Lock() + ref := f.confReorged[txid] + f.mu.Unlock() + + require.NotNil(t, ref) + err := ref.Tell(t.Context(), chainsource.ConfReorgedEvent{Txid: txid}) + require.NoError(t, err) +} + +// emitConfDone delivers a finality event for one tracked txid. +func (f *fakeChainSourceRef) emitConfDone(t *testing.T, txid chainhash.Hash) { + t.Helper() + + f.mu.Lock() + ref := f.confDone[txid] + f.mu.Unlock() + + require.NotNil(t, ref) + err := ref.Tell(t.Context(), chainsource.ConfDoneEvent{Txid: txid}) + require.NoError(t, err) +} + // emitBlock delivers a new block epoch to the shared block subscriber. func (f *fakeChainSourceRef) emitBlock(t *testing.T, height int32) { t.Helper() @@ -815,15 +947,27 @@ func TestEnsureConfirmedDedupesTwoSubscribers(t *testing.T) { require.IsType(t, &TxConfirmed{}, confirmedA) require.IsType(t, &TxConfirmed{}, confirmedB) + + // TxConfirmed alone does not release the chainsource conf watch in + // the reorg-aware model: the watch stays alive until the backend + // finalizes the confirmation. Driving Done evicts the tracked + // entry and unregisters. + chain.emitConfDone(t, tx.TxHash()) + mustAwaitNotification(t, subA) + mustAwaitNotification(t, subB) mustEventually(t, func() bool { return chain.unregisterConfCount() == 1 }) } -// TestConfirmationDeliveryRetriesAfterTellFailure verifies that a transient -// subscriber delivery failure does not permanently drop a terminal -// confirmation notification. -func TestConfirmationDeliveryRetriesAfterTellFailure(t *testing.T) { +// TestLifecycleDeliveryRetriesAfterTellFailure verifies that a transient +// subscriber delivery failure does not permanently drop a lifecycle +// notification. Both the initial TxConfirmed and the terminal +// TxFinalized are reliable: if the first Tell attempt fails, the +// per-tick retry path keeps re-attempting until delivery lands, and +// the entry only evicts after every retained subscriber has +// acknowledged both events. +func TestLifecycleDeliveryRetriesAfterTellFailure(t *testing.T) { chain := newFakeChainSourceRef(100) ref, _ := newTestActor(t, Config{ ChainSource: chain, @@ -831,6 +975,9 @@ func TestConfirmationDeliveryRetriesAfterTellFailure(t *testing.T) { tx := makeTestTx(false) txid := tx.TxHash() + // Fail the first Tell attempt: TxConfirmed delivery has to be + // retried before pendingConfirmed clears, after which TxFinalized + // can be attempted. sub := newRetryNotifyRef("sub-retry", 1) resp := mustEnsure(t, ref.Ref(), &EnsureConfirmedReq{ @@ -844,31 +991,40 @@ func TestConfirmationDeliveryRetriesAfterTellFailure(t *testing.T) { return sub.attemptsCount() == 1 }) + // First TxConfirmed attempt failed; the entry stays alive with + // pendingConfirmed=true and no notification has reached the + // subscriber mailbox yet. msg, ok := sub.awaitMessage(100 * time.Millisecond) - require.False(t, ok, "unexpected notification: %v", msg) - mustEventually(t, func() bool { - return chain.unregisterConfCount() == 1 - }) - - chain.emitBlock(t, 102) - msg, ok = sub.awaitMessage(testTimeout) - require.True(t, ok, "expected retried notification") - - confirmed, ok := msg.(*TxConfirmed) - require.True(t, ok) + require.False(t, ok, "unexpected early notification: %v", msg) + require.Equal(t, 0, chain.unregisterConfCount()) + + // Finalization fires. notifyFinalized retries TxConfirmed first + // (the at-least-once initial-confirmation contract must land + // before TxFinalized is allowed through). The second Tell + // succeeds, so TxConfirmed lands and TxFinalized follows in the + // same notifyFinalized pass. + chain.emitConfDone(t, txid) + + first, ok := sub.awaitMessage(testTimeout) + require.True(t, ok, "expected retried TxConfirmed") + confirmed, ok := first.(*TxConfirmed) + require.True( + t, ok, "first notification must be TxConfirmed, got %T", first, + ) require.Equal(t, txid, confirmed.Txid) - require.Equal(t, int32(101), confirmed.BlockHeight) - require.Equal(t, uint32(1), confirmed.NumConfs) - require.Equal(t, 2, sub.attemptsCount()) - freshSub := actor.NewChannelTellOnlyRef[Notification]("sub-fresh", 4) - replayResp := mustEnsure(t, ref.Ref(), &EnsureConfirmedReq{ - Tx: tx, - Subscriber: freshSub, + second, ok := sub.awaitMessage(testTimeout) + require.True(t, ok, "expected TxFinalized") + finalized, ok := second.(*TxFinalized) + require.True( + t, ok, "second notification must be TxFinalized, got %T", + second, + ) + require.Equal(t, txid, finalized.Txid) + + mustEventually(t, func() bool { + return chain.unregisterConfCount() == 1 }) - require.True(t, replayResp.Created) - require.Equal(t, 2, chain.registerConfCount()) - require.Equal(t, 2, chain.broadcastCallCount()) } // TestTerminalNotificationsDoNotInheritCallerContext verifies that terminal @@ -884,17 +1040,17 @@ func TestTerminalNotificationsDoNotInheritCallerContext(t *testing.T) { cancel() txid := chainhash.Hash{1} - confirmedSub := &contextInspectNotifyRef{id: "confirmed-sub"} - ok := behavior.notifyOneConfirmed( - ctx, confirmedSub, txid, 101, 1, + finalizedSub := &contextInspectNotifyRef{id: "finalized-sub"} + ok := behavior.notifyOneFinalized( + ctx, finalizedSub, txid, 101, 1, ) require.True(t, ok) - hasTx, ctxErr, msgs := confirmedSub.snapshot() + hasTx, ctxErr, msgs := finalizedSub.snapshot() require.False(t, hasTx) require.NoError(t, ctxErr) require.Len(t, msgs, 1) - require.IsType(t, &TxConfirmed{}, msgs[0]) + require.IsType(t, &TxFinalized{}, msgs[0]) failedSub := &contextInspectNotifyRef{id: "failed-sub"} ok = behavior.notifyOneFailed(ctx, failedSub, txid, "boom") @@ -937,14 +1093,14 @@ func TestTerminalNotificationTimeoutDoesNotBlockActor(t *testing.T) { }() start := time.Now() - ok := behavior.notifyOneConfirmed( + ok := behavior.notifyOneFinalized( context.Background(), sub, txid, 101, 1, ) require.False(t, ok) require.Less(t, time.Since(start), testTimeout) require.True(t, <-started) - key := terminalNotifyKey(txid, sub.ID(), "confirmed") + key := terminalNotifyKey(txid, sub.ID(), "finalized") _, inflight := behavior.terminalNotifyInflight[key] require.True(t, inflight) require.Equal(t, 1, sub.attemptsCount()) @@ -1051,6 +1207,9 @@ func TestEnsureConfirmedAlreadyConfirmedUsesSuccessPath(t *testing.T) { subA := actor.NewChannelTellOnlyRef[Notification]("sub-a", 4) subB := actor.NewChannelTellOnlyRef[Notification]("sub-b", 4) + // Opt in to reorg-aware notifications so the tracked entry stays + // alive past TxConfirmed and the second EnsureConfirmedReq can + // attach to the existing tracking state. resp := mustEnsure(t, ref.Ref(), &EnsureConfirmedReq{ Tx: tx, Subscriber: subA, @@ -1062,21 +1221,21 @@ func TestEnsureConfirmedAlreadyConfirmedUsesSuccessPath(t *testing.T) { require.True(t, ok) require.Equal(t, int32(99), confirmed.BlockHeight) - // Once subA's TxConfirmed has been delivered, terminal eviction drops - // the tracked entry. A subsequent EnsureConfirmedReq for the same - // txid therefore starts fresh tracking rather than replaying cached - // state. Chainsource immediately re-fires the confirmation for the - // already-confirmed tx, so subB still receives TxConfirmed. + // TxConfirmed is no longer terminal: the tracked entry remains alive + // (still subscribed to chainsource reorg/done) until a Done event + // fires. A subsequent EnsureConfirmedReq therefore attaches to the + // existing tracking state and replays TxConfirmed without + // re-broadcasting or re-registering with chainsource. replayResp := mustEnsure(t, ref.Ref(), &EnsureConfirmedReq{ Tx: tx, Subscriber: subB, }) - require.True(t, replayResp.Created) + require.False(t, replayResp.Created) replayed := mustAwaitNotification(t, subB) require.IsType(t, &TxConfirmed{}, replayed) - require.Equal(t, 2, chain.broadcastCallCount()) - require.Equal(t, 2, chain.registerConfCount()) + require.Equal(t, 1, chain.broadcastCallCount()) + require.Equal(t, 1, chain.registerConfCount()) } // TestEnsureConfirmedBroadcastFailureNotifiesFailure verifies that terminal @@ -1462,6 +1621,11 @@ func TestUnregisterConfMatchesRegisterServiceKey(t *testing.T) { confirmed := mustAwaitNotification(t, sub) require.IsType(t, &TxConfirmed{}, confirmed) + // The conf watch is only released once chainsource fires Done. + chain.emitConfDone(t, tx.TxHash()) + finalized := mustAwaitNotification(t, sub) + require.IsType(t, &TxFinalized{}, finalized) + mustEventually(t, func() bool { return chain.unregisterConfCount() == 1 }) @@ -1531,7 +1695,17 @@ func TestTerminalEntriesEvictedAfterConfirmation(t *testing.T) { require.IsType(t, &TxConfirmed{}, confirmed) } - // Every confirmation should have produced exactly one unregister. + // Finalization drives terminal eviction: TxConfirmed alone is now + // reversible and the tracked entry stays alive until the backend + // fires Done. + for i := 0; i < numTxs; i++ { + chain.emitConfDone(t, txids[i]) + + finalized := mustAwaitNotification(t, subs[i]) + require.IsType(t, &TxFinalized{}, finalized) + } + + // Every finalized entry should have produced exactly one unregister. mustEventually(t, func() bool { return chain.unregisterConfCount() == numTxs }) @@ -1907,3 +2081,81 @@ func TestEnsureConfirmedFailsPermanentBroadcastError(t *testing.T) { failed := mustAwaitNotification(t, sub) require.IsType(t, &TxFailed{}, failed) } + +// TestInitialConfirmedAsyncDeliveryRetainsSubscriber regression-tests an +// initial TxConfirmed delivery whose Tell exceeds terminalNotifyTimeout +// before landing successfully via the async-completion goroutine. The +// subscriber must remain attached to the entry afterwards so the +// reorg-aware tail of its lifecycle (TxReorged / TxFinalized) still +// reaches it; the previous handleTerminalNotifyResult deleted the +// subscriber on any successful async terminal completion, silently +// dropping every subsequent reorg event for slow mailboxes. +func TestInitialConfirmedAsyncDeliveryRetainsSubscriber(t *testing.T) { + // Shorten the actor-path budget so the test reliably exercises the + // async-completion code path within a normal test runtime. + oldTimeout := terminalNotifyTimeout + terminalNotifyTimeout = 20 * time.Millisecond + t.Cleanup(func() { + terminalNotifyTimeout = oldTimeout + }) + + chain := newFakeChainSourceRef(100) + ref, _ := newTestActor(t, Config{ + ChainSource: chain, + }) + + tx := makeTestTx(false) + txid := tx.TxHash() + sub := newDeferringNotifyRef("sub-async-confirmed") + + resp := mustEnsure(t, ref.Ref(), &EnsureConfirmedReq{ + Tx: tx, + Subscriber: sub, + }) + require.True(t, resp.Created) + + // Fire the confirmation. notifyOneConfirmed -> notifyOneTerminal + // will park on sub.Tell, blow through terminalNotifyTimeout, and + // hand the in-flight Tell off to completeTerminalNotifyAsync. + chain.emitConfirmation(t, txid, 101) + sub.waitStarted(t) + + // Wait long enough that the actor-side notifyCtx is guaranteed to + // have timed out and the async-completion goroutine is in flight. + // Without this delay the Tell could land synchronously and the + // async path the regression targets would not be exercised. + time.Sleep(10 * terminalNotifyTimeout) + + // Release the parked Tell so the async goroutine reports back to + // the txconfirm actor with a successful terminalNotifyResultMsg. + sub.releaseTell() + + // Wait for TxConfirmed to actually land in the subscriber mailbox. + require.Eventually(t, func() bool { + for _, m := range sub.snapshotMessages() { + if _, ok := m.(*TxConfirmed); ok { + return true + } + } + + return false + }, testTimeout, 10*time.Millisecond, "TxConfirmed never delivered") + + // Drive a reorg. If the subscriber was wrongly evicted on the + // async-confirmed completion, notifyReorged has nobody to fan to + // and the TxReorged below never arrives. + chain.emitConfReorged(t, txid) + + require.Eventually(t, func() bool { + for _, m := range sub.snapshotMessages() { + if _, ok := m.(*TxReorged); ok { + return true + } + } + + return false + }, testTimeout, 10*time.Millisecond, + "TxReorged never reached subscriber whose initial TxConfirmed "+ + "completed via the async path — handleTerminalNotify"+ + "Result probably evicted it on completion") +} diff --git a/txconfirm/broadcaster_test.go b/txconfirm/broadcaster_test.go index cbf638e3e..716348281 100644 --- a/txconfirm/broadcaster_test.go +++ b/txconfirm/broadcaster_test.go @@ -361,7 +361,7 @@ func newTrackedTxForState(t *testing.T, state trackedTxState) *trackedTx { return &trackedTx{ data: data, fsm: &fsm, - subscribers: make(map[string]actor.TellOnlyRef[Notification]), + subscribers: make(map[string]trackedSubscriber), } } @@ -1619,11 +1619,13 @@ func TestActorValidationAndCleanup(t *testing.T) { }, } entry := newTrackedTxForState(t, awaitConf) - entry.subscribers["fail"] = &failingNotifyRef{} + entry.subscribers["fail"] = trackedSubscriber{ + Ref: &failingNotifyRef{}, + } behavior.tracked[entry.data.Txid] = entry - behavior.notifyOneConfirmed( - t.Context(), &failingNotifyRef{}, entry.data.Txid, 1, 1, + behavior.notifyOneFinalized( + t.Context(), &failingNotifyRef{}, entry.data.Txid, 0, 0, ) behavior.notifyOneFailed( t.Context(), &failingNotifyRef{}, entry.data.Txid, diff --git a/txconfirm/fsm_types.go b/txconfirm/fsm_types.go index 387962b43..3cde4011f 100644 --- a/txconfirm/fsm_types.go +++ b/txconfirm/fsm_types.go @@ -168,6 +168,21 @@ type trackedTxFailed struct { // trackedTxEventSealed marks trackedTxFailed as a tracked-tx event. func (e *trackedTxFailed) trackedTxEventSealed() {} +// trackedTxReorged records that a previously delivered confirmation was +// reorged out of the canonical chain. Only valid from +// trackedTxStateConfirmed. +type trackedTxReorged struct{} + +// trackedTxEventSealed marks trackedTxReorged as a tracked-tx event. +func (e *trackedTxReorged) trackedTxEventSealed() {} + +// trackedTxFinalized records that a confirmation is past the backend's +// reorg-safety depth. Only valid from trackedTxStateConfirmed. +type trackedTxFinalized struct{} + +// trackedTxEventSealed marks trackedTxFinalized as a tracked-tx event. +func (e *trackedTxFinalized) trackedTxEventSealed() {} + // trackedTxErrorReporter reports tracked-tx FSM errors through the package // logger. type trackedTxErrorReporter struct { @@ -225,6 +240,9 @@ func txStateFromTrackedState(state trackedTxState) TxState { case *trackedTxStateConfirmed: return TxStateConfirmed + case *trackedTxStateFinalized: + return TxStateFinalized + case *trackedTxStateFailed: return TxStateFailed @@ -279,6 +297,9 @@ func trackedTxLastBroadcastHeight(state trackedTxState) fn.Option[int32] { case *trackedTxStateConfirmed: return s.LastBroadcastHeight + case *trackedTxStateFinalized: + return s.LastBroadcastHeight + case *trackedTxStateFailed: return s.LastBroadcastHeight @@ -302,12 +323,16 @@ func trackedTxBroadcastFailures(state trackedTxState) int { // trackedTxConfirmHeight returns the state's confirmation height if the // transaction has already confirmed. func trackedTxConfirmHeight(state trackedTxState) (int32, bool) { - confirmed, ok := state.(*trackedTxStateConfirmed) - if !ok { + switch s := state.(type) { + case *trackedTxStateConfirmed: + return s.ConfirmHeight, true + + case *trackedTxStateFinalized: + return s.ConfirmHeight, true + + default: return 0, false } - - return confirmed.ConfirmHeight, true } // trackedTxFailureReason returns the state's terminal failure reason when diff --git a/txconfirm/funded_anchor_test.go b/txconfirm/funded_anchor_test.go index 53c07bb46..f0f51e992 100644 --- a/txconfirm/funded_anchor_test.go +++ b/txconfirm/funded_anchor_test.go @@ -575,9 +575,12 @@ func TestBumpNowParentFeeSufficient(t *testing.T) { } // TestBumpNowUntrackedAndTerminal covers the remaining no-op guards: an -// untracked txid, and a transaction that confirmed and was evicted from -// tracking (a confirmed entry whose terminal notification is delivered is -// dropped from the map, so a late bump lands in the untracked branch). +// untracked txid, and a transaction that has reached its confirmation target. +// Under the reorg-aware lifecycle a confirmed entry is NOT evicted at its +// confirmation — it stays tracked in the reversible Confirmed state until the +// backend signals finality (Done), so it can observe a later reorg — so a bump +// of a confirmed-but-not-final tx lands in the "already confirmed" no-op +// branch rather than the untracked branch. func TestBumpNowUntrackedAndTerminal(t *testing.T) { t.Parallel() @@ -608,9 +611,10 @@ func TestBumpNowUntrackedAndTerminal(t *testing.T) { Subscriber: sub, }) - // Drain the confirmation callback the fake chain delivered to the - // self ref so the entry reaches its terminal state and, with its - // notification delivered, is evicted from tracking. + // Drain the confirmation callback the fake chain delivered to the self + // ref so the entry advances into the reversible Confirmed state and its + // TxConfirmed notification is delivered. It is NOT evicted: the entry + // stays tracked until finality so a reorg can still be observed. selfRef, ok := behavior.selfRef.(*actor.ChannelTellOnlyRef[Msg]) require.True(t, ok) for { @@ -622,11 +626,13 @@ func TestBumpNowUntrackedAndTerminal(t *testing.T) { } require.NotNil(t, mustAwaitNotification(t, sub)) - // A bump after eviction reports the untracked no-op: there is no - // entry left to bump, which is the correct answer for a confirmed tx. + // A bump of the confirmed (but not-yet-final) entry is the confirmed + // no-op: it is still tracked, so the handler reports "already + // confirmed" rather than the untracked branch. Either way a confirmed + // tx cannot be fee-bumped. bumpResp = mustReceiveBump(t, behavior, &BumpNowReq{ Txid: tx.TxHash(), }) require.False(t, bumpResp.Bumped) - require.Contains(t, bumpResp.Reason, "not tracked") + require.Contains(t, bumpResp.Reason, "already confirmed") } diff --git a/txconfirm/messages.go b/txconfirm/messages.go index 2208788f2..e595e6401 100644 --- a/txconfirm/messages.go +++ b/txconfirm/messages.go @@ -58,9 +58,15 @@ const ( TxStateFeeBumping // TxStateConfirmed indicates the tracked transaction reached its target - // confirmation count. + // confirmation count on the canonical chain. This state is reversible: + // a reorg moves the tracked entry back to TxStateAwaitingConfirmation, + // and finality moves it to TxStateFinalized. TxStateConfirmed + // TxStateFinalized indicates the tracked transaction confirmed and is + // past the backend's reorg-safety depth. Terminal. + TxStateFinalized + // TxStateFailed indicates the actor encountered a terminal // failure while // trying to confirm the transaction. @@ -85,6 +91,9 @@ func (s TxState) String() string { case TxStateConfirmed: return "confirmed" + case TxStateFinalized: + return "finalized" + case TxStateFailed: return "failed" @@ -126,8 +135,17 @@ type EnsureConfirmedReq struct { // parents and for callers that do not fee-bump. ParentFee btcutil.Amount - // Subscriber receives TxConfirmed or TxFailed notifications for this - // request. + // Subscriber receives the full reorg-aware notification lifecycle + // for this request: TxConfirmed when the tx reaches the + // confirmation target, TxReorged if a previously delivered + // TxConfirmed is reorged out, re-TxConfirmed on the new canonical + // chain, TxFinalized once the confirmation is past the backend's + // reorg-safety depth, and TxFailed on terminal failure. + // TxConfirmed and TxFailed deliveries are reliable (retry on + // timeout from the per-tick retry path); TxReorged is best-effort + // because the next lifecycle event (re-TxConfirmed / TxFinalized / + // TxFailed) re-establishes state. TxFinalized is reliable so the + // caller can free reorg-recovery bookkeeping deterministically. Subscriber actor.TellOnlyRef[Notification] } @@ -306,6 +324,65 @@ func (m *TxConfirmed) MessageType() string { // set. func (m *TxConfirmed) txConfirmNotificationSealed() {} +// TxReorged notifies a subscriber that a previously delivered TxConfirmed +// has been reorged out of the canonical chain. After receiving this event +// a consumer should consider the prior confirmation no longer valid; if +// the transaction re-confirms on the new canonical chain a fresh +// TxConfirmed will follow on the same subscription. +type TxReorged struct { + actor.BaseMessage + + // Txid identifies the reorged transaction. + Txid chainhash.Hash +} + +// MessageType returns the stable message type identifier. +func (m *TxReorged) MessageType() string { + return "TxReorged" +} + +// txConfirmNotificationSealed seals TxReorged into the package notification +// set. +func (m *TxReorged) txConfirmNotificationSealed() {} + +// TxFinalized notifies a subscriber that the tracked transaction is past +// the backend's reorg-safety depth and will receive no further events. +// Subscribers may use this signal to drop any reorg-recovery bookkeeping +// they were holding for the registration. Not all backends synthesize +// this event (the lndclient transport does not), so consumers must treat +// its absence as a normal operating condition rather than an error. +// +// BlockHeight and NumConfs replay the authoritative confirmation +// numbers carried by the last TxConfirmed before finalization. Because +// reversible TxConfirmed deliveries are fire-and-forget for opt-in +// subscribers and may be dropped on a momentarily-full mailbox, the +// finalization event must carry enough information for height-dependent +// consumers to recover without out-of-band lookups. +type TxFinalized struct { + actor.BaseMessage + + // Txid identifies the finalized transaction. + Txid chainhash.Hash + + // BlockHeight is the height of the block carrying the latest + // observed confirmation (post-any-reorg) at finalization time. + BlockHeight int32 + + // NumConfs is the confirmation count that triggered the + // finalization event — typically the EnsureConfirmedReq's + // TargetConfs. + NumConfs uint32 +} + +// MessageType returns the stable message type identifier. +func (m *TxFinalized) MessageType() string { + return "TxFinalized" +} + +// txConfirmNotificationSealed seals TxFinalized into the package +// notification set. +func (m *TxFinalized) txConfirmNotificationSealed() {} + // TxFailed notifies a subscriber that the actor encountered a terminal // failure while trying to confirm the tracked transaction. type TxFailed struct { diff --git a/txconfirm/reorg_test.go b/txconfirm/reorg_test.go new file mode 100644 index 000000000..dcc49e76e --- /dev/null +++ b/txconfirm/reorg_test.go @@ -0,0 +1,147 @@ +package txconfirm + +import ( + "testing" + + "github.com/lightninglabs/darepo-client/baselib/actor" + "github.com/stretchr/testify/require" +) + +// TestEnsureConfirmedReorgLifecycle drives the full reorg-aware lifecycle +// (Confirmed -> Reorged -> Confirmed -> Finalized) through the +// TxBroadcasterActor and asserts that: +// +// - Each chainsource event reaches the subscriber as a matching public +// notification, in order. +// - The conf watch is held open across the reorg-out / re-confirm +// bounce (no unregister fires before Done). +// - Finalization releases the conf watch and evicts the tracked entry. +func TestEnsureConfirmedReorgLifecycle(t *testing.T) { + chain := newFakeChainSourceRef(100) + ref, _ := newTestActor(t, Config{ + ChainSource: chain, + }) + + tx := makeTestTx(false) + txid := tx.TxHash() + sub := actor.NewChannelTellOnlyRef[Notification]("sub", 16) + + resp := mustEnsure(t, ref.Ref(), &EnsureConfirmedReq{ + Tx: tx, + Subscriber: sub, + }) + require.True(t, resp.Created) + require.Equal(t, TxStateAwaitingConfirmation, resp.State) + require.Equal(t, 1, chain.registerConfCount()) + + // 1. First confirmation on the canonical chain. + chain.emitConfirmation(t, txid, 101) + first := mustAwaitNotification(t, sub) + confirmed, ok := first.(*TxConfirmed) + require.True(t, ok, "first event must be TxConfirmed") + require.Equal(t, txid, confirmed.Txid) + require.Equal(t, int32(101), confirmed.BlockHeight) + + // The conf watch must NOT have been released yet: the entry is + // still in the reversible Confirmed state. + require.Equal(t, 0, chain.unregisterConfCount()) + + // 2. Reorg evicts that confirmation. + chain.emitConfReorged(t, txid) + second := mustAwaitNotification(t, sub) + reorged, ok := second.(*TxReorged) + require.True(t, ok, "second event must be TxReorged") + require.Equal(t, txid, reorged.Txid) + require.Equal(t, 0, chain.unregisterConfCount()) + + // 3. Transaction re-confirms on the new tip. + chain.emitConfirmation(t, txid, 102) + third := mustAwaitNotification(t, sub) + reConfirmed, ok := third.(*TxConfirmed) + require.True(t, ok, "third event must be TxConfirmed") + require.Equal(t, int32(102), reConfirmed.BlockHeight) + require.Equal(t, 0, chain.unregisterConfCount()) + + // 4. Finality. After TxFinalized the entry evicts. + chain.emitConfDone(t, txid) + fourth := mustAwaitNotification(t, sub) + finalized, ok := fourth.(*TxFinalized) + require.True(t, ok, "fourth event must be TxFinalized") + require.Equal(t, txid, finalized.Txid) + + mustEventually(t, func() bool { + return chain.unregisterConfCount() == 1 + }) + + // Cancel for the now-evicted entry must observe an empty map: Removed + // is false because there is nothing to remove. + cancelResp := mustCancel(t, ref.Ref(), &CancelInterestReq{ + Txid: txid, + SubscriberID: sub.ID(), + }) + require.False(t, cancelResp.Removed) +} + +// TestEnsureConfirmedDoneDuringReorgGapDropped pins the documented +// edge-case semantics: if the chainsource backend fires Done while the +// tracked entry is in AwaitingConfirmation (post-reorg, pre-re-confirm), +// txconfirm drops the Done rather than advancing to Finalized. The +// realistic backends (chainntnfs, lndclient) do not fire Done during a +// reorg gap, but this test pins the guard so a future backend change +// cannot silently land the entry in Finalized off a non-Confirmed +// state, and so an accidental relaxation of the guard fails loudly. +func TestEnsureConfirmedDoneDuringReorgGapDropped(t *testing.T) { + chain := newFakeChainSourceRef(100) + ref, _ := newTestActor(t, Config{ + ChainSource: chain, + }) + + tx := makeTestTx(false) + txid := tx.TxHash() + sub := actor.NewChannelTellOnlyRef[Notification]("sub", 16) + + mustEnsure(t, ref.Ref(), &EnsureConfirmedReq{ + Tx: tx, + Subscriber: sub, + }) + + // Confirm, then reorg out — entry is now in AwaitingConfirmation. + chain.emitConfirmation(t, txid, 101) + first := mustAwaitNotification(t, sub) + _, ok := first.(*TxConfirmed) + require.True(t, ok, "first event must be TxConfirmed") + + chain.emitConfReorged(t, txid) + second := mustAwaitNotification(t, sub) + _, ok = second.(*TxReorged) + require.True(t, ok, "second event must be TxReorged") + + // Fire Done out-of-band, before any re-confirmation. txconfirm + // must NOT advance the entry to Finalized, must NOT deliver + // TxFinalized to the subscriber, and must NOT unregister the + // conf watch. + chain.emitConfDone(t, txid) + + // Re-issuing EnsureConfirmedReq for the same (txid, params) is a + // no-op attach that flushes the mailbox: by the time the response + // returns, the queued confirmationDoneMsg has been processed (or + // in this case, logged and dropped). The reported state must + // remain AwaitingConfirmation. + probe := mustEnsure(t, ref.Ref(), &EnsureConfirmedReq{ + Tx: tx, + Subscriber: sub, + }) + require.False(t, probe.Created) + require.Equal( + t, TxStateAwaitingConfirmation, probe.State, + "Done during reorg gap must not promote entry to Finalized", + ) + + // No TxFinalized notification should have been delivered. + mustHaveNoNotification(t, sub) + + require.Equal( + t, 0, chain.unregisterConfCount(), + "dropped Done must not release the conf watch", + ) +} diff --git a/txconfirm/states.go b/txconfirm/states.go index 3555c5ce7..1b173fdad 100644 --- a/txconfirm/states.go +++ b/txconfirm/states.go @@ -249,7 +249,10 @@ func (s *trackedTxStateFeeBumping) ProcessEvent(_ context.Context, } } -// trackedTxStateConfirmed is the terminal confirmed state. +// trackedTxStateConfirmed is the reorg-reversible confirmed state. A +// transaction in this state has reached its target confirmation count on +// the canonical chain; a subsequent reorg moves it back to +// AwaitingConfirmation, and finality moves it to Finalized. type trackedTxStateConfirmed struct { trackedTxData trackedTxProgress @@ -263,19 +266,74 @@ func (s *trackedTxStateConfirmed) String() string { return "Confirmed" } -// IsTerminal returns true because confirmed is terminal. +// IsTerminal returns false because the confirmation is reversible until +// the backend reports finality via trackedTxFinalized. func (s *trackedTxStateConfirmed) IsTerminal() bool { - return true + return false } // trackedTxStateSealed marks trackedTxStateConfirmed as a tracked-tx state. func (s *trackedTxStateConfirmed) trackedTxStateSealed() {} -// ProcessEvent rejects unexpected events in the terminal confirmed state. +// ProcessEvent applies one event to the confirmed state. A reorg moves +// the FSM back to AwaitingConfirmation; finality moves it to the terminal +// Finalized state. func (s *trackedTxStateConfirmed) ProcessEvent(_ context.Context, event trackedTxEvent, _ *trackedTxEnvironment) ( *trackedTxStateTransition, error) { + switch event.(type) { + case *trackedTxReorged: + return &trackedTxStateTransition{ + NextState: &trackedTxStateAwaitingConfirmation{ + trackedTxData: s.trackedTxData, + trackedTxProgress: s.trackedTxProgress, + }, + }, nil + + case *trackedTxFinalized: + return &trackedTxStateTransition{ + NextState: &trackedTxStateFinalized{ + trackedTxData: s.trackedTxData, + trackedTxProgress: s.trackedTxProgress, + ConfirmHeight: s.ConfirmHeight, + }, + }, nil + + default: + return nil, fmt.Errorf("unexpected event %T in %s", event, s) + } +} + +// trackedTxStateFinalized is the terminal "confirmed past reorg safety +// depth" state. No further events are accepted. +type trackedTxStateFinalized struct { + trackedTxData + trackedTxProgress + + // ConfirmHeight is the block height where the tx confirmed before + // being finalized. + ConfirmHeight int32 +} + +// String returns a human-readable representation of the finalized state. +func (s *trackedTxStateFinalized) String() string { + return "Finalized" +} + +// IsTerminal returns true because finalized is terminal. +func (s *trackedTxStateFinalized) IsTerminal() bool { + return true +} + +// trackedTxStateSealed marks trackedTxStateFinalized as a tracked-tx state. +func (s *trackedTxStateFinalized) trackedTxStateSealed() {} + +// ProcessEvent rejects unexpected events in the terminal finalized state. +func (s *trackedTxStateFinalized) ProcessEvent(_ context.Context, + event trackedTxEvent, _ *trackedTxEnvironment) ( + *trackedTxStateTransition, error) { + return nil, fmt.Errorf("unexpected event %T in %s", event, s) } From 85aece8a5fdf7630564278b3e47a5602f1c851a9 Mon Sep 17 00:00:00 2001 From: Elle Mouton Date: Mon, 13 Jul 2026 09:01:14 -0700 Subject: [PATCH 3/8] chainbackends: forward lnd/lndclient reorg and finality signals Forward the lnd and lndclient chain-notifier reorg/finality signals (NegativeConf -> reorg) into the reorg-aware chainsource lifecycle with buffered forwarding. --- chainbackends/lnd.go | 203 ++++++++++++++---- chainbackends/lnd_reorg_test.go | 308 ++++++++++++++++++++++++++++ chainbackends/lndclient_adapters.go | 234 +++++++++++++++++++-- 3 files changed, 696 insertions(+), 49 deletions(-) create mode 100644 chainbackends/lnd_reorg_test.go diff --git a/chainbackends/lnd.go b/chainbackends/lnd.go index 43fab8519..61c9015ca 100644 --- a/chainbackends/lnd.go +++ b/chainbackends/lnd.go @@ -27,6 +27,20 @@ import ( // unexported there and so must be matched by value. const lndNeutrinoBroadcastMsg = "broadcast-unverified" +// reorgSignalBufferSize bounds the buffered reorg notifications forwarded +// from lnd's notifier to a chainsource registration. The forwarder sends +// to this channel with a blocking send, so a full buffer head-of-line +// blocks delivery of unrelated Confirmed / Done events on the same +// forwarder goroutine. A reorg burst (e.g. a multi-block reorg emitting +// one NegativeConf per disconnected block) can produce several signals +// back-to-back; sizing the buffer to absorb a typical reorg depth keeps +// the forwarder moving. The signal is coalescing — the consumer +// re-queries chain state on any reorg — so exact depth need not be +// preserved; the buffer only needs to be deep enough to avoid stalling, +// not to record every event. Eight comfortably covers realistic reorg +// depths while staying negligible in memory. +const reorgSignalBufferSize = 8 + // TxBroadcaster is a minimal interface for broadcasting transactions. This // allows LNDBackend to work with both lnwallet.WalletController (in-process // lnd) and lndclient wrappers (remote lnd via gRPC). @@ -330,37 +344,98 @@ func (b *LNDBackend) RegisterConf(ctx context.Context, txid *chainhash.Hash, // context is released. notifyCtx, cancel := context.WithCancel(context.Background()) - // Create a channel to convert lnd's TxConfirmation to our type. + // Create channels to convert lnd's confirmation lifecycle to our + // backend-agnostic types. NegativeConf carries a reorg depth that + // the lndclient gRPC transport cannot preserve, so we forward the + // forwarder's sequence number instead (see below). confChan := make(chan *chainsource.TxConfirmation, 1) + reorgChan := make(chan uint64, reorgSignalBufferSize) + doneChan := make(chan struct{}, 1) go func() { + // seq is a per-registration monotonic counter stamped onto + // every Confirmed and Reorged signal in the order this single + // forwarder goroutine observes them. Confirmed and Reorged + // leave on separate channels, so a select over both at any + // downstream hop (here, and again in the chainsource conf + // actor) cannot recover their order; the shared sequence lets + // the final consumer apply highest-seq-wins and discard a + // stale signal that lost a cross-channel race. This goroutine + // is the single authoritative ordering point — whatever order + // it reads lnd's channels in is the order the consumer honors. + var seq uint64 + // Defers run in LIFO order. event.Cancel() must run first so + // the upstream notifier stops writing to its internal + // channels before we cancel notifyCtx (which any in-flight + // downstream sends are still using) and finally close the + // outgoing chans. Reversing this order would race the + // upstream notifier against closed channels. defer close(confChan) + defer close(reorgChan) + defer close(doneChan) defer cancel() defer event.Cancel() - select { - case lndConf, ok := <-event.Confirmed: - if !ok { - return - } + for { + select { + case lndConf, ok := <-event.Confirmed: + if !ok { + return + } - conf := &chainsource.TxConfirmation{ - BlockHash: lndConf.BlockHash, - BlockHeight: lndConf.BlockHeight, - TxIndex: lndConf.TxIndex, - Tx: lndConf.Tx, - Block: lndConf.Block, - } + seq++ + conf := &chainsource.TxConfirmation{ + BlockHash: lndConf.BlockHash, + BlockHeight: lndConf.BlockHeight, + TxIndex: lndConf.TxIndex, + Tx: lndConf.Tx, + Block: lndConf.Block, + Seq: seq, + } + + select { + case confChan <- conf: + case <-notifyCtx.Done(): + return + } - confChan <- conf + case _, ok := <-event.NegativeConf: + if !ok { + event.NegativeConf = nil + continue + } - case <-notifyCtx.Done(): - return + seq++ + select { + case reorgChan <- seq: + case <-notifyCtx.Done(): + return + } + + case _, ok := <-event.Done: + if !ok { + event.Done = nil + continue + } + + select { + case doneChan <- struct{}{}: + case <-notifyCtx.Done(): + return + } + + return + + case <-notifyCtx.Done(): + return + } } }() return &chainsource.ConfRegistration{ Confirmed: confChan, + Reorged: reorgChan, + Done: doneChan, Cancel: func() { cancel() event.Cancel() @@ -393,39 +468,97 @@ func (b *LNDBackend) RegisterSpend(ctx context.Context, outpoint *wire.OutPoint, // Keep spend delivery alive independently of the actor request context. notifyCtx, cancel := context.WithCancel(context.Background()) - // Create a channel to convert lnd's SpendDetail to our type. + // Create channels to convert lnd's spend lifecycle to our + // backend-agnostic types. lnd's spend Reorg carries no payload, so + // we forward the forwarder's sequence number instead (see below). spendChan := make(chan *chainsource.SpendDetail, 1) + reorgChan := make(chan uint64, reorgSignalBufferSize) + doneChan := make(chan struct{}, 1) - // Start a goroutine to convert and forward the spend. + // Start a goroutine to convert and forward spend lifecycle events. go func() { + // seq is a per-registration monotonic counter stamped onto + // every Spend and Reorged signal in the order this single + // forwarder observes them. Spend and Reorged leave on separate + // channels, so a downstream select cannot recover their order; + // the shared sequence lets the final consumer apply + // highest-seq-wins and discard a stale signal that lost a + // cross-channel race. This goroutine is the single + // authoritative ordering point. + var seq uint64 + // Defer order is LIFO: event.Cancel() stops the upstream + // notifier first, then cancel() ends notifyCtx, and only + // then are the outgoing channels closed. This avoids a race + // where the upstream notifier writes to a freshly-closed + // downstream channel. defer close(spendChan) + defer close(reorgChan) + defer close(doneChan) defer cancel() defer event.Cancel() - select { - case lndSpend, ok := <-event.Spend: - if !ok { - return - } + for { + select { + case lndSpend, ok := <-event.Spend: + if !ok { + return + } - // Convert to our type. - spend := &chainsource.SpendDetail{ - SpentOutPoint: lndSpend.SpentOutPoint, - SpenderTxHash: lndSpend.SpenderTxHash, - SpendingTx: lndSpend.SpendingTx, - SpenderInputIndex: lndSpend.SpenderInputIndex, - SpendingHeight: lndSpend.SpendingHeight, - } + // Convert to our type. + seq++ + spend := &chainsource.SpendDetail{ + SpentOutPoint: lndSpend.SpentOutPoint, + SpenderTxHash: lndSpend.SpenderTxHash, + SpendingTx: lndSpend.SpendingTx, + SpenderInputIndex: lndSpend. + SpenderInputIndex, + SpendingHeight: lndSpend.SpendingHeight, + Seq: seq, + } + + select { + case spendChan <- spend: + case <-notifyCtx.Done(): + return + } + + case _, ok := <-event.Reorg: + if !ok { + event.Reorg = nil + continue + } + + seq++ + select { + case reorgChan <- seq: + case <-notifyCtx.Done(): + return + } + + case _, ok := <-event.Done: + if !ok { + event.Done = nil + continue + } + + select { + case doneChan <- struct{}{}: + case <-notifyCtx.Done(): + return + } - spendChan <- spend + return - case <-notifyCtx.Done(): - return + case <-notifyCtx.Done(): + return + } } }() return &chainsource.SpendRegistration{ - Spend: spendChan, + Spend: spendChan, + Reorged: reorgChan, + Done: doneChan, Cancel: func() { cancel() event.Cancel() diff --git a/chainbackends/lnd_reorg_test.go b/chainbackends/lnd_reorg_test.go new file mode 100644 index 000000000..622f8516d --- /dev/null +++ b/chainbackends/lnd_reorg_test.go @@ -0,0 +1,308 @@ +package chainbackends + +import ( + "testing" + "time" + + "github.com/btcsuite/btcd/chainhash/v2" + "github.com/btcsuite/btcd/wire/v2" + "github.com/lightninglabs/darepo-client/chainsource" + "github.com/lightningnetwork/lnd/chainntnfs" + "github.com/stretchr/testify/require" +) + +// reorgWaitTimeout is the per-step deadline used by the LNDBackend +// forwarder reorg tests. The forwarder is a tight goroutine, so the +// timeout exists only to make a hang surface as a fast failure on slow CI. +const reorgWaitTimeout = 2 * time.Second + +// TestRegisterConfForwardsReorgAndDone drives the full confirmation +// lifecycle through the chainntnfs notifier into the LNDBackend forwarder +// and asserts each event arrives on the matching chainsource registration +// channel. The lifecycle is: +// +// Confirmed -> NegativeConf -> Confirmed -> Done +// +// and the test additionally verifies the forwarder exits after Done by +// observing that the chainsource channels close. +func TestRegisterConfForwardsReorgAndDone(t *testing.T) { + t.Parallel() + + confChan := make(chan *chainntnfs.TxConfirmation, 2) + negChan := make(chan int32, 1) + doneChan := make(chan struct{}, 1) + notifier := &stubNotifier{ + confEvent: &chainntnfs.ConfirmationEvent{ + Confirmed: confChan, + NegativeConf: negChan, + Done: doneChan, + Cancel: func() {}, + }, + } + backend := NewLNDBackend( + notifier, &stubFeeEstimator{}, &stubBroadcaster{}, + ) + + reg, err := backend.RegisterConf( + t.Context(), &chainhash.Hash{0x42}, []byte{0x51}, 1, 100, false, + ) + require.NoError(t, err) + + // 1. First confirmation crosses the forwarder. + hash1 := chainhash.Hash{0xaa} + confChan <- &chainntnfs.TxConfirmation{ + BlockHash: &hash1, + BlockHeight: 123, + Tx: wire.NewMsgTx(2), + } + + conf1 := awaitConfForward(t, reg.Confirmed) + require.Equal(t, uint32(123), conf1.BlockHeight) + require.Equal(t, hash1, *conf1.BlockHash) + + // 2. Reorg ping is forwarded as a single struct{} on Reorged. The + // depth value carried on NegativeConf is intentionally dropped at + // this layer; consumers must not rely on it. + negChan <- 1 + + awaitSeq(t, reg.Reorged, "Reorged forward") + + // 3. Transaction re-confirms in a different block on the new tip. + hash2 := chainhash.Hash{0xbb} + confChan <- &chainntnfs.TxConfirmation{ + BlockHash: &hash2, + BlockHeight: 124, + Tx: wire.NewMsgTx(2), + } + + conf2 := awaitConfForward(t, reg.Confirmed) + require.Equal(t, uint32(124), conf2.BlockHeight) + require.Equal(t, hash2, *conf2.BlockHash) + + // 4. Done signal is forwarded; the forwarder then exits and the + // chainsource channels close. + doneChan <- struct{}{} + + awaitStruct(t, reg.Done, "Done forward") + + // All three forwarded channels must close once the forwarder exits. + requireConfClosedSoon(t, reg.Confirmed) + requireSeqClosedSoon(t, reg.Reorged) + requireStructClosedSoon(t, reg.Done) +} + +// TestRegisterSpendForwardsReorgAndDone is the spend-side equivalent of +// the confirmation lifecycle test above. +func TestRegisterSpendForwardsReorgAndDone(t *testing.T) { + t.Parallel() + + spendChan := make(chan *chainntnfs.SpendDetail, 2) + reorgChan := make(chan struct{}, 1) + doneChan := make(chan struct{}, 1) + notifier := &stubNotifier{ + spendEvent: &chainntnfs.SpendEvent{ + Spend: spendChan, + Reorg: reorgChan, + Done: doneChan, + Cancel: func() {}, + }, + } + backend := NewLNDBackend( + notifier, &stubFeeEstimator{}, &stubBroadcaster{}, + ) + + outpoint := &wire.OutPoint{Index: 1} + reg, err := backend.RegisterSpend( + t.Context(), outpoint, []byte{0x51}, 100, + ) + require.NoError(t, err) + + // 1. First spend. + hash1 := chainhash.Hash{0x10} + spendChan <- &chainntnfs.SpendDetail{ + SpentOutPoint: outpoint, + SpenderTxHash: &hash1, + SpendingTx: wire.NewMsgTx(2), + SpendingHeight: 150, + } + + spend1 := awaitSpendForward(t, reg.Spend) + require.Equal(t, int32(150), spend1.SpendingHeight) + require.Equal(t, hash1, *spend1.SpenderTxHash) + + // 2. Reorg evicts the spend. + reorgChan <- struct{}{} + + awaitSeq(t, reg.Reorged, "spend Reorged forward") + + // 3. A different spender wins the new chain. + hash2 := chainhash.Hash{0x20} + spendChan <- &chainntnfs.SpendDetail{ + SpentOutPoint: outpoint, + SpenderTxHash: &hash2, + SpendingTx: wire.NewMsgTx(2), + SpendingHeight: 151, + } + + spend2 := awaitSpendForward(t, reg.Spend) + require.Equal(t, int32(151), spend2.SpendingHeight) + require.Equal(t, hash2, *spend2.SpenderTxHash) + + // 4. Done. + doneChan <- struct{}{} + + awaitStruct(t, reg.Done, "spend Done forward") + + requireSpendClosedSoon(t, reg.Spend) + requireSeqClosedSoon(t, reg.Reorged) + requireStructClosedSoon(t, reg.Done) +} + +// awaitConfForward reads a single confirmation off the forwarded channel +// with a deadline, failing the test on timeout or unexpected close. +func awaitConfForward(t *testing.T, + ch <-chan *chainsource.TxConfirmation) *chainsource.TxConfirmation { + + t.Helper() + + select { + case conf, ok := <-ch: + if !ok { + t.Fatal("conf channel closed before delivery") + } + + return conf + + case <-time.After(reorgWaitTimeout): + t.Fatal("timeout waiting for conf forward") + + return nil + } +} + +// awaitSpendForward reads a single spend off the forwarded channel with a +// deadline, failing the test on timeout or unexpected close. +func awaitSpendForward(t *testing.T, + ch <-chan *chainsource.SpendDetail) *chainsource.SpendDetail { + + t.Helper() + + select { + case spend, ok := <-ch: + if !ok { + t.Fatal("spend channel closed before delivery") + } + + return spend + + case <-time.After(reorgWaitTimeout): + t.Fatal("timeout waiting for spend forward") + + return nil + } +} + +// awaitStruct reads a single struct{} off the forwarded channel with a +// deadline. Used for the Reorged and Done channels. +func awaitStruct(t *testing.T, ch <-chan struct{}, label string) { + t.Helper() + + select { + case _, ok := <-ch: + if !ok { + t.Fatalf("%s channel closed before delivery", label) + } + + case <-time.After(reorgWaitTimeout): + t.Fatalf("timeout waiting for %s", label) + } +} + +// requireConfClosedSoon asserts the confirmation channel closes within +// the reorg wait timeout. Used to verify the forwarder's defer-close +// chain runs after Done is delivered. +func requireConfClosedSoon(t *testing.T, + ch <-chan *chainsource.TxConfirmation) { + + t.Helper() + + require.Eventually(t, func() bool { + select { + case _, ok := <-ch: + return !ok + + default: + return false + } + }, reorgWaitTimeout, 10*time.Millisecond, + "conf channel did not close after Done") +} + +// requireSpendClosedSoon asserts the spend channel closes within the +// reorg wait timeout. +func requireSpendClosedSoon(t *testing.T, ch <-chan *chainsource.SpendDetail) { + t.Helper() + + require.Eventually(t, func() bool { + select { + case _, ok := <-ch: + return !ok + + default: + return false + } + }, reorgWaitTimeout, 10*time.Millisecond, + "spend channel did not close after Done") +} + +// requireStructClosedSoon asserts that a struct{} signal channel closes +// within the reorg wait timeout. +func requireStructClosedSoon(t *testing.T, ch <-chan struct{}) { + t.Helper() + + require.Eventually(t, func() bool { + select { + case _, ok := <-ch: + return !ok + + default: + return false + } + }, reorgWaitTimeout, 10*time.Millisecond, + "struct channel did not close after Done") +} + +// awaitSeq reads a single sequence number off the forwarded Reorged +// channel with a deadline. The reorg signal now carries the forwarder's +// monotonic sequence (see chainsource.TxConfirmation.Seq) rather than a +// bare struct{}. +func awaitSeq(t *testing.T, ch <-chan uint64, label string) { + t.Helper() + + select { + case _, ok := <-ch: + if !ok { + t.Fatalf("%s channel closed before delivery", label) + } + + case <-time.After(reorgWaitTimeout): + t.Fatalf("timeout waiting for %s", label) + } +} + +// requireSeqClosedSoon asserts that a sequence-carrying signal channel +// closes within the reorg wait timeout. +func requireSeqClosedSoon(t *testing.T, ch <-chan uint64) { + t.Helper() + + require.Eventually(t, func() bool { + select { + case _, ok := <-ch: + return !ok + + default: + return false + } + }, reorgWaitTimeout, 10*time.Millisecond, + "seq channel did not close after Done") +} diff --git a/chainbackends/lndclient_adapters.go b/chainbackends/lndclient_adapters.go index 326a93fb0..38cab8ee4 100644 --- a/chainbackends/lndclient_adapters.go +++ b/chainbackends/lndclient_adapters.go @@ -17,6 +17,16 @@ import ( "github.com/lightningnetwork/lnd/lnwallet/chainfee" ) +// lndRegistrationTimeout bounds how long a conf/spend registration call into +// lndclient may block before we give up and return an error. lnd can be slow +// to answer a RegisterConfirmationsNtfn / RegisterSpendNtfn while it is busy +// processing a fresh block, and a registration that hangs forever would pin +// the chainsource sub-actor's Receive call and back-pressure the factory +// actor onto every other in-flight registration. Fifteen seconds is well +// above lnd's normal under-load response time yet short enough that a +// genuinely wedged backend surfaces as an error rather than a silent hang. +const lndRegistrationTimeout = 15 * time.Second + // LndClientTxBroadcaster implements TxBroadcaster using // lndclient.WalletKitClient. type LndClientTxBroadcaster struct { @@ -131,7 +141,16 @@ func (n *LndClientChainNotifier) RegisterConfirmationsNtfn(txid *chainhash.Hash, opt(notifierOpts) } - var lndOpts []lndclient.NotifierOption + // Ask lndclient to keep the confirmation stream alive past the first + // Confirmed event and forward any subsequent reorg signal on a + // dedicated channel. Without WithReOrgChan, lndclient's receive + // loop tears the stream down after one delivery and any later + // reorg is silently dropped at the gRPC layer. + reorgPing := make(chan struct{}, 1) + + lndOpts := []lndclient.NotifierOption{ + lndclient.WithReOrgChan(reorgPing), + } if notifierOpts.IncludeBlock { lndOpts = append(lndOpts, lndclient.WithIncludeBlock()) } @@ -177,11 +196,11 @@ func (n *LndClientChainNotifier) RegisterConfirmationsNtfn(txid *chainhash.Hash, confChan = r.confChan errChan = r.errChan - case <-time.After(15 * time.Second): + case <-time.After(lndRegistrationTimeout): cancel() - return nil, fmt.Errorf("register confirmations timed out " + - "after 15s") + return nil, fmt.Errorf("register confirmations timed out "+ + "after %s", lndRegistrationTimeout) } go func() { @@ -199,26 +218,147 @@ func (n *LndClientChainNotifier) RegisterConfirmationsNtfn(txid *chainhash.Hash, } }() + // Forward the confirmation lifecycle through a SINGLE goroutine so the + // reorg ping and the (re-)confirmation reach the downstream chainntnfs + // channels in lndclient's emission order. lndclient drives both off one + // ordered gRPC receive loop and writes the reorg ping before the + // replacement Confirmed, but it splits them across two channels + // (confChan and reorgPing); if we forwarded each on its own goroutine, + // the downstream ConfActor's select could consume a re-Confirmed before + // the Reorged, reset confirmHeight to 0 with no further Confirmed + // coming, and strand the watch. Draining a pending reorg with priority + // on each iteration, and handing every event off with a blocking send, + // makes the ConfActor observe exactly the forwarder's (lndclient's) + // order. + // + // lndclient does not preserve the reorg depth across the gRPC boundary, + // so a sentinel value of 0 is forwarded on NegativeConf; callers over + // this transport must not rely on the integer value. + orderedConfirmed := make(chan *chainntnfs.TxConfirmation, 1) + negativeConf := make(chan int32, 1) + go func() { + defer close(orderedConfirmed) + defer close(negativeConf) + + forwardOrderedReorg( + ctx, reorgPing, confChan, orderedConfirmed, + negativeConf, + ) + }() + + // Done is allocated but never written to because lnd's internal + // "past reorg-safety depth" signal is not surfaced through the + // lndclient gRPC transport. Consumers needing such a gate must + // compute it themselves from block height. return &chainntnfs.ConfirmationEvent{ - Confirmed: confChan, - Cancel: cancel, - Done: make(chan struct{}, 1), + Confirmed: orderedConfirmed, + NegativeConf: negativeConf, + Cancel: cancel, + Done: make(chan struct{}, 1), }, nil } +// forwardOrderedReorg copies confirmations and reorg pings from lndclient's two +// source channels onto the downstream Confirmed / NegativeConf channels, +// forwarding each event in the order this single goroutine observes it. It does +// not bias either channel: the authoritative lifecycle ordering is +// re-established downstream by the per-registration sequence number the +// LNDBackend forwarder stamps (see chainsource.TxConfirmation.Seq), so the +// consumer applies highest-seq-wins regardless of how near-simultaneous events +// interleave here. lndclient's two-channel split makes a perfectly ordered +// merge impossible at this layer, so forwarding in natural arrival order keeps +// the stamped sequence faithful to what was actually observed rather than +// injecting an artificial reorg-first bias. lndclient does not preserve the +// reorg depth across the gRPC boundary, so a sentinel value of 0 is forwarded +// on NegativeConf; callers over this transport must not rely on the value. +func forwardOrderedReorg(ctx context.Context, reorgPing <-chan struct{}, + confChan <-chan *chainntnfs.TxConfirmation, + outConfirmed chan<- *chainntnfs.TxConfirmation, + outNegConf chan<- int32) { + + for confChan != nil || reorgPing != nil { + select { + case _, ok := <-reorgPing: + if !ok { + reorgPing = nil + + continue + } + + select { + case outNegConf <- 0: + case <-ctx.Done(): + return + } + + case c, ok := <-confChan: + if !ok { + confChan = nil + + continue + } + + select { + case outConfirmed <- c: + case <-ctx.Done(): + return + } + + case <-ctx.Done(): + return + } + } +} + // RegisterSpendNtfn registers for spend notifications using lndclient's // ChainNotifier. func (n *LndClientChainNotifier) RegisterSpendNtfn(outpoint *wire.OutPoint, pkScript []byte, heightHint uint32) (*chainntnfs.SpendEvent, error) { + // Ask lndclient to keep the spend stream alive past the first + // Spend event so it can forward reorg pings. Without WithReOrgChan + // the stream is torn down after the first delivery and any later + // spend reorg is silently dropped at the gRPC layer. + reorgPing := make(chan struct{}, 1) + ctx, cancel := context.WithCancel(context.Background()) - spendChan, errChan, err := n.cfg.LND.ChainNotifier.RegisterSpendNtfn( - ctx, outpoint, pkScript, int32(heightHint), - ) - if err != nil { + + // Run the registration in a goroutine with a timeout to prevent + // hanging when LND is slow under block load, mirroring the conf path. + type regResult struct { + spendChan chan *chainntnfs.SpendDetail + errChan chan error + err error + } + + resultCh := make(chan regResult, 1) + go func() { + sc, ec, err := n.cfg.LND.ChainNotifier.RegisterSpendNtfn( + ctx, outpoint, pkScript, int32(heightHint), + lndclient.WithReOrgChan(reorgPing), + ) + resultCh <- regResult{sc, ec, err} + }() + + var spendChan chan *chainntnfs.SpendDetail + var errChan chan error + + select { + case r := <-resultCh: + if r.err != nil { + cancel() + + return nil, fmt.Errorf("register spend: %w", r.err) + } + + spendChan = r.spendChan + errChan = r.errChan + + case <-time.After(lndRegistrationTimeout): cancel() - return nil, fmt.Errorf("register spend: %w", err) + return nil, fmt.Errorf("register spend timed out after %s", + lndRegistrationTimeout) } go func() { @@ -236,14 +376,80 @@ func (n *LndClientChainNotifier) RegisterSpendNtfn(outpoint *wire.OutPoint, } }() + // Forward the spend lifecycle through a SINGLE goroutine so the reorg + // ping and the (re-)spend reach the downstream chainntnfs channels in + // lndclient's emission order, for the same reason as the conf path: the + // two-channel split (spendChan and reorgPing) would otherwise let the + // downstream SpendActor's select consume a re-Spend before the Reorged + // and strand the watch. Draining a pending reorg with priority and + // using blocking hand-offs makes the SpendActor observe lndclient's + // order. + orderedSpend := make(chan *chainntnfs.SpendDetail, 1) + reorgChan := make(chan struct{}, 1) + go func() { + defer close(orderedSpend) + defer close(reorgChan) + + forwardOrderedSpendReorg( + ctx, reorgPing, spendChan, orderedSpend, reorgChan, + ) + }() + + // Done is allocated but never written to because lnd's "past + // reorg-safety depth" signal is not surfaced through the lndclient + // gRPC transport. return &chainntnfs.SpendEvent{ - Spend: spendChan, - Reorg: make(chan struct{}, 1), + Spend: orderedSpend, + Reorg: reorgChan, Done: make(chan struct{}, 1), Cancel: cancel, }, nil } +// forwardOrderedSpendReorg is the spend-path analogue of forwardOrderedReorg: +// it copies spends and reorg pings from lndclient's two source channels onto +// the downstream Spend / Reorg channels in the order this single goroutine +// observes them, without biasing either channel. The authoritative lifecycle +// ordering is re-established downstream by the per-registration sequence number +// the LNDBackend forwarder stamps (see chainsource.SpendDetail.Seq). +func forwardOrderedSpendReorg(ctx context.Context, reorgPing <-chan struct{}, + spendChan <-chan *chainntnfs.SpendDetail, + outSpend chan<- *chainntnfs.SpendDetail, outReorg chan<- struct{}) { + + for spendChan != nil || reorgPing != nil { + select { + case _, ok := <-reorgPing: + if !ok { + reorgPing = nil + + continue + } + + select { + case outReorg <- struct{}{}: + case <-ctx.Done(): + return + } + + case sp, ok := <-spendChan: + if !ok { + spendChan = nil + + continue + } + + select { + case outSpend <- sp: + case <-ctx.Done(): + return + } + + case <-ctx.Done(): + return + } + } +} + // RegisterBlockEpochNtfn registers for block epoch notifications using // lndclient's ChainNotifier. func (n *LndClientChainNotifier) RegisterBlockEpochNtfn( From bc889934e9c532c8602f316db66dc0b3bee20921 Mon Sep 17 00:00:00 2001 From: Elle Mouton Date: Mon, 13 Jul 2026 09:01:14 -0700 Subject: [PATCH 4/8] lwwallet: reorg-aware Esplora chain backend (TipPoller) Reorg-aware lwwallet/Esplora backend: TipPoller same-height and deeper reorg detection via PrevBlock continuity, a unified ChainEvent stream, and BlockDisconnected emission to btcwallet before connecting the replacement tip. --- lwwallet/chain_backend.go | 706 +++++++++++++--- lwwallet/chain_backend_reorg_test.go | 1136 ++++++++++++++++++++++++++ lwwallet/chain_backend_test.go | 59 +- lwwallet/esplora_chain.go | 99 ++- lwwallet/esplora_chain_reorg_test.go | 175 ++++ lwwallet/esplora_chain_test.go | 11 + lwwallet/tip_poller.go | 824 ++++++++++++++++++- lwwallet/tip_poller_reorg_test.go | 274 +++++++ lwwallet/tip_poller_test.go | 95 ++- 9 files changed, 3167 insertions(+), 212 deletions(-) create mode 100644 lwwallet/chain_backend_reorg_test.go create mode 100644 lwwallet/esplora_chain_reorg_test.go create mode 100644 lwwallet/tip_poller_reorg_test.go diff --git a/lwwallet/chain_backend.go b/lwwallet/chain_backend.go index 573bec48d..442700ac4 100644 --- a/lwwallet/chain_backend.go +++ b/lwwallet/chain_backend.go @@ -16,6 +16,7 @@ import ( "github.com/btcsuite/btcd/wire/v2" "github.com/btcsuite/btclog/v2" "github.com/lightninglabs/darepo-client/chainsource" + fn "github.com/lightningnetwork/lnd/fn/v2" "golang.org/x/sync/singleflight" ) @@ -42,6 +43,29 @@ const ( // raise Esplora load over the default poll cadence. const recheckHeartbeatInterval = 60 * time.Second +// regState is the lifecycle state of a conf/spend registration. +// +// stateWatching: no positive event has been delivered yet (or the +// last positive event was reorged out). The next checkSingle... that +// finds the tx confirmed / outpoint spent will emit Confirmed / +// Spend and transition to statePositive. +// +// statePositive: a Confirmed / Spend event has been delivered and the +// associated block-hash is cached on the registration. The next +// checkSingle... that does NOT find the tx confirmed / outpoint +// spent in the same block leaves the registration alone (the +// chainsource actor handles Done synthesis at FinalityDepth via +// block epochs). A reorg event that names the cached block-hash in +// its Disconnected set fires Reorged, resets cached state, and +// transitions back to stateWatching so the next re-check can fire +// Confirmed / Spend again on the new chain. +type regState uint8 + +const ( + stateWatching regState = iota + statePositive +) + // confRegistration tracks a pending confirmation registration within the // polling loop. type confRegistration struct { @@ -63,8 +87,37 @@ type confRegistration struct { // confChan is the channel to send the confirmation on. confChan chan *chainsource.TxConfirmation + // reorgChan is the channel that fires when a previously delivered + // confirmation is reorged out of the canonical chain. Buffered to + // 1; never written by the backend before stateWatching has been + // re-entered. The value is the chainsource ordering sequence; this + // backend does not stamp sequences (it emits conf/reorg from a + // single ordered handler goroutine), so it always sends 0, which + // the chainsource actor treats as always-apply. + reorgChan chan uint64 + + // doneChan is allocated for API symmetry with the chainsource + // contract. The Esplora-backed backend does not write to it: the + // chainsource ConfActor synthesizes Done at FinalityDepth from + // block epochs once the backend stops re-firing events. + doneChan chan struct{} + // cancelCh signals that this registration has been cancelled. cancelCh chan struct{} + + // regMu guards the fields below that mutate over the + // registration's lifetime. It is held only briefly during state + // transitions; channel sends are performed without it. + regMu sync.Mutex + + // state is the current lifecycle state. + state regState + + // lastBlockHash is the BlockHash of the last delivered + // confirmation when state == statePositive. Used to detect when + // a reorg's Disconnected hash list invalidates this + // registration's last event. + lastBlockHash chainhash.Hash } // spendRegistration tracks a pending spend registration within the @@ -82,8 +135,34 @@ type spendRegistration struct { // spendChan is the channel to send the spend detail on. spendChan chan *chainsource.SpendDetail + // reorgChan is the channel that fires when a previously delivered + // spend is reorged out of the canonical chain. Buffered to 1. + reorgChan chan uint64 + + // doneChan is allocated for API symmetry with the chainsource + // contract. The Esplora-backed backend does not write to it: the + // chainsource SpendActor synthesizes Done at FinalityDepth. + doneChan chan struct{} + // cancelCh signals that this registration has been cancelled. cancelCh chan struct{} + + // regMu guards the fields below. + regMu sync.Mutex + + // state is the current lifecycle state. + state regState + + // lastSpenderHash is the SpenderTxHash of the last delivered + // spend when state == statePositive. + lastSpenderHash chainhash.Hash + + // lastSpendingBlockHash is the block hash that the last + // delivered spend confirmed in, parsed from the same + // outspend response that confirmed the spend. Reorg + // comparison against ReorgEvent.Disconnected uses this so + // the spend-watch path does not need an extra HTTP round-trip. + lastSpendingBlockHash chainhash.Hash } // blockRegistration tracks a block epoch subscription. @@ -205,7 +284,8 @@ func (b *ChainBackend) Start() error { } } - height, hash, _, sub, err := b.tipPoller.BestBlockAndSubscribe() + height, hash, _, chainSub, err := + b.tipPoller.BestBlockAndSubscribeChain() if err != nil { return fmt.Errorf("subscribe to tip poller: %w", err) } @@ -216,7 +296,7 @@ func (b *ChainBackend) Start() error { b.mu.Unlock() b.wg.Add(1) - go b.handleTipEvents(sub) + go b.handleChainEvents(chainSub) b.log.InfoS(context.Background(), "Chain backend started", slog.Int("tip_height", int(height)), @@ -388,11 +468,20 @@ func (b *ChainBackend) SubmitPackage(ctx context.Context, parents []*wire.MsgTx, } // RegisterConf registers for confirmation notifications of a transaction. +// The registration is reorg-aware: the returned ConfRegistration's +// Reorged channel fires when a previously delivered confirmation is +// reorged out of the canonical chain, and the Confirmed channel may +// fire again when the tx re-confirms on the new chain. The Done +// channel is allocated but never written by this backend; the +// chainsource ConfActor synthesizes Done at its configured +// FinalityDepth using block epochs. func (b *ChainBackend) RegisterConf(ctx context.Context, txid *chainhash.Hash, pkScript []byte, numConfs uint32, heightHint uint32, includeBlock bool) (*chainsource.ConfRegistration, error) { confChan := make(chan *chainsource.TxConfirmation, 1) + reorgChan := make(chan uint64, 1) + doneChan := make(chan struct{}, 1) cancelCh := make(chan struct{}) reg := &confRegistration{ @@ -402,6 +491,8 @@ func (b *ChainBackend) RegisterConf(ctx context.Context, txid *chainhash.Hash, heightHint: heightHint, includeBlock: includeBlock, confChan: confChan, + reorgChan: reorgChan, + doneChan: doneChan, cancelCh: cancelCh, } @@ -424,11 +515,7 @@ func (b *ChainBackend) RegisterConf(ctx context.Context, txid *chainhash.Hash, ) cancelFn := func() { - close(cancelCh) - - b.mu.Lock() - delete(b.confRegs, id) - b.mu.Unlock() + b.cancelConfReg(id, reg) } // Run an immediate single-shot check scoped to JUST this @@ -450,16 +537,41 @@ func (b *ChainBackend) RegisterConf(ctx context.Context, txid *chainhash.Hash, return &chainsource.ConfRegistration{ Confirmed: confChan, + Reorged: reorgChan, + Done: doneChan, Cancel: cancelFn, }, nil } +// cancelConfReg tears down a confirmation registration. It is safe +// to invoke from any state; the close(cancelCh) is guarded by a +// once-style check to make double-Cancel a no-op. +func (b *ChainBackend) cancelConfReg(id uint64, reg *confRegistration) { + reg.regMu.Lock() + select { + case <-reg.cancelCh: + // Already cancelled. + reg.regMu.Unlock() + + return + + default: + } + close(reg.cancelCh) + reg.regMu.Unlock() + + b.mu.Lock() + delete(b.confRegs, id) + b.mu.Unlock() +} + // runConfOneShot performs the per-registration confirmation check // triggered at RegisterConf time. It snapshots the current best // height under the chain backend's lock, asks Esplora for this one // registration's status, and delivers the result if confirmed. The -// goroutine exits on any of: cancellation, stopCh closure, a -// successful delivery, or a non-confirmed status. +// registration is NOT deleted after delivery: a reorg may later +// reset it to stateWatching and the next re-check must fire +// Confirmed again on the new chain. func (b *ChainBackend) runConfOneShot(id uint64, reg *confRegistration) { defer b.wg.Done() @@ -477,11 +589,49 @@ func (b *ChainBackend) runConfOneShot(id uint64, reg *confRegistration) { currentHeight := b.bestHeight b.mu.Unlock() + b.deliverConfIfNew(id, reg, currentHeight) +} + +// deliverConfIfNew runs a confirmation re-check and fires the +// Confirmed channel iff the registration is in stateWatching and a +// confirmation is now available. It is the single delivery path +// shared by the one-shot at registration time and the broad +// re-check driven by tip / reorg events. The registration's regMu +// is held only across the state read and the state transition; the +// channel send happens outside the lock so a slow consumer never +// blocks the broad re-check goroutine. +func (b *ChainBackend) deliverConfIfNew(id uint64, reg *confRegistration, + currentHeight int32) { + + reg.regMu.Lock() + if reg.state == statePositive { + reg.regMu.Unlock() + + return + } + reg.regMu.Unlock() + conf := b.checkSingleConf(reg, currentHeight) if conf == nil { return } + // Re-check state under the lock: a concurrent reorg handler + // could have flipped us back to stateWatching after the check + // started, but it could not have flipped us forward to + // statePositive (that's exclusively this function's job). + reg.regMu.Lock() + if reg.state == statePositive { + reg.regMu.Unlock() + + return + } + reg.state = statePositive + if conf.BlockHash != nil { + reg.lastBlockHash = *conf.BlockHash + } + reg.regMu.Unlock() + select { case reg.confChan <- conf: case <-reg.cancelCh: @@ -493,22 +643,26 @@ func (b *ChainBackend) runConfOneShot(id uint64, reg *confRegistration) { b.log.DebugS( context.Background(), - "Confirmation registration fulfilled (one-shot)", + "Confirmation registration fulfilled", slog.Uint64("reg_id", id), slog.Int("block_height", int(conf.BlockHeight)), ) - - b.mu.Lock() - delete(b.confRegs, id) - b.mu.Unlock() } // RegisterSpend registers for spend notifications of a transaction output. +// The registration is reorg-aware: the returned SpendRegistration's +// Reorged channel fires when a previously delivered spend is reorged +// out of the canonical chain, and the Spend channel may fire again +// when the outpoint is re-spent on the new chain. The Done channel +// is allocated but never written by this backend; the chainsource +// SpendActor synthesizes Done at its configured FinalityDepth. func (b *ChainBackend) RegisterSpend(ctx context.Context, outpoint *wire.OutPoint, pkScript []byte, heightHint uint32) ( *chainsource.SpendRegistration, error) { spendChan := make(chan *chainsource.SpendDetail, 1) + reorgChan := make(chan uint64, 1) + doneChan := make(chan struct{}, 1) cancelCh := make(chan struct{}) reg := &spendRegistration{ @@ -516,6 +670,8 @@ func (b *ChainBackend) RegisterSpend(ctx context.Context, pkScript: pkScript, heightHint: heightHint, spendChan: spendChan, + reorgChan: reorgChan, + doneChan: doneChan, cancelCh: cancelCh, } @@ -537,11 +693,7 @@ func (b *ChainBackend) RegisterSpend(ctx context.Context, ) cancelFn := func() { - close(cancelCh) - - b.mu.Lock() - delete(b.spendRegs, id) - b.mu.Unlock() + b.cancelSpendReg(id, reg) } // Per-registration one-shot to handle outpoints that are @@ -552,16 +704,37 @@ func (b *ChainBackend) RegisterSpend(ctx context.Context, go b.runSpendOneShot(id, reg) return &chainsource.SpendRegistration{ - Spend: spendChan, - Cancel: cancelFn, + Spend: spendChan, + Reorged: reorgChan, + Done: doneChan, + Cancel: cancelFn, }, nil } +// cancelSpendReg tears down a spend registration. Idempotent: a +// double-Cancel is treated as a no-op. +func (b *ChainBackend) cancelSpendReg(id uint64, reg *spendRegistration) { + reg.regMu.Lock() + select { + case <-reg.cancelCh: + reg.regMu.Unlock() + + return + + default: + } + close(reg.cancelCh) + reg.regMu.Unlock() + + b.mu.Lock() + delete(b.spendRegs, id) + b.mu.Unlock() +} + // runSpendOneShot performs the per-registration spend check -// triggered at RegisterSpend time. It exits on cancellation, stopCh -// closure, a successful delivery, or any non-spent / unconfirmed -// status; the broad checkSpends called from processTipEvent re-runs -// it on every tip advance. +// triggered at RegisterSpend time. The registration is NOT deleted +// after delivery: a reorg may later reset it to stateWatching and +// the next re-check must fire Spend again on the new chain. func (b *ChainBackend) runSpendOneShot(id uint64, reg *spendRegistration) { defer b.wg.Done() @@ -579,11 +752,46 @@ func (b *ChainBackend) runSpendOneShot(id uint64, reg *spendRegistration) { return } - detail := b.checkSingleSpend(reg) + b.deliverSpendIfNew(id, reg) +} + +// deliverSpendIfNew runs a spend re-check and fires the Spend +// channel iff the registration is in stateWatching and a spend is +// now available. Mirror of deliverConfIfNew; see that function's +// comment for the regMu / channel-send ordering rationale. +func (b *ChainBackend) deliverSpendIfNew(id uint64, reg *spendRegistration) { + reg.regMu.Lock() + if reg.state == statePositive { + reg.regMu.Unlock() + + return + } + reg.regMu.Unlock() + + // checkSingleSpend returns the spending block hash alongside + // the detail (parsed from the same /outspend response that + // confirmed the spend). A zero hash here means the response + // did not parse and reorgSpendReg will fall back to a + // conservative re-check rather than try to match against + // ReorgEvent.Disconnected. + detail, spendingBlockHash := b.checkSingleSpend(reg) if detail == nil { return } + reg.regMu.Lock() + if reg.state == statePositive { + reg.regMu.Unlock() + + return + } + reg.state = statePositive + if detail.SpenderTxHash != nil { + reg.lastSpenderHash = *detail.SpenderTxHash + } + reg.lastSpendingBlockHash = spendingBlockHash + reg.regMu.Unlock() + select { case reg.spendChan <- detail: case <-reg.cancelCh: @@ -594,15 +802,11 @@ func (b *ChainBackend) runSpendOneShot(id uint64, reg *spendRegistration) { } b.log.DebugS(context.Background(), - "Spend registration fulfilled (one-shot)", + "Spend registration fulfilled", slog.Uint64("reg_id", id), slog.String("outpoint", reg.outpoint.String()), slog.String("spender_txid", detail.SpenderTxHash.String())) - - b.mu.Lock() - delete(b.spendRegs, id) - b.mu.Unlock() } // RegisterBlocks registers for new block notifications. @@ -637,17 +841,258 @@ func (b *ChainBackend) RegisterBlocks(_ context.Context) ( }, nil } -// handleTipEvents drains TipBlock events from the shared poller and -// translates them into chain backend work: emit a BlockEpoch to each -// block-registration subscriber, advance the cached tip, and re-check -// pending confirmation/spend registrations. -// -// On stopCh the loop exits and cancels its subscription so the -// poller does not waste effort fanning to a dead consumer. The -// subscription's Quit channel covers the inverse direction: if the -// poller is shut down externally, we exit promptly without waiting -// for stopCh. -func (b *ChainBackend) handleTipEvents(sub *TipSubscription) { +// processReorgEvent reconciles all active registrations with one +// ReorgEvent. It is invoked from the unified handleChainEvents +// goroutine in producer order: a ReorgEvent is fully processed +// (registration state reset, Reorged channels fired) before any +// subsequent TipBlock dispatches block epochs or runs broad +// re-check on the replacement chain. Registrations whose last +// positive event names a disconnected hash are reset and +// re-checked; all others are left alone (the broad tip-driven +// re-check still runs separately). +func (b *ChainBackend) processReorgEvent(event *ReorgEvent) { + if event == nil { + return + } + + b.log.InfoS(context.Background(), "Processing reorg", + slog.Int("fork_height", int(event.ForkHeight)), + slog.Int("disconnected", len(event.Disconnected)), + slog.Int("connected", len(event.Connected)), + ) + + disconnectedSet := make( + map[chainhash.Hash]struct{}, len(event.Disconnected), + ) + for _, hash := range event.Disconnected { + disconnectedSet[hash] = struct{}{} + } + + // Snapshot the registration sets under the chain backend + // lock so we don't hold it across the per-registration + // channel sends below. + b.mu.Lock() + confRegs := make(map[uint64]*confRegistration, len(b.confRegs)) + for id, reg := range b.confRegs { + confRegs[id] = reg + } + spendRegs := make(map[uint64]*spendRegistration, len(b.spendRegs)) + for id, reg := range b.spendRegs { + spendRegs[id] = reg + } + currentHeight := b.bestHeight + b.mu.Unlock() + + for id, reg := range confRegs { + b.reorgConfReg(id, reg, disconnectedSet, currentHeight) + } + for id, reg := range spendRegs { + b.reorgSpendReg(id, reg, disconnectedSet) + } +} + +// reorgConfReg processes one confirmation registration against a +// reorg event. If the registration is in statePositive and its +// last-known block hash is in disconnectedSet, fire Reorged, reset +// to stateWatching, and run a fresh check that may immediately +// fire Confirmed against the new chain. +func (b *ChainBackend) reorgConfReg(id uint64, reg *confRegistration, + disconnectedSet map[chainhash.Hash]struct{}, currentHeight int32) { + + select { + case <-reg.cancelCh: + return + + default: + } + + reg.regMu.Lock() + if reg.state != statePositive { + reg.regMu.Unlock() + + return + } + cachedHash := reg.lastBlockHash + reg.regMu.Unlock() + + // Fast path: the registration's cached block hash appears in + // the reorg event's disconnected set. This covers every + // reorg of a block we previously broadcast. + _, fastHit := disconnectedSet[cachedHash] + + if !fastHit { + // Fallback: the registration may have delivered against + // a block older than the poller's seeded hash history + // (e.g. on a fresh daemon where RegisterConf landed an + // immediate historical positive). The reorg's + // disconnected set is bounded by recentHashes, so a + // reorg deep enough to invalidate that historical block + // would not appear here. Re-query canonical status and + // compare against the cached block hash. + // + // confirmedBlockHash returns a tri-state so we never + // fire a spurious reorg: Some(h) means the tx is still + // confirmed in block h (independent of the numConfs + // threshold), None means it is definitively unconfirmed, + // and a non-nil error means canonical status could not + // be determined right now. + gotHash, err := b.confirmedBlockHash(reg) + switch { + // Transient backend failure: we cannot tell whether this + // is a reorg. Leave the registration in statePositive and + // bail; a later tip or reorg event re-evaluates once the + // backend recovers. Firing Reorged here would strand the + // consumer on a false alarm. + case err != nil: + b.log.DebugS( + context.Background(), + "Skipping conf reorg eval; canonical "+ + "status undeterminable", + slog.Uint64("reg_id", id), + btclog.Fmt("err", "%v", err), + ) + + return + + // Still confirmed in the same block: definitively not a + // reorg of this registration. + case gotHash.IsSome() && + gotHash.UnsafeFromSome() == cachedHash: + return + } + + // Otherwise the tx is confirmed elsewhere or definitively + // unconfirmed: fall through and fire Reorged. + } + + reg.regMu.Lock() + if reg.state != statePositive { + reg.regMu.Unlock() + + return + } + reg.state = stateWatching + reg.lastBlockHash = chainhash.Hash{} + reg.regMu.Unlock() + + // Fire Reorged non-blocking: the channel is buffered to 1 + // and the consumer (chainsource conf actor) is single- + // threaded so a missed coalesced reorg is correct — the + // consumer will re-query state anyway. + select { + case reg.reorgChan <- uint64(0): + case <-reg.cancelCh: + return + + default: + b.log.DebugS( + context.Background(), + "Conf reorged signal coalesced", + slog.Uint64("reg_id", id), + ) + } + + b.log.InfoS( + context.Background(), + "Conf registration reorged; re-checking", + slog.Uint64("reg_id", id), + ) + + // Re-check now so a re-confirmation on the new chain fires + // Confirmed in the same reorg-handler turn. + b.deliverConfIfNew(id, reg, currentHeight) +} + +// reorgSpendReg is the spend-side equivalent of reorgConfReg. +func (b *ChainBackend) reorgSpendReg(id uint64, reg *spendRegistration, + disconnectedSet map[chainhash.Hash]struct{}) { + + select { + case <-reg.cancelCh: + return + + default: + } + + reg.regMu.Lock() + if reg.state != statePositive { + reg.regMu.Unlock() + + return + } + cachedBlockHash := reg.lastSpendingBlockHash + cachedSpenderHash := reg.lastSpenderHash + reg.regMu.Unlock() + + // Fast path: the cached spending-block hash appears in the + // reorg event's disconnected set. Empty cachedBlockHash + // (e.g. delivered before the outspend response was parseable) + // falls through to the fallback re-query rather than being + // treated as an automatic hit. + fastHit := false + if cachedBlockHash != (chainhash.Hash{}) { + _, fastHit = disconnectedSet[cachedBlockHash] + } + + if !fastHit { + // Fallback: re-query canonical chain. Covers (a) regs + // delivered against a block older than the poller's + // seeded hash history, and (b) regs whose + // cachedBlockHash is zero because the outspend response + // was unparseable at delivery time. If the outpoint is + // still spent by the same spender in the same block, + // no reorg for this registration. + current, currentBlock := b.checkSingleSpend(reg) + if current != nil && + current.SpenderTxHash != nil && + *current.SpenderTxHash == cachedSpenderHash && + cachedBlockHash != (chainhash.Hash{}) && + currentBlock == cachedBlockHash { + return + } + } + + reg.regMu.Lock() + if reg.state != statePositive { + reg.regMu.Unlock() + + return + } + reg.state = stateWatching + reg.lastSpenderHash = chainhash.Hash{} + reg.lastSpendingBlockHash = chainhash.Hash{} + reg.regMu.Unlock() + + select { + case reg.reorgChan <- uint64(0): + case <-reg.cancelCh: + return + + default: + b.log.DebugS( + context.Background(), + "Spend reorged signal coalesced", + slog.Uint64("reg_id", id), + ) + } + + b.log.InfoS( + context.Background(), + "Spend registration reorged; re-checking", + slog.Uint64("reg_id", id), + ) + + b.deliverSpendIfNew(id, reg) +} + +// handleChainEvents drains the unified chain stream and dispatches +// each event in producer order. Using a single subscription + single +// goroutine guarantees that a ReorgEvent is fully processed +// (registration state reset, Reorged channels fired) before any +// post-reorg TipBlock drives block-epoch dispatch or broad re-check +// — otherwise a block-epoch driven Confirmed could land on a stale +// registration before reorgConfReg has a chance to reset it. +func (b *ChainBackend) handleChainEvents(sub *ChainSubscription) { defer b.wg.Done() defer sub.Cancel() @@ -661,18 +1106,15 @@ func (b *ChainBackend) handleTipEvents(sub *TipSubscription) { return } - b.processTipEvent(event) + switch { + case event.Reorg != nil: + b.processReorgEvent(event.Reorg) + + case event.Tip != nil: + b.processTipEvent(event.Tip) + } case <-heartbeat.C: - // Re-run the broad checks even when the tip - // hasn't moved. Esplora's status indexer lags - // the block-found event by 1-3 seconds, so a - // processTipEvent that ran before the indexer - // caught up would otherwise not retry until - // the next block lands. Coalesced via the same - // singleflight keys used by processTipEvent so - // a tip event arriving on the same tick does - // not produce two parallel scans. b.runRecheckHeartbeat() case <-sub.Quit(): @@ -774,8 +1216,13 @@ func (b *ChainBackend) processTipEvent(event *TipBlock) { }) } -// checkConfirmations iterates over all pending confirmation registrations -// and checks their status via the Esplora API. +// checkConfirmations iterates over all pending confirmation +// registrations and re-checks their status via the Esplora API. +// Registrations already in statePositive are skipped: the reorg +// handler is responsible for transitioning them back to +// stateWatching, and an unconditional re-check would just burn +// Esplora calls for no behavior change (the chainsource actor's +// FinalityDepth synthesis handles Done from block epochs). func (b *ChainBackend) checkConfirmations() { b.mu.Lock() regs := make(map[uint64]*confRegistration, len(b.confRegs)) @@ -794,28 +1241,7 @@ func (b *ChainBackend) checkConfirmations() { default: } - conf := b.checkSingleConf(reg, currentHeight) - if conf == nil { - continue - } - - // Send the confirmation. - select { - case reg.confChan <- conf: - case <-reg.cancelCh: - continue - } - - b.log.DebugS(context.Background(), - "Confirmation registration fulfilled", - slog.Uint64("reg_id", id), - slog.Int("block_height", - int(conf.BlockHeight))) - - // Remove the fulfilled registration. - b.mu.Lock() - delete(b.confRegs, id) - b.mu.Unlock() + b.deliverConfIfNew(id, reg, currentHeight) } } @@ -948,8 +1374,73 @@ func (b *ChainBackend) checkConfByScript(reg *confRegistration, return nil } -// checkSpends iterates over all pending spend registrations and checks -// their status via the Esplora API. +// confirmedBlockHash reports the block hash a registration's transaction is +// currently confirmed in, independent of the registration's numConfs +// threshold. It exists for the reorg-detection fallback, which must +// distinguish three states that checkSingleConf collapses into a single nil: +// +// - Some(hash), nil: the tx is confirmed in block hash (any depth); +// - None, nil: the tx is definitively unconfirmed (genuine reorg); +// - None, err: canonical status could not be determined right now +// (transient backend error) — callers MUST NOT treat this as a reorg. +// +// Unlike checkSingleConf it never gates on numConfs, because a reorg that +// merely reduces a tx's confirmation depth without moving it out of its block +// is not a reorg of that registration. +func (b *ChainBackend) confirmedBlockHash(reg *confRegistration) ( + fn.Option[chainhash.Hash], error) { + + none := fn.None[chainhash.Hash]() + + // Explicit-txid registrations resolve via a direct status lookup. + if reg.txid != nil { + status, err := b.esplora.GetTxStatus( + context.Background(), *reg.txid, + ) + if err != nil { + return none, err + } + if !status.Confirmed { + return none, nil + } + + hash, err := chainhash.NewHashFromStr(status.BlockHash) + if err != nil { + return none, err + } + + return fn.Some(*hash), nil + } + + // Script registrations resolve via the first confirmed UTXO paying + // the watched script. + utxos, err := b.esplora.GetScriptUtxos( + context.Background(), reg.pkScript, + ) + if err != nil { + return none, err + } + + for _, utxo := range utxos { + if !utxo.Status.Confirmed { + continue + } + + hash, err := chainhash.NewHashFromStr(utxo.Status.BlockHash) + if err != nil { + return none, err + } + + return fn.Some(*hash), nil + } + + return none, nil +} + +// checkSpends iterates over all pending spend registrations and +// re-checks their status via the Esplora API. Registrations in +// statePositive are skipped — see checkConfirmations for the +// rationale. func (b *ChainBackend) checkSpends() { b.mu.Lock() regs := make(map[uint64]*spendRegistration, len(b.spendRegs)) @@ -971,68 +1462,59 @@ func (b *ChainBackend) checkSpends() { continue } - detail := b.checkSingleSpend(reg) - if detail == nil { - continue - } - - // Send the spend detail. - select { - case reg.spendChan <- detail: - case <-reg.cancelCh: - continue - } - - b.log.DebugS(context.Background(), - "Spend registration fulfilled", - slog.Uint64("reg_id", id), - slog.String("outpoint", - reg.outpoint.String()), - slog.String( - "spender_txid", detail.SpenderTxHash.String(), - )) - - // Remove the fulfilled registration. - b.mu.Lock() - delete(b.spendRegs, id) - b.mu.Unlock() + b.deliverSpendIfNew(id, reg) } } // checkSingleSpend resolves the spend status of a single spend // registration via Esplora. Returns nil when the outpoint is not yet // confirmed-spent, when any HTTP / parse error occurs, or when the -// registration has no outpoint. The caller is responsible for +// registration has no outpoint. The second return value is the block +// hash containing the spending tx (parsed from outspend.Status); the +// zero hash indicates the spending block hash was unparseable but the +// spend itself is still valid. The caller is responsible for // delivery, logging, and removing the fulfilled registration; this // helper only resolves the on-chain question. -func (b *ChainBackend) checkSingleSpend( - reg *spendRegistration) *chainsource.SpendDetail { +func (b *ChainBackend) checkSingleSpend(reg *spendRegistration) ( + *chainsource.SpendDetail, chainhash.Hash) { if reg.outpoint == nil { - return nil + return nil, chainhash.Hash{} } outspend, err := b.esplora.GetOutspend( context.Background(), reg.outpoint.Hash, reg.outpoint.Index, ) if err != nil { - return nil + return nil, chainhash.Hash{} } if !outspend.Spent || !outspend.Status.Confirmed { - return nil + return nil, chainhash.Hash{} } spenderHash, err := chainhash.NewHashFromStr(outspend.Txid) if err != nil { - return nil + return nil, chainhash.Hash{} } spendingTx, err := b.esplora.GetRawTx( context.Background(), *spenderHash, ) if err != nil { - return nil + return nil, chainhash.Hash{} + } + + // outspend.Status.BlockHash is hex from the same /outspend + // response that confirmed the spend; an unparseable value is + // not fatal — the spend itself is valid and the reorg path can + // fall back to a conservative re-check (see reorgSpendReg). + var spendingBlockHash chainhash.Hash + if outspend.Status.BlockHash != "" { + h, err := chainhash.NewHashFromStr(outspend.Status.BlockHash) + if err == nil { + spendingBlockHash = *h + } } return &chainsource.SpendDetail{ @@ -1041,7 +1523,7 @@ func (b *ChainBackend) checkSingleSpend( SpendingTx: spendingTx, SpenderInputIndex: outspend.Vin, SpendingHeight: int32(outspend.Status.BlockHeight), - } + }, spendingBlockHash } // Compile-time check that ChainBackend implements diff --git a/lwwallet/chain_backend_reorg_test.go b/lwwallet/chain_backend_reorg_test.go new file mode 100644 index 000000000..593138b38 --- /dev/null +++ b/lwwallet/chain_backend_reorg_test.go @@ -0,0 +1,1136 @@ +package lwwallet + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "runtime" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/btcsuite/btcd/chainhash/v2" + "github.com/btcsuite/btcd/wire/v2" + "github.com/btcsuite/btclog/v2" + "github.com/lightninglabs/darepo-client/chainsource" + "github.com/stretchr/testify/require" +) + +// reorgTestTimeout is the per-step wait used by the lwwallet reorg +// tests. Generous enough to absorb scheduling jitter on overloaded +// CI machines but short enough that a hung backend surfaces as a +// fast failure. +const reorgTestTimeout = 3 * time.Second + +// fakeChain is a mutable chain fixture for the reorg tests. Each +// height maps to a block whose hash, raw header, and contents the +// embedded HTTP handler will serve. Hashes are computable so the +// EsploraClient's content-hash verification passes. +// +// fakeChain is intentionally orthogonal to stubChain (used by the +// existing tip-poller tests): it supports same-height hash +// replacement (reorg), per-tx status overrides, per-outpoint +// outspend overrides, and serves /block//header so the tip +// poller's continuity check can resolve PrevBlock. +type fakeChain struct { + t *testing.T + mu sync.Mutex + + // tip is the current tip height. + tip int32 + + // blocks holds the per-height block model. Replacing the entry + // at height h simulates a same-height reorg; appending new + // heights simulates chain advance. + blocks map[int32]*fakeBlock + + // txStatus is the response to /tx//status. + txStatus map[chainhash.Hash]esploraTxStatus + + // rawTx is the response to /tx//raw. + rawTx map[chainhash.Hash][]byte + + // outspends is the response to /tx//outspend/. + outspends map[wire.OutPoint]esploraOutspend + + // failRawHeader holds block hashes for which the + // /block//header endpoint should return 500. Used to + // simulate a transient Esplora flake during continuity + // checking. + failRawHeader map[chainhash.Hash]struct{} +} + +// fakeBlock describes one height's block in the fakeChain. The hash +// is derived from a synthetic 80-byte header so the EsploraClient's +// content-hash verification accepts the raw-header response. +type fakeBlock struct { + height int32 + hash chainhash.Hash + prevHash chainhash.Hash + header *wire.BlockHeader + timestamp int64 +} + +// newFakeChain seeds a fakeChain with a single block at the given +// tip height. tag is mixed into the block hash so independent +// fakeChains in parallel tests produce distinct hashes. +func newFakeChain(t *testing.T, tip int32, tag string) *fakeChain { + t.Helper() + + c := &fakeChain{ + t: t, + tip: tip, + blocks: make(map[int32]*fakeBlock), + txStatus: make(map[chainhash.Hash]esploraTxStatus), + rawTx: make(map[chainhash.Hash][]byte), + outspends: make(map[wire.OutPoint]esploraOutspend), + failRawHeader: make(map[chainhash.Hash]struct{}), + } + + c.blocks[tip] = c.mintBlock(tip, chainhash.Hash{}, tag) + + return c +} + +// mintBlock builds a fakeBlock at height with the given prev hash +// and a tag that varies the resulting block hash. We synthesize a +// minimal 80-byte header with a unique nonce so BlockHash() varies +// reliably across invocations. +func (c *fakeChain) mintBlock(height int32, prev chainhash.Hash, + tag string) *fakeBlock { + + c.t.Helper() + + hdr := &wire.BlockHeader{ + Version: 1, + PrevBlock: prev, + MerkleRoot: chainhash.HashH([]byte(tag + "-merkle")), + Timestamp: time.Unix(int64(height)*600, 0), + Bits: 0x207fffff, + Nonce: uint32(height) ^ + uint32(chainhash.HashH([]byte(tag)).String()[0])<<16, + } + + // Make the nonce truly unique per (height, tag) so two + // adjacent tags do not collide on PrevBlock=zero replays. + salt := chainhash.HashH([]byte(fmt.Sprintf("%d-%s", height, tag))) + hdr.Nonce = uint32(salt[0])<<24 | uint32(salt[1])<<16 | + uint32(salt[2])<<8 | uint32(salt[3]) + + return &fakeBlock{ + height: height, + hash: hdr.BlockHash(), + prevHash: prev, + header: hdr, + timestamp: hdr.Timestamp.Unix(), + } +} + +// replaceTip swaps in a brand-new block at the current tip height +// (same height, different hash) and returns the new block. Used to +// drive a same-height reorg. +func (c *fakeChain) replaceTip(tag string) *fakeBlock { + c.mu.Lock() + defer c.mu.Unlock() + + prev, ok := c.blocks[c.tip-1] + var prevHash chainhash.Hash + if ok { + prevHash = prev.hash + } + + blk := c.mintBlock(c.tip, prevHash, tag) + c.blocks[c.tip] = blk + + return blk +} + +// extend appends a new block on top of the current tip and returns +// it. tag varies the resulting hash for parallel tests. +func (c *fakeChain) extend(tag string) *fakeBlock { + c.mu.Lock() + defer c.mu.Unlock() + + tipBlk := c.blocks[c.tip] + blk := c.mintBlock(c.tip+1, tipBlk.hash, tag) + c.tip++ + c.blocks[c.tip] = blk + + return blk +} + +// rewriteFrom rebuilds the chain from height start upward, giving +// each new block a different hash than the previous occupant at +// the same height. Used to simulate a deeper reorg where multiple +// heights diverge at once. +func (c *fakeChain) rewriteFrom(start int32, tagPrefix string) { + c.mu.Lock() + defer c.mu.Unlock() + + var prev chainhash.Hash + if prior, ok := c.blocks[start-1]; ok { + prev = prior.hash + } + for h := start; h <= c.tip; h++ { + blk := c.mintBlock(h, prev, + fmt.Sprintf("%s-%d", tagPrefix, h)) + c.blocks[h] = blk + prev = blk.hash + } +} + +// setTxStatus pins the response for /tx//status. +func (c *fakeChain) setTxStatus(txid chainhash.Hash, status esploraTxStatus) { + c.mu.Lock() + defer c.mu.Unlock() + + c.txStatus[txid] = status +} + +// setRawTx pins the response for /tx//raw. +func (c *fakeChain) setRawTx(txid chainhash.Hash, raw []byte) { + c.mu.Lock() + defer c.mu.Unlock() + + c.rawTx[txid] = raw +} + +// setOutspend pins the response for /tx//outspend/. +func (c *fakeChain) setOutspend(op wire.OutPoint, outspend esploraOutspend) { + c.mu.Lock() + defer c.mu.Unlock() + + c.outspends[op] = outspend +} + +// handler returns an http.HandlerFunc that serves the routes the +// chain backend / tip poller exercise. Anything else returns 404. +func (c *fakeChain) handler() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + c.mu.Lock() + defer c.mu.Unlock() + + path := r.URL.Path + switch { + case path == "/blocks/tip/height": + _, _ = fmt.Fprint(w, c.tip) + + case path == "/blocks/tip/hash": + blk, ok := c.blocks[c.tip] + if !ok { + http.Error(w, "no tip", http.StatusNotFound) + + return + } + + _, _ = fmt.Fprint(w, blk.hash.String()) + + case strings.HasPrefix(path, "/block-height/"): + heightStr := strings.TrimPrefix( + path, "/block-height/", + ) + height, err := strconv.ParseInt(heightStr, 10, 32) + if err != nil { + http.Error( + w, "bad height", http.StatusBadRequest, + ) + + return + } + blk, ok := c.blocks[int32(height)] + if !ok { + http.Error(w, "not found", + http.StatusNotFound) + + return + } + + _, _ = fmt.Fprint(w, blk.hash.String()) + + case strings.HasPrefix(path, "/block/"): + c.serveBlockReq(w, r, path) + + case strings.HasPrefix(path, "/tx/"): + c.serveTxReq(w, r, path) + + default: + http.Error(w, "not found", http.StatusNotFound) + } + } +} + +// serveBlockReq handles /block/ and /block//header. +// Caller holds c.mu. +func (c *fakeChain) serveBlockReq(w http.ResponseWriter, _ *http.Request, + path string) { + + rest := strings.TrimPrefix(path, "/block/") + hashStr := rest + suffix := "" + if idx := strings.Index(rest, "/"); idx >= 0 { + hashStr = rest[:idx] + suffix = rest[idx:] + } + + hash, err := chainhash.NewHashFromStr(hashStr) + if err != nil { + http.Error(w, "bad hash", http.StatusBadRequest) + + return + } + + var found *fakeBlock + for _, b := range c.blocks { + if b.hash == *hash { + found = b + + break + } + } + if found == nil { + http.Error(w, "not found", http.StatusNotFound) + + return + } + + switch suffix { + case "": + // JSON header. + _, _ = fmt.Fprintf( + w, `{"id":%q,"height":%d,"timestamp":%d}`, + found.hash.String(), found.height, found.timestamp, + ) + + case "/header": + // Raw 80-byte header, hex-encoded. + if _, fail := c.failRawHeader[found.hash]; fail { + http.Error( + w, "raw header unavailable", + http.StatusInternalServerError, + ) + + return + } + var buf bytes.Buffer + require.NoError(c.t, found.header.Serialize(&buf)) + _, _ = fmt.Fprint(w, hex.EncodeToString(buf.Bytes())) + + default: + http.Error(w, "not implemented", + http.StatusNotImplemented) + } +} + +// serveTxReq handles /tx//status, /tx//raw, and +// /tx//outspend/. Caller holds c.mu. +func (c *fakeChain) serveTxReq(w http.ResponseWriter, _ *http.Request, + path string) { + + rest := strings.TrimPrefix(path, "/tx/") + parts := strings.SplitN(rest, "/", 3) + if len(parts) < 2 { + http.Error(w, "not found", http.StatusNotFound) + + return + } + + txid, err := chainhash.NewHashFromStr(parts[0]) + if err != nil { + http.Error(w, "bad txid", http.StatusBadRequest) + + return + } + + switch parts[1] { + case "status": + status, ok := c.txStatus[*txid] + if !ok { + // Default to unconfirmed when no override exists. + status = esploraTxStatus{Confirmed: false} + } + + err := json.NewEncoder(w).Encode(status) + require.NoError(c.t, err) + + case "raw": + raw, ok := c.rawTx[*txid] + if !ok { + http.Error(w, "not found", + http.StatusNotFound) + + return + } + _, err := w.Write(raw) + require.NoError(c.t, err) + + case "outspend": + if len(parts) < 3 { + http.Error(w, "not found", + http.StatusNotFound) + + return + } + vout, err := strconv.ParseUint(parts[2], 10, 32) + if err != nil { + http.Error(w, "bad vout", + http.StatusBadRequest) + + return + } + op := wire.OutPoint{ + Hash: *txid, Index: uint32(vout), + } + outspend, ok := c.outspends[op] + if !ok { + outspend = esploraOutspend{Spent: false} + } + err = json.NewEncoder(w).Encode(outspend) + require.NoError(c.t, err) + + default: + http.Error(w, "not found", http.StatusNotFound) + } +} + +// fakeChainServer wraps a fakeChain with an httptest.Server for +// drop-in use by the backend tests. The server is auto-closed via +// t.Cleanup. +func fakeChainServer(t *testing.T, chain *fakeChain) *httptest.Server { + t.Helper() + + srv := httptest.NewServer(chain.handler()) + t.Cleanup(srv.Close) + + return srv +} + +// awaitConf reads one TxConfirmation with a deadline. +func awaitConf(t *testing.T, ch <-chan *chainsourceConf) *chainsourceConf { + t.Helper() + + select { + case c, ok := <-ch: + require.True(t, ok, "conf channel closed unexpectedly") + + return c + + case <-time.After(reorgTestTimeout): + t.Fatal("timeout waiting for confirmation") + + return nil + } +} + +// awaitSpend reads one SpendDetail with a deadline. +func awaitSpend(t *testing.T, ch <-chan *chainsourceSpend) *chainsourceSpend { + t.Helper() + + select { + case s, ok := <-ch: + require.True(t, ok, "spend channel closed unexpectedly") + + return s + + case <-time.After(reorgTestTimeout): + t.Fatal("timeout waiting for spend") + + return nil + } +} + +// awaitSeqSignal is awaitSignal for the sequence-carrying Reorged +// channel (chainsource retyped it from struct{} to the ordering +// sequence; this backend always sends 0). +func awaitSeqSignal(t *testing.T, ch <-chan uint64, label string) { + t.Helper() + + select { + case _, ok := <-ch: + require.True(t, ok, + "%s channel closed unexpectedly", label) + + case <-time.After(reorgTestTimeout): + t.Fatalf("timeout waiting for %s", label) + } +} + +// requireQuiet asserts that ch has no event for a short window. +// Used to ensure registrations are NOT double-firing. +func requireQuiet(t *testing.T, ch <-chan struct{}, label string, + dur time.Duration) { + + t.Helper() + + select { + case <-ch: + t.Fatalf("unexpected %s signal", label) + + case <-time.After(dur): + } +} + +// chainsourceConf / chainsourceSpend are type aliases to keep the +// test helper signatures short. +type ( + chainsourceConf = chainsource.TxConfirmation + chainsourceSpend = chainsource.SpendDetail +) + +// TestChainBackendSameHeightHashDrift verifies that a same-height +// hash replacement is detected by the tip poller and routed to the +// chain backend as a reorg, even when the chain height does not +// advance. This is the core "same-height reorgs invisible" gap the +// PR closes. +func TestChainBackendSameHeightHashDrift(t *testing.T) { + t.Parallel() + + chain := newFakeChain(t, 100, "drift-init") + srv := fakeChainServer(t, chain) + + esplora := NewEsploraClient(srv.URL, btclog.Disabled) + backend := NewChainBackend( + esplora, 20*time.Millisecond, btclog.Disabled, + ) + require.NoError(t, backend.Start()) + t.Cleanup(func() { + require.NoError(t, backend.Stop()) + }) + + // Subscribe directly to the poller's reorg stream to verify + // detection. The backend also subscribes; both subscribers + // must observe the reorg. + reorgSub, err := backend.tipPoller.SubscribeReorgs() + require.NoError(t, err) + defer reorgSub.Cancel() + + // Replace the tip block at the SAME height with a new hash. + chain.replaceTip("drift-new") + + select { + case ev := <-reorgSub.Updates(): + require.NotNil(t, ev) + require.Equal(t, int32(99), ev.ForkHeight) + require.Len(t, ev.Disconnected, 1) + require.Len(t, ev.Connected, 1) + require.Equal(t, int32(100), ev.Connected[0].Height) + + case <-time.After(reorgTestTimeout): + t.Fatal("timed out waiting for same-height reorg") + } +} + +// TestChainBackendConfReorgRoundTrip drives a registration through +// Confirmed -> Reorged -> Confirmed by replacing the block that +// confirmed the tx with a new block at the same height that still +// confirms the tx (different BlockHash). +func TestChainBackendConfReorgRoundTrip(t *testing.T) { + t.Parallel() + + chain := newFakeChain(t, 100, "conf-init") + srv := fakeChainServer(t, chain) + + // Use minimalRawTx so the EsploraClient's content-hash + // verification accepts the raw-tx response. + rawTx := minimalRawTx() + txid := minimalRawTxID(t) + chain.setRawTx(txid, rawTx) + + confBlock := chain.blocks[100] + chain.setTxStatus(txid, esploraTxStatus{ + Confirmed: true, + BlockHeight: 100, + BlockHash: confBlock.hash.String(), + }) + + esplora := NewEsploraClient(srv.URL, btclog.Disabled) + backend := NewChainBackend( + esplora, 20*time.Millisecond, btclog.Disabled, + ) + require.NoError(t, backend.Start()) + t.Cleanup(func() { + require.NoError(t, backend.Stop()) + }) + + reg, err := backend.RegisterConf( + t.Context(), &txid, nil, 1, 99, false, + ) + require.NoError(t, err) + defer reg.Cancel() + + conf1 := awaitConf(t, reg.Confirmed) + require.Equal(t, uint32(100), conf1.BlockHeight) + require.Equal(t, confBlock.hash, *conf1.BlockHash) + + // Same-height reorg: replace the confirming block with a + // different one at height 100, and pin the tx status to + // the new block hash so the post-reorg re-check finds the + // tx confirmed in the new block. + newBlock := chain.replaceTip("conf-replaced") + chain.setTxStatus(txid, esploraTxStatus{ + Confirmed: true, + BlockHeight: 100, + BlockHash: newBlock.hash.String(), + }) + + awaitSeqSignal(t, reg.Reorged, "conf Reorged") + + conf2 := awaitConf(t, reg.Confirmed) + require.Equal(t, uint32(100), conf2.BlockHeight) + require.Equal(t, newBlock.hash, *conf2.BlockHash) + require.NotEqual( + t, conf1.BlockHash.String(), conf2.BlockHash.String(), + "reorg should surface a different block hash", + ) +} + +// TestChainBackendConfReorgEvictedDoesNotReConfirm verifies that +// when the post-reorg chain no longer contains the tx, Reorged +// fires but Confirmed does not re-fire. +func TestChainBackendConfReorgEvictedDoesNotReConfirm(t *testing.T) { + t.Parallel() + + chain := newFakeChain(t, 100, "evict-init") + srv := fakeChainServer(t, chain) + + rawTx := minimalRawTx() + txid := minimalRawTxID(t) + chain.setRawTx(txid, rawTx) + + confBlock := chain.blocks[100] + chain.setTxStatus(txid, esploraTxStatus{ + Confirmed: true, + BlockHeight: 100, + BlockHash: confBlock.hash.String(), + }) + + esplora := NewEsploraClient(srv.URL, btclog.Disabled) + backend := NewChainBackend( + esplora, 20*time.Millisecond, btclog.Disabled, + ) + require.NoError(t, backend.Start()) + t.Cleanup(func() { + require.NoError(t, backend.Stop()) + }) + + reg, err := backend.RegisterConf( + t.Context(), &txid, nil, 1, 99, false, + ) + require.NoError(t, err) + defer reg.Cancel() + + awaitConf(t, reg.Confirmed) + + // Reorg evicts the confirming block AND the tx is no longer + // found on the new chain. + chain.replaceTip("evict-new") + chain.setTxStatus(txid, esploraTxStatus{Confirmed: false}) + + awaitSeqSignal(t, reg.Reorged, "conf Reorged") + + // No re-confirmation: assert silence. + requireQuiet( + t, structOnlyChan(reg.Confirmed), + "unexpected re-Confirmed", 500*time.Millisecond, + ) +} + +// structOnlyChan adapts a TxConfirmation channel to a struct{} +// channel for use with requireQuiet. The forwarder goroutine is +// short-lived; once the test ends both channels go out of scope. +func structOnlyChan(in <-chan *chainsourceConf) <-chan struct{} { + out := make(chan struct{}, 4) + + go func() { + for c := range in { + if c == nil { + continue + } + + select { + case out <- struct{}{}: + default: + } + } + }() + + return out +} + +// TestChainBackendSpendReorgRoundTrip drives a spend registration +// through Spend -> Reorged -> Spend by reorging the block that +// confirmed the spending tx and pinning a different (still spent) +// outspend on the new chain. +func TestChainBackendSpendReorgRoundTrip(t *testing.T) { + t.Parallel() + + chain := newFakeChain(t, 100, "spend-init") + srv := fakeChainServer(t, chain) + + rawTx := minimalRawTx() + spenderTxid := minimalRawTxID(t) + chain.setRawTx(spenderTxid, rawTx) + + fundingTxid := chainhash.HashH([]byte("funding")) + outpoint := wire.OutPoint{Hash: fundingTxid, Index: 0} + + chain.setOutspend(outpoint, esploraOutspend{ + Spent: true, Txid: spenderTxid.String(), Vin: 0, + Status: esploraStatus{ + Confirmed: true, BlockHeight: 100, + }, + }) + + esplora := NewEsploraClient(srv.URL, btclog.Disabled) + backend := NewChainBackend( + esplora, 20*time.Millisecond, btclog.Disabled, + ) + require.NoError(t, backend.Start()) + t.Cleanup(func() { + require.NoError(t, backend.Stop()) + }) + + reg, err := backend.RegisterSpend( + t.Context(), &outpoint, nil, 90, + ) + require.NoError(t, err) + defer reg.Cancel() + + spend1 := awaitSpend(t, reg.Spend) + require.Equal(t, spenderTxid, *spend1.SpenderTxHash) + require.Equal(t, int32(100), spend1.SpendingHeight) + + // Replace the confirming block AND swap the spender on the + // new chain. The spender tx id is the same (we're reusing + // minimalRawTx) but the block hash differs, which is what + // the reorg handler keys off. + chain.replaceTip("spend-replaced") + chain.setOutspend(outpoint, esploraOutspend{ + Spent: true, Txid: spenderTxid.String(), Vin: 0, + Status: esploraStatus{ + Confirmed: true, BlockHeight: 100, + }, + }) + + awaitSeqSignal(t, reg.Reorged, "spend Reorged") + + spend2 := awaitSpend(t, reg.Spend) + require.Equal(t, spenderTxid, *spend2.SpenderTxHash) + require.Equal(t, int32(100), spend2.SpendingHeight) +} + +// TestTipPollerSeedsHashHistoryOnStart pins the property that the +// poller seeds its recent-hash ring back through historySize-1 +// heights at Start. Without this, a fresh poller would only ever +// cache the initial tip, leaving any reorg-event consumer with an +// incomplete disconnected set for reorgs deeper than 1 block but +// within the configured history. The chain.Interface consumer +// (EsploraChainService) depends on a complete disconnected set to +// emit a BlockDisconnected for every height btcwallet must roll +// back. +func TestTipPollerSeedsHashHistoryOnStart(t *testing.T) { + t.Parallel() + + // Build a chain at tip 100 plus three lower heights. The poller + // starts at tip 100; with seed-history, recentHashes will be + // pre-populated with {97, 98, 99, 100}. + chain := newFakeChain(t, 100, "seed-100") + // fakeChain only populates the tip block; the test needs lower + // heights in the response. Extend backwards by minting blocks + // at 99, 98, 97 with the correct PrevBlock chain so the + // poller's seed walk resolves each height. + chain.mu.Lock() + var prev chainhash.Hash + for h := int32(97); h <= 100; h++ { + blk := chain.mintBlock(h, prev, fmt.Sprintf("seed-%d", h)) + chain.blocks[h] = blk + prev = blk.hash + } + chain.mu.Unlock() + + srv := fakeChainServer(t, chain) + esplora := NewEsploraClient(srv.URL, btclog.Disabled) + + // Cap historySize at 4 so the seed walk fills exactly the + // heights we care about. + tipPoller := NewTipPollerWithConfig( + esplora, 20*time.Millisecond, 4, btclog.Disabled, + ) + require.NoError(t, tipPoller.Start()) + t.Cleanup(tipPoller.Stop) + + reorgSub, err := tipPoller.SubscribeReorgs() + require.NoError(t, err) + defer reorgSub.Cancel() + + // Reorg the deepest 3 heights (98, 99, 100). With seed-history + // the poller's recentHashes contains 97, 98, 99, 100 so the + // walk-back terminates at fork point 97 and disconnects 3 + // hashes. Without the seed, recentHashes would only contain + // 100; walk-back would stop at 99 (no cached hash) and the + // disconnected list would carry only 1 hash. + chain.rewriteFrom(98, "seed-rewritten") + + select { + case ev := <-reorgSub.Updates(): + require.NotNil(t, ev) + require.Equal( + t, int32(97), ev.ForkHeight, "fork point should be "+ + "the deepest unchanged height; "+ + "seed-history failed if it lands above 97", + ) + require.Len( + t, ev.Disconnected, 3, "all three reorged heights "+ + "must appear in Disconnected; seed-history "+ + "is what makes this property hold for "+ + "reorgs of depth > 1 against a freshly "+ + "started poller", + ) + + case <-time.After(reorgTestTimeout): + t.Fatal("timed out waiting for reorg event") + } +} + +// TestChainBackendDeeperReorgDetected verifies that a multi-block +// reorg (where height advances AND PrevBlock continuity is broken) +// produces a ReorgEvent with the correct fork point. +// +// The test first advances the chain past the eventual fork point so +// the poller's recent-hash ring buffer contains the to-be-rewritten +// heights. Without that, the walk-back would terminate early on the +// first !haveCached probe and report a shallower fork than the test +// intends to exercise. +func TestChainBackendDeeperReorgDetected(t *testing.T) { + t.Parallel() + + chain := newFakeChain(t, 100, "deep-init") + + srv := fakeChainServer(t, chain) + esplora := NewEsploraClient(srv.URL, btclog.Disabled) + tp := NewTipPoller(esplora, 20*time.Millisecond, btclog.Disabled) + require.NoError(t, tp.Start()) + t.Cleanup(tp.Stop) + + // Subscribe BEFORE the rewrite so we receive the reorg + // event live. + reorgSub, err := tp.SubscribeReorgs() + require.NoError(t, err) + defer reorgSub.Cancel() + + tipSub, err := tp.Subscribe() + require.NoError(t, err) + defer tipSub.Cancel() + + // Advance the chain through the poller so it caches + // heights 101 and 102 in its ring buffer. + chain.extend("init-101") + chain.extend("init-102") + + for expected := int32(101); expected <= 102; expected++ { + select { + case ev := <-tipSub.Updates(): + require.Equal(t, expected, ev.Height) + + case <-time.After(reorgTestTimeout): + t.Fatalf("timed out waiting for height %d to be cached", + expected) + } + } + + // Now rewrite from height 101 upward (heights 101 and 102 + // get new hashes) and extend by one to push the new tip + // past the old. + chain.rewriteFrom(101, "fork") + chain.extend("fork-103") + + // Wait for ReorgEvent. ForkHeight should be 100 because + // the rewrite started at 101 (so 100 is the last height + // that agrees between old and new chains). + select { + case ev := <-reorgSub.Updates(): + require.NotNil(t, ev) + require.Equal(t, int32(100), ev.ForkHeight) + require.GreaterOrEqual(t, len(ev.Connected), 2) + require.GreaterOrEqual(t, len(ev.Disconnected), 2) + + case <-time.After(reorgTestTimeout): + t.Fatal("timed out waiting for deeper reorg") + } +} + +// TestChainBackendCancelCleanup verifies that Cancel on a +// conf/spend registration removes it from internal maps and does +// not leak goroutines. +func TestChainBackendCancelCleanup(t *testing.T) { + t.Parallel() + + chain := newFakeChain(t, 100, "cancel-init") + srv := fakeChainServer(t, chain) + + esplora := NewEsploraClient(srv.URL, btclog.Disabled) + backend := NewChainBackend( + esplora, time.Hour, btclog.Disabled, + ) + require.NoError(t, backend.Start()) + t.Cleanup(func() { + require.NoError(t, backend.Stop()) + }) + + txid := minimalRawTxID(t) + outpoint := wire.OutPoint{ + Hash: chainhash.HashH([]byte("funding")), Index: 0, + } + + // Sample the baseline goroutine count AFTER the backend is + // started so the poller / tip handler / reorg handler + // goroutines are already included. + time.Sleep(50 * time.Millisecond) + baseline := runtime.NumGoroutine() + + const iterations = 20 + for i := 0; i < iterations; i++ { + confReg, err := backend.RegisterConf( + t.Context(), &txid, nil, 1, 99, false, + ) + require.NoError(t, err) + + spendReg, err := backend.RegisterSpend( + t.Context(), &outpoint, nil, 90, + ) + require.NoError(t, err) + + confReg.Cancel() + spendReg.Cancel() + + // Double Cancel must be a safe no-op. + confReg.Cancel() + spendReg.Cancel() + } + + // Give the spawned one-shot goroutines time to exit. + require.Eventually(t, func() bool { + + // Allow a small tolerance for scheduling jitter; the + // upper bound here is generous (20) versus the + // per-iteration spawn count (2) to absorb any + // in-flight Esplora request goroutines that have not + // yet returned. The key invariant is that the count + // does not GROW unbounded. + return runtime.NumGoroutine() <= baseline+5 + }, 2*time.Second, 50*time.Millisecond, + "goroutines leaked after Cancel: baseline=%d current=%d", + baseline, runtime.NumGoroutine()) + + // After Cancel, internal maps should be empty. + backend.mu.Lock() + confLen := len(backend.confRegs) + spendLen := len(backend.spendRegs) + backend.mu.Unlock() + require.Equal(t, 0, confLen, + "confRegs not cleaned up after Cancel") + require.Equal(t, 0, spendLen, + "spendRegs not cleaned up after Cancel") +} + +// TestChainBackendConfStaysAliveAfterFirstFire verifies that the +// confirmation registration is NOT deleted after the first +// Confirmed delivery, since the chainsource ConfActor needs the +// registration to remain alive to receive future Reorged events +// and finally a synthesized Done. +func TestChainBackendConfStaysAliveAfterFirstFire(t *testing.T) { + t.Parallel() + + chain := newFakeChain(t, 100, "alive-init") + srv := fakeChainServer(t, chain) + + rawTx := minimalRawTx() + txid := minimalRawTxID(t) + chain.setRawTx(txid, rawTx) + chain.setTxStatus(txid, esploraTxStatus{ + Confirmed: true, + BlockHeight: 100, + BlockHash: chain.blocks[100].hash.String(), + }) + + esplora := NewEsploraClient(srv.URL, btclog.Disabled) + backend := NewChainBackend( + esplora, 50*time.Millisecond, btclog.Disabled, + ) + require.NoError(t, backend.Start()) + t.Cleanup(func() { + require.NoError(t, backend.Stop()) + }) + + reg, err := backend.RegisterConf( + t.Context(), &txid, nil, 1, 99, false, + ) + require.NoError(t, err) + defer reg.Cancel() + + awaitConf(t, reg.Confirmed) + + // Give the polling loop time to fire several heartbeats. + // The registration must still be present. + time.Sleep(200 * time.Millisecond) + + backend.mu.Lock() + confLen := len(backend.confRegs) + backend.mu.Unlock() + + require.Equal( + t, 1, confLen, "reg deleted after first fire; reorg-aware "+ + "contract requires it to stay alive", + ) +} + +// TestChainBackendConfReorgBelowSeededHistory pins the property that a +// reorg deeper than the poller's seeded hash history still surfaces +// Reorged for a registration that delivered against the +// now-out-of-window block. The poller's disconnected set is bounded +// by the cached history, so without a canonical re-query fallback the +// reorg signal would be silently dropped. +func TestChainBackendConfReorgBelowSeededHistory(t *testing.T) { + t.Parallel() + + // Seed a chain at height 100 and extend up to 105 so the + // walk-back has live block hashes to compare against. Use a + // tiny history size so the registration's conf block falls + // outside the seeded window from the poller's perspective. + chain := newFakeChain(t, 100, "below-100") + chain.extend("below-101") + chain.extend("below-102") + chain.extend("below-103") + chain.extend("below-104") + chain.extend("below-105") + + confBlock := chain.blocks[100] + srv := fakeChainServer(t, chain) + + rawTx := minimalRawTx() + txid := minimalRawTxID(t) + chain.setRawTx(txid, rawTx) + chain.setTxStatus(txid, esploraTxStatus{ + Confirmed: true, + BlockHeight: 100, + BlockHash: confBlock.hash.String(), + }) + + esplora := NewEsploraClient(srv.URL, btclog.Disabled) + // historySize=3 means the poller only retains the last three + // canonical heights it has observed. The conf block at height + // 100 will be five heights below the seeded tip (105) and is + // guaranteed never to enter recentHashes. + tipPoller := NewTipPollerWithConfig( + esplora, 20*time.Millisecond, 3, btclog.Disabled, + ) + require.NoError(t, tipPoller.Start()) + t.Cleanup(tipPoller.Stop) + backend, err := NewChainBackendWithPoller( + esplora, tipPoller, btclog.Disabled, + ) + require.NoError(t, err) + require.NoError(t, backend.Start()) + t.Cleanup(func() { + require.NoError(t, backend.Stop()) + }) + + reg, err := backend.RegisterConf( + t.Context(), &txid, nil, 1, 99, false, + ) + require.NoError(t, err) + defer reg.Cancel() + + conf1 := awaitConf(t, reg.Confirmed) + require.Equal(t, confBlock.hash, *conf1.BlockHash) + + // Deep reorg: rewrite every height from 100 upward with + // different hashes. The new block at height 100 has a new + // hash, but the tx is still pinned there so the canonical + // re-query in reorgConfReg can fire a fresh Confirmed. + chain.rewriteFrom(100, "below-new") + newConfBlock := chain.blocks[100] + chain.setTxStatus(txid, esploraTxStatus{ + Confirmed: true, + BlockHeight: 100, + BlockHash: newConfBlock.hash.String(), + }) + + awaitSeqSignal(t, reg.Reorged, "conf Reorged on below-history reorg") + + conf2 := awaitConf(t, reg.Confirmed) + require.Equal(t, newConfBlock.hash, *conf2.BlockHash) + require.NotEqual( + t, conf1.BlockHash.String(), conf2.BlockHash.String(), + "re-confirmation must surface the new block hash", + ) +} + +// TestChainBackendAbortsOnRawHeaderFailure pins the property that a +// failed raw-header fetch during continuity check aborts the poll +// cycle rather than optimistically advancing. The optimistic path +// would permanently hide a reorg that crossed the old-tip boundary +// if the raw-header fetch flaked at exactly the wrong moment. +func TestChainBackendAbortsOnRawHeaderFailure(t *testing.T) { + t.Parallel() + + chain := newFakeChain(t, 100, "abort-init") + srv := fakeChainServer(t, chain) + + esplora := NewEsploraClient(srv.URL, btclog.Disabled) + tipPoller := NewTipPoller( + esplora, 20*time.Millisecond, btclog.Disabled, + ) + require.NoError(t, tipPoller.Start()) + t.Cleanup(tipPoller.Stop) + + tipSub, err := tipPoller.Subscribe() + require.NoError(t, err) + defer tipSub.Cancel() + reorgSub, err := tipPoller.SubscribeReorgs() + require.NoError(t, err) + defer reorgSub.Cancel() + + // Add a height-101 block whose PrevBlock does NOT chain off + // the seeded tip at height 100. Then poison the raw-header + // endpoint for that block so the continuity check cannot + // resolve. The poller must abort the cycle: not broadcast + // the new block, not update its cached tip, not fire a reorg. + chain.mu.Lock() + stranger := chain.mintBlock( + 101, chainhash.Hash{0xde, 0xad}, "abort-stranger", + ) + chain.blocks[101] = stranger + chain.tip = 101 + chain.failRawHeader[stranger.hash] = struct{}{} + chain.mu.Unlock() + + // Sleep long enough for a few poll cycles to fire; verify + // neither tip advance nor reorg ever fires. + select { + case ev := <-tipSub.Updates(): + t.Fatalf("poller broadcast a tip event despite raw-header "+ + "failure: height=%d hash=%s", ev.Height, ev.Hash) + + case ev := <-reorgSub.Updates(): + t.Fatalf("poller broadcast a reorg event despite raw-header "+ + "failure: fork=%d", ev.ForkHeight) + + case <-time.After(200 * time.Millisecond): + // Expected: silent. + } + + height, hash, _ := tipPoller.BestBlock() + require.Equal( + t, int32(100), height, + "cached tip height advanced despite aborted cycle", + ) + require.Equal( + t, chain.blocks[100].hash, hash, + "cached tip hash advanced despite aborted cycle", + ) +} diff --git a/lwwallet/chain_backend_test.go b/lwwallet/chain_backend_test.go index bcea6be99..6ee39c5ad 100644 --- a/lwwallet/chain_backend_test.go +++ b/lwwallet/chain_backend_test.go @@ -2,6 +2,7 @@ package lwwallet import ( "bytes" + "encoding/hex" "encoding/json" "fmt" "net/http" @@ -80,15 +81,20 @@ func TestChainBackendBlockNotification(t *testing.T) { mu sync.Mutex tipHeight int32 = 100 blockHashes = make(map[int32]chainhash.Hash) + blockHdrs = make(map[int32]*wire.BlockHeader) ) - // Pre-generate block hashes. + // Pre-generate real chained block headers so the tip poller's + // PrevBlock continuity check during advance can resolve the raw + // header endpoint with a header that actually links back to the + // previous height. + var prevHash chainhash.Hash for h := int32(100); h <= 103; h++ { - blockHashes[h] = chainhash.HashH( - []byte( - fmt.Sprintf("block-%d", h), - ), - ) + hdr := mintStubHeader(h, prevHash) + blockHdrs[h] = hdr + hash := hdr.BlockHash() + blockHashes[h] = hash + prevHash = hash } srv := mockEsploraServer( @@ -109,7 +115,7 @@ func TestChainBackendBlockNotification(t *testing.T) { default: handleBlockReqs( - t, w, r, &mu, blockHashes, + t, w, r, &mu, blockHashes, blockHdrs, ) } }, @@ -147,10 +153,13 @@ func TestChainBackendBlockNotification(t *testing.T) { } } -// handleBlockReqs handles /block-height/:height and /block/:hash -// requests in the mock Esplora server. +// handleBlockReqs handles /block-height/:height, /block/:hash, and +// /block/:hash/header requests in the mock Esplora server. The +// blockHdrs map supplies real wire.BlockHeaders so the raw-header +// continuity check the tip poller runs on advance can succeed. func handleBlockReqs(t *testing.T, w http.ResponseWriter, r *http.Request, - mu *sync.Mutex, blockHashes map[int32]chainhash.Hash) { + mu *sync.Mutex, blockHashes map[int32]chainhash.Hash, + blockHdrs map[int32]*wire.BlockHeader) { t.Helper() @@ -176,20 +185,40 @@ func handleBlockReqs(t *testing.T, w http.ResponseWriter, r *http.Request, return } - // Handle /block/:hash (block header). - for _, h := range blockHashes { + // Handle /block/:hash and /block/:hash/header. + mu.Lock() + defer mu.Unlock() + for height, h := range blockHashes { hashStr := h.String() path := "/block/" + hashStr - if r.URL.Path == path { + headerPath := path + "/header" + + switch r.URL.Path { + case path: resp := esploraBlock{ ID: hashStr, - Height: 100, + Height: height, Timestamp: 1700000000, } - err := json.NewEncoder(w).Encode(resp) require.NoError(t, err) + return + + case headerPath: + hdr := blockHdrs[height] + if hdr == nil { + http.Error(w, "no header", + http.StatusNotFound) + + return + } + var buf bytes.Buffer + require.NoError(t, hdr.Serialize(&buf)) + _, _ = fmt.Fprint( + w, hex.EncodeToString(buf.Bytes()), + ) + return } } diff --git a/lwwallet/esplora_chain.go b/lwwallet/esplora_chain.go index bd37534ee..2d1c944fc 100644 --- a/lwwallet/esplora_chain.go +++ b/lwwallet/esplora_chain.go @@ -110,12 +110,15 @@ func NewEsploraChainService(esplora *EsploraClient, tipPoller *TipPoller, // Start seeds the initial chain tip from the configured TipPoller // (which the caller must have started already) and spawns the -// goroutine that translates each TipBlock event into btcwallet -// chain notifications. +// goroutine that translates each chain event into btcwallet chain +// notifications. The service subscribes to the unified chain stream +// so reorg and tip updates arrive on a single producer-ordered +// channel; this is what lets chain.BlockDisconnected reliably precede +// the replacement chain.BlockConnected events for the new tip. func (s *EsploraChainService) Start(ctx context.Context) error { //nolint:contextcheck // tip subscription lifecycle is owned by Stop - tipHeight, tipHash, tipTime, sub, err := - s.tipPoller.BestBlockAndSubscribe() + tipHeight, tipHash, tipTime, chainSub, err := + s.tipPoller.BestBlockAndSubscribeChain() if err != nil { return fmt.Errorf("subscribe to tip poller: %w", err) } @@ -134,7 +137,7 @@ func (s *EsploraChainService) Start(ctx context.Context) error { s.notifications <- chain.ClientConnected{} s.wg.Add(1) - go s.handleTipEvents(ctx, sub) + go s.handleChainEvents(ctx, chainSub) s.log.InfoS(ctx, "Esplora chain service started", slog.Int("tip_height", int(tipHeight)), @@ -685,26 +688,35 @@ func (s *EsploraChainService) MapRPCErr(err error) error { return err } -// handleTipEvents drains TipBlock events from the shared poller and -// translates each event into the FilteredBlockConnected + -// BlockConnected notification pair that btcwallet's wallet syncer -// expects. The loop exits when the chain service is stopped, when -// the poller signals shutdown via Quit, or when the subscription's -// Updates channel is closed by Cancel. -func (s *EsploraChainService) handleTipEvents(ctx context.Context, - sub *TipSubscription) { +// handleChainEvents drains the unified chain stream and translates +// each event into btcwallet chain notifications. Using a single +// subscription delivers ReorgEvent and TipBlock updates in producer +// order through one channel, which is what guarantees +// BlockDisconnected lands on btcwallet's notification queue before +// the BlockConnected events for the replacement chain (btcwallet's +// disconnectBlock would otherwise refuse the rollback if the cached +// hash at that height had already been overwritten by a stale +// BlockConnected). +func (s *EsploraChainService) handleChainEvents(ctx context.Context, + sub *ChainSubscription) { defer s.wg.Done() defer sub.Cancel() for { select { - case event, ok := <-sub.Updates(): + case ev, ok := <-sub.Updates(): if !ok { return } - s.processTipEvent(ctx, event) + switch { + case ev.Reorg != nil: + s.processReorgEvent(ctx, ev.Reorg) + + case ev.Tip != nil: + s.processTipEvent(ctx, ev.Tip) + } case <-sub.Quit(): return @@ -729,6 +741,63 @@ func (s *EsploraChainService) handleTipEvents(ctx context.Context, // per-event cap branch without producing 256+ heights of traffic. const defaultMaxGapFillPerTipEvent int32 = 256 +// processReorgEvent translates a ReorgEvent into chain.BlockDisconnected +// notifications. The replacement Connected blocks arrive separately on +// the tip stream and are processed by processTipEvent in the usual way; +// emitting Disconnected here is what lets btcwallet roll back the +// reorged-out heights via its disconnectBlock path before it re-sees +// the canonical chain. Newest height is disconnected first so btcwallet +// walks back from its own cached tip. +// +// After emitting the disconnects, s.bestBlock is rolled back to +// event.ForkHeight so the subsequent replacement-chain TipBlock events +// (which arrive at heights forkHeight+1..newTipHeight) pass +// processTipEvent's "event.Height <= lastDelivered" duplicate guard. +// Without this rollback, the replacement BlockConnected events would +// silently be dropped because s.bestBlock still points at the old tip. +// The Hash is intentionally cleared because the reorg event does not +// carry the fork-point hash; the next BlockConnected fully repopulates +// s.bestBlock and only the Height is load-bearing for the duplicate +// guard. +func (s *EsploraChainService) processReorgEvent(ctx context.Context, + event *ReorgEvent) { + + if event == nil { + return + } + + for i := len(event.Disconnected) - 1; i >= 0; i-- { + hash := event.Disconnected[i] + height := event.ForkHeight + int32(i) + 1 + + meta := wtxmgr.BlockMeta{ + Block: wtxmgr.Block{ + Hash: hash, + Height: height, + }, + } + + select { + case s.notifications <- chain.BlockDisconnected(meta): + case <-s.quit: + return + } + } + + s.mu.Lock() + if s.bestBlock.Height > event.ForkHeight { + s.bestBlock = waddrmgr.BlockStamp{ + Height: event.ForkHeight, + } + } + s.mu.Unlock() + + s.log.InfoS(ctx, "Chain service emitted BlockDisconnected", + slog.Int("fork_height", int(event.ForkHeight)), + slog.Int("disconnected", len(event.Disconnected)), + ) +} + // processTipEvent applies one TipBlock to btcwallet's notification // channel. The chain service owns its own delivery cursor // (s.bestBlock); this function first walks any gap between diff --git a/lwwallet/esplora_chain_reorg_test.go b/lwwallet/esplora_chain_reorg_test.go new file mode 100644 index 000000000..a4b22c1c1 --- /dev/null +++ b/lwwallet/esplora_chain_reorg_test.go @@ -0,0 +1,175 @@ +package lwwallet + +import ( + "testing" + "time" + + "github.com/btcsuite/btcd/chainhash/v2" + "github.com/btcsuite/btclog/v2" + "github.com/btcsuite/btcwallet/chain" + "github.com/btcsuite/btcwallet/wtxmgr" + "github.com/stretchr/testify/require" +) + +// awaitNotification pulls the next notification from the chain +// service or fails the test on timeout. +func awaitNotification(t *testing.T, s *EsploraChainService) interface{} { + t.Helper() + + select { + case n := <-s.Notifications(): + return n + + case <-time.After(reorgTestTimeout): + t.Fatalf("timed out waiting for chain notification") + + return nil + } +} + +// drainUntilConnected drains notifications until a BlockConnected +// event at the given height is observed (or the test times out). +// Used to skip over the startup ClientConnected and initial connected +// events so the reorg-specific assertions can run on a known cursor. +func drainUntilConnected(t *testing.T, s *EsploraChainService, height int32) { + t.Helper() + + deadline := time.After(reorgTestTimeout) + + for { + select { + case n := <-s.Notifications(): + conn, ok := n.(chain.BlockConnected) + if !ok { + continue + } + if conn.Block.Height == height { + return + } + + case <-deadline: + t.Fatalf("timed out waiting for BlockConnected at %d", + height) + } + } +} + +// TestEsploraChainServiceReorgEmitsBlockDisconnected pins the property +// that a chain reorg surfaces chain.BlockDisconnected notifications +// before btcwallet sees the BlockConnected events that announce the +// new canonical chain, AND that the disconnect events arrive in +// newest-height-first order (the order btcwallet's disconnectBlock +// expects when walking back from its cached tip). Both ordering +// properties are load-bearing: without disconnect-before-connect, +// btcwallet would refuse the rollback because the cached hash at +// each height would already be overwritten; without +// newest-first, btcwallet's per-step rollback could trip on a +// height it hasn't seen disconnected yet. +func TestEsploraChainServiceReorgEmitsBlockDisconnected(t *testing.T) { + t.Parallel() + + chainModel := newFakeChain(t, 100, "svc-reorg-100") + chainModel.extend("svc-reorg-101") + chainModel.extend("svc-reorg-102") + + srv := fakeChainServer(t, chainModel) + + esplora := NewEsploraClient(srv.URL, btclog.Disabled) + tipPoller := NewTipPoller( + esplora, 20*time.Millisecond, btclog.Disabled, + ) + require.NoError(t, tipPoller.Start()) + t.Cleanup(tipPoller.Stop) + + service := NewEsploraChainService( + esplora, tipPoller, btclog.Disabled, + ) + require.NoError(t, service.Start(t.Context())) + t.Cleanup(func() { + service.Stop() + service.WaitForShutdown() + }) + + // First notification is always ClientConnected. + first := awaitNotification(t, service) + _, ok := first.(chain.ClientConnected) + require.True(t, ok, "expected ClientConnected first, got %T", first) + + // Drain forward to a known tip + a couple of new blocks so the + // service has a known set of recently-emitted connected blocks + // before the reorg fires. + chainModel.extend("svc-reorg-103") + drainUntilConnected(t, service, 103) + + // Stash hashes of the blocks the reorg is about to invalidate. + oldHash102 := chainModel.blocks[102].hash + oldHash103 := chainModel.blocks[103].hash + + // Rewrite heights 102..103 with a new chain. The tip stays at + // 103 but with a different hash, so the poller detects a + // same-height drift at 103 and walks back to fork point 101. + chainModel.rewriteFrom(102, "svc-reorg-new") + newHash102 := chainModel.blocks[102].hash + newHash103 := chainModel.blocks[103].hash + + // Drain notifications and pin the strict ordering. Each + // BlockConnected for a new-chain hash must arrive AFTER both + // BlockDisconnected events for the old chain — the unified + // chain stream guarantees this by serializing reorg + tip + // events through a single producer-ordered channel. + disconnectsSeen := 0 + gotDisconnected := []chainhash.Hash{} + gotConnectedNew := map[chainhash.Hash]struct{}{} + deadline := time.After(reorgTestTimeout) + for len(gotConnectedNew) < 2 { + select { + case n := <-service.Notifications(): + switch ev := n.(type) { + case chain.BlockDisconnected: + gotDisconnected = append( + gotDisconnected, ev.Block.Hash, + ) + disconnectsSeen++ + + case chain.BlockConnected: + meta := wtxmgr.BlockMeta(ev) + if meta.Hash != newHash102 && + meta.Hash != newHash103 { + + continue + } + + require.Equal( + t, 2, disconnectsSeen, "BlockConnect"+ + "ed for replacement chain "+ + "arrived before all "+ + "BlockDisconnected events: "+ + "saw %d disconnects so far", + disconnectsSeen, + ) + gotConnectedNew[meta.Hash] = struct{}{} + } + + case <-deadline: + t.Fatalf("timed out: disconnects=%v connected_new=%v", + gotDisconnected, gotConnectedNew) + } + } + + require.Equal( + t, oldHash103, gotDisconnected[0], "first "+ + "BlockDisconnected should be the newest old-chain "+ + "height (103) for btcwallet's per-step walk-back", + ) + require.Equal( + t, oldHash102, gotDisconnected[1], + "second BlockDisconnected should be height 102", + ) + + _, sawConn102 := gotConnectedNew[newHash102] + _, sawConn103 := gotConnectedNew[newHash103] + require.True(t, sawConn102, + "BlockConnected for new 102 not emitted") + require.True(t, sawConn103, + "BlockConnected for new 103 not emitted") +} diff --git a/lwwallet/esplora_chain_test.go b/lwwallet/esplora_chain_test.go index ee43b022f..151363cc0 100644 --- a/lwwallet/esplora_chain_test.go +++ b/lwwallet/esplora_chain_test.go @@ -2,6 +2,7 @@ package lwwallet import ( "bytes" + "encoding/hex" "encoding/json" "fmt" "net/http" @@ -299,6 +300,16 @@ func (c *rawBlockStubChain) serveBlockRoute(t *testing.T, w http.ResponseWriter, require.NoError(t, block.Serialize(&buf)) _, _ = w.Write(buf.Bytes()) + case "/header": + // Hex-encoded 80-byte block header. The TipPoller fetches + // this per new tip to verify PrevBlock continuity against + // the cached tip hash; without it the poller cannot + // distinguish a clean tip advance from a reorg crossing the + // boundary and aborts the cycle. + var buf bytes.Buffer + require.NoError(t, block.Header.Serialize(&buf)) + _, _ = fmt.Fprint(w, hex.EncodeToString(buf.Bytes())) + default: http.Error(w, "not implemented", http.StatusNotImplemented) diff --git a/lwwallet/tip_poller.go b/lwwallet/tip_poller.go index dbce6c97d..f2fd3557f 100644 --- a/lwwallet/tip_poller.go +++ b/lwwallet/tip_poller.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log/slog" + "sort" "sync" "time" @@ -11,6 +12,13 @@ import ( "github.com/btcsuite/btclog/v2" ) +// DefaultHashHistorySize is the default upper bound on entries retained +// in the TipPoller's bounded height -> hash history map. It is sized to +// at least twice the conventional Bitcoin reorg-safety depth (6) so a +// reorg at finality depth still has its disconnected hashes available +// for walk-back, with headroom. Configurable via NewTipPollerWithConfig. +const DefaultHashHistorySize = 100 + // TipBlock describes a newly detected block emitted by TipPoller. // Each subscriber receives one TipBlock per advance: when the tip // moves from oldHeight to newHeight the poller fans out (newHeight - @@ -38,6 +46,62 @@ type TipBlock struct { // for ergonomic call-site naming. type TipSubscription = Subscription[*TipBlock] +// ReorgEvent describes a chain reorganization observed by the +// TipPoller. The poller emits one ReorgEvent every time it detects +// that one or more previously broadcast blocks are no longer in the +// canonical chain. Disconnected hashes are listed in canonical +// (low-height first) order; Connected blocks are the new tip's path +// from ForkHeight+1 onward, also in canonical order. Either slice may +// be empty: a same-height hash-replacement reorg has Disconnected of +// length 1 and Connected of length 1. +type ReorgEvent struct { + // ForkHeight is the highest height at which the old chain and the + // new chain agree on the block hash. The first disconnected / + // connected block is at ForkHeight+1. + ForkHeight int32 + + // Disconnected lists block hashes that were previously broadcast + // as part of the canonical chain and are no longer on it, in + // ascending height order (ForkHeight+1 first). + Disconnected []chainhash.Hash + + // Connected lists blocks now on the canonical chain starting at + // ForkHeight+1, in ascending height order. The poller will also + // fan these out individually as TipBlock events on the standard + // Subscribe channel after the ReorgEvent is delivered. + Connected []*TipBlock +} + +// ReorgSubscription is the typed handle returned by +// TipPoller.SubscribeReorgs. +type ReorgSubscription = Subscription[*ReorgEvent] + +// ChainEvent is the unified update delivered on the ordered chain +// stream returned by SubscribeChain. Exactly one of Reorg and Tip is +// non-nil per event: +// +// - Reorg-only events announce a reorg's disconnected range and +// precede the replacement tip events on the same stream. +// - Tip-only events announce a new block (either a forward advance +// or a post-reorg connected block). +// +// Cross-event ordering on this stream is producer-ordered: the +// embedded EventServer delivers updates in SendUpdate order to a +// single subscriber goroutine, so a downstream consumer that +// dispatches on event type from one channel observes the same order +// the TipPoller emitted. That is the load-bearing property +// downstream needs to emit BlockDisconnected before BlockConnected +// for the replacement chain (btcwallet's disconnectBlock requires +// it; chainsource finality synthesis requires it too). +type ChainEvent struct { + Reorg *ReorgEvent + Tip *TipBlock +} + +// ChainSubscription is the typed handle returned by +// TipPoller.SubscribeChain. It carries the unified ChainEvent stream. +type ChainSubscription = Subscription[*ChainEvent] + // TipPoller is the single source of truth for the lwwallet chain // tip. Exactly one polling goroutine periodically asks the Esplora // backend for the current best height. When new blocks are detected @@ -58,18 +122,46 @@ type TipPoller struct { pollInterval time.Duration log btclog.Logger + // historySize caps the bounded height -> hash history map that + // lets the poller resolve old-chain hashes during reorg walk-back. + historySize int + // events is the typed event server that fans TipBlock updates // out to all active subscribers. Its Start/Stop are driven by // TipPoller.Start/Stop. events *EventServer[*TipBlock] + // reorgs is the typed event server that fans ReorgEvent updates + // out to subscribers that opt in via SubscribeReorgs. Reorgs are + // rare enough that a separate server is cheaper than re-fitting + // every TipBlock consumer with reorg-aware logic. + reorgs *EventServer[*ReorgEvent] + + // chain is the unified event server that fans both tip and reorg + // updates out on a single ordered stream. Consumers that need + // strict reorg-before-replacement-tip ordering (chain.Interface + // adapter, chainsource backend) subscribe here rather than to + // the separate events / reorgs servers, which would race across + // two independent translator goroutines. + chain *EventServer[*ChainEvent] + // mu guards the cached tip so BestBlock readers see a - // consistent height/hash/timestamp triple. + // consistent height/hash/timestamp triple. It also guards + // recentHashes, which is sized small enough that linear scans + // under the lock are cheap. mu sync.Mutex tipHeight int32 tipHash chainhash.Hash tipTime time.Time + // recentHashes maps a recent canonical-chain height to the hash + // the poller broadcast for it. It is bounded to historySize + // entries: on every insert we prune any entry whose height is + // more than historySize below the current tip. The buffer lets + // reorg detection walk back to the fork point using the cached + // hashes rather than re-fetching the entire pre-fork chain. + recentHashes map[int32]chainhash.Hash + // started gates re-entrant Start calls; the underlying // EventServer is also idempotent on Start. started bool @@ -82,15 +174,35 @@ type TipPoller struct { // NewTipPoller constructs a TipPoller bound to the given Esplora // client. The poll interval controls how often the goroutine asks // Esplora for the latest tip height; subscribers do not influence -// the cadence. +// the cadence. The bounded hash-history map is sized to +// DefaultHashHistorySize. func NewTipPoller(esplora *EsploraClient, pollInterval time.Duration, log btclog.Logger) *TipPoller { + return NewTipPollerWithConfig( + esplora, pollInterval, DefaultHashHistorySize, log, + ) +} + +// NewTipPollerWithConfig is the explicit-history-size constructor. +// historySize <= 0 falls back to DefaultHashHistorySize so callers +// cannot accidentally disable reorg walk-back by passing a zero value. +func NewTipPollerWithConfig(esplora *EsploraClient, pollInterval time.Duration, + historySize int, log btclog.Logger) *TipPoller { + + if historySize <= 0 { + historySize = DefaultHashHistorySize + } + return &TipPoller{ esplora: esplora, pollInterval: pollInterval, + historySize: historySize, log: log, events: NewEventServer[*TipBlock](log), + reorgs: NewEventServer[*ReorgEvent](log), + chain: NewEventServer[*ChainEvent](log), + recentHashes: make(map[int32]chainhash.Hash), quit: make(chan struct{}), } } @@ -170,12 +282,60 @@ func (t *TipPoller) Start() error { return fmt.Errorf("start event server: %w", err) } + if err := t.reorgs.Start(); err != nil { + // Roll back the tip-event server so a partial Start does + // not leave a half-running poller behind. + _ = t.events.Stop() + resetStarted() + + return fmt.Errorf("start reorg event server: %w", err) + } + + if err := t.chain.Start(); err != nil { + _ = t.events.Stop() + _ = t.reorgs.Stop() + resetStarted() + + return fmt.Errorf("start chain event server: %w", err) + } + + // Seed recentHashes by walking back historySize-1 heights so a + // reorg whose disconnected range extends below the seeded tip + // but within the configured history can still resolve every + // disconnected hash from the cache. Without this, a fresh + // poller would only ever cache the initial tip, leaving a + // downstream chain.Interface consumer unable to enumerate every + // hash btcwallet must roll back on a multi-block reorg below + // the seeded tip. The walk-back is best-effort: a single + // per-height fetch failure ends the seed loop early; the next + // poll tick still drives the cache forward as the chain grows. t.mu.Lock() t.tipHeight = height t.tipHash = hash t.tipTime = tipTime + t.recordHashLocked(height, hash) t.mu.Unlock() + for h := height - 1; h > height-int32(t.historySize) && h >= 0; h-- { + liveHash, err := t.esplora.GetBlockHashByHeight( + context.Background(), h, + ) + if err != nil { + t.log.WarnS( + context.Background(), + "Tip poller history seed fetch failed", + err, + slog.Int("height", int(h)), + ) + + break + } + + t.mu.Lock() + t.recordHashLocked(h, liveHash) + t.mu.Unlock() + } + t.wg.Add(1) go t.pollLoop() @@ -197,7 +357,7 @@ func (t *TipPoller) Stop() { t.wg.Wait() - // Stop the event server after the poll loop has exited so + // Stop the event servers after the poll loop has exited so // that no SendUpdate is in flight when the server tears down // its subscriber handler. if err := t.events.Stop(); err != nil { @@ -207,6 +367,22 @@ func (t *TipPoller) Stop() { err, ) } + + if err := t.reorgs.Stop(); err != nil { + t.log.WarnS( + context.Background(), + "Tip poller reorg event server stop returned error", + err, + ) + } + + if err := t.chain.Stop(); err != nil { + t.log.WarnS( + context.Background(), + "Tip poller chain event server stop returned error", + err, + ) + } } // BestBlock returns a snapshot of the currently cached tip. Callers @@ -233,6 +409,18 @@ func (t *TipPoller) Subscribe() (*TipSubscription, error) { return t.events.Subscribe() } +// SubscribeReorgs returns a typed subscription that receives a +// ReorgEvent every time the poller detects that one or more +// previously broadcast blocks have been replaced on the canonical +// chain. Callers that need both tip events and reorg events should +// subscribe to both streams; reorg events are delivered BEFORE the +// connected blocks are fanned out on the TipBlock stream so that +// consumers can mark their registrations dirty before the +// re-confirmation arrives. +func (t *TipPoller) SubscribeReorgs() (*ReorgSubscription, error) { + return t.reorgs.Subscribe() +} + // BestBlockAndSubscribe atomically reads the current cached tip and // registers a new subscription. The poll goroutine holds t.mu // during the {update tip + SendUpdate} pair, and this function @@ -257,6 +445,85 @@ func (t *TipPoller) BestBlockAndSubscribe() (int32, chainhash.Hash, time.Time, return t.tipHeight, t.tipHash, t.tipTime, sub, nil } +// BestBlockAndSubscribeAll atomically reads the cached tip and +// registers both a TipBlock subscription and a ReorgEvent +// subscription. Reorg-aware consumers (e.g. ChainBackend) use this +// to avoid the small window where a reorg could land between +// independently registering on the two streams. +func (t *TipPoller) BestBlockAndSubscribeAll() (int32, chainhash.Hash, + time.Time, *TipSubscription, *ReorgSubscription, error) { + + t.mu.Lock() + defer t.mu.Unlock() + + sub, err := t.events.Subscribe() + if err != nil { + return 0, chainhash.Hash{}, time.Time{}, nil, nil, + fmt.Errorf("subscribe to tip events: %w", err) + } + + reorgSub, err := t.reorgs.Subscribe() + if err != nil { + sub.Cancel() + + return 0, chainhash.Hash{}, time.Time{}, nil, nil, + fmt.Errorf("subscribe to reorg events: %w", err) + } + + return t.tipHeight, t.tipHash, t.tipTime, sub, reorgSub, nil +} + +// SubscribeChain returns a typed subscription on the unified chain +// event stream. Updates arrive in producer order through a single +// channel; consumers that need strict reorg-before-replacement-tip +// ordering must subscribe here rather than to the separate tip / +// reorg streams (which race across two independent translator +// goroutines). +func (t *TipPoller) SubscribeChain() (*ChainSubscription, error) { + return t.chain.Subscribe() +} + +// BestBlockAndSubscribeChain atomically reads the cached tip and +// registers a ChainSubscription. Same atomicity guarantee as +// BestBlockAndSubscribe, applied to the unified stream. +func (t *TipPoller) BestBlockAndSubscribeChain() (int32, chainhash.Hash, + time.Time, *ChainSubscription, error) { + + t.mu.Lock() + defer t.mu.Unlock() + + sub, err := t.chain.Subscribe() + if err != nil { + return 0, chainhash.Hash{}, time.Time{}, nil, + fmt.Errorf("subscribe to chain events: %w", err) + } + + return t.tipHeight, t.tipHash, t.tipTime, sub, nil +} + +// recordHashLocked inserts a (height, hash) pair into the recent- +// hash history map and prunes any entry whose height has fallen out +// of the historySize window relative to the new entry. Caller must +// hold t.mu. +func (t *TipPoller) recordHashLocked(height int32, hash chainhash.Hash) { + t.recentHashes[height] = hash + + cutoff := height - int32(t.historySize) + for h := range t.recentHashes { + if h <= cutoff { + delete(t.recentHashes, h) + } + } +} + +// hashAtHeightLocked returns the cached hash for a height plus +// whether it was present. Caller must hold t.mu. +func (t *TipPoller) hashAtHeightLocked(height int32) (chainhash.Hash, bool) { + h, ok := t.recentHashes[height] + + return h, ok +} + // pollLoop is the single tip-polling goroutine. It ticks at // pollInterval, asks Esplora for the latest tip, and walks the // gap from the cached tip to the new tip emitting one TipBlock per @@ -278,12 +545,26 @@ func (t *TipPoller) pollLoop() { } } -// poll performs one tip-detection cycle. On detected progress it -// fetches the hash and header for each new height, broadcasts a -// TipBlock to every subscriber, and advances the cached tip -// monotonically. A failure to fetch any single block aborts the -// remainder of the cycle so subscribers never see an out-of-order -// event; the next tick re-attempts from the same starting point. +// poll performs one tip-detection cycle. The cycle is reorg-aware: +// +// 1. Query (tip height, tip hash). +// 2. If the tip is at the same height as the cached tip but the hash +// differs, walk back through cached hashes to find the fork point +// and emit a ReorgEvent + a re-broadcast TipBlock at the same +// height. This is the "same-height reorg" case that earlier +// versions of the poller could not detect. +// 3. If the new height is strictly greater than the cached height, +// fetch each intervening hash; if the first new block's PrevBlock +// header field does not point at the cached tip hash, walk back +// to find the fork point, emit a ReorgEvent for the disconnected +// range, and then fan TipBlock events for the new chain in +// ascending order. If the new chain extends the old tip cleanly +// (the common case), no ReorgEvent fires and TipBlock events are +// dispatched as before. +// +// A failure to fetch any single block aborts the remainder of the +// cycle so subscribers never see an out-of-order event; the next +// tick re-attempts from the same starting point. func (t *TipPoller) poll() { newHeight, err := t.esplora.GetTipHeight(context.Background()) if err != nil { @@ -298,23 +579,123 @@ func (t *TipPoller) poll() { t.mu.Lock() oldHeight := t.tipHeight + oldHash := t.tipHash t.mu.Unlock() - // Known limitation: a same-height reorg (block at height N - // replaced by a different block at height N) is invisible to - // this loop until the chain advances to N+1, because we gate - // progress on height alone rather than (height, hash). This - // matches the behavior of the per-component pollers that - // preceded the unified TipPoller and has historically been - // acceptable for lwwallet's confirmation-target use case - // (downstream callers re-check status against Esplora on every - // tip event, so a stale hash at height N converges within one - // extra tip advance). Documented here so it is not filed as a - // regression by a future reader. - if newHeight <= oldHeight { + switch { + case newHeight < oldHeight: + // The remote reports fewer blocks than we have cached. + // Two distinct cases hide behind a lower height: + // + // 1. A transient indexer hiccup where the remote + // momentarily lags our cached tip but is still on + // the same chain (our tip remains canonical). + // 2. A genuine reorg onto a SHORTER but higher-work + // chain: blocks above newHeight were orphaned and + // the surviving chain has fewer total blocks. Bitcoin + // follows most-work, not most-blocks, so a shorter + // chain can legitimately win. + // + // Disambiguate by comparing the live hash at newHeight to + // our cached hash for that height. Agreement means the + // remote is merely behind (case 1, no-op); divergence + // means the chain reorged out from under us (case 2) and + // we must walk back to the fork point. Without this an + // orphaning reorg to a shorter chain would be silently + // ignored until the new chain grew past our stale tip. + newTipHash, err := t.esplora.GetBlockHashByHeight( + context.Background(), newHeight, + ) + if err != nil { + t.log.WarnS( + context.Background(), + "Tip poller shorter-chain hash check failed", + err, + slog.Int("height", int(newHeight)), + ) + + return + } + + t.mu.Lock() + cachedHash, haveCached := t.hashAtHeightLocked(newHeight) + t.mu.Unlock() + + switch { + // newHeight predates our retained history floor, so we + // cannot prove a reorg by hash comparison. A reorg this + // deep exceeds historySize and is an operator-level + // problem rather than a productionally recoverable one + // (mirrors handleReorg's own deep-reorg guard); log and + // re-check next tick rather than walk a pruned history. + case !haveCached: + t.log.WarnS( + context.Background(), + "Tip poller remote tip below retained "+ + "history floor; ignoring", + fmt.Errorf("history floor exceeded"), + slog.Int("new_height", int(newHeight)), + slog.Int("old_height", int(oldHeight)), + ) + + return + + // Same hash at newHeight: the remote is merely lagging + // our cached tip on the same chain. No-op. + case newTipHash == cachedHash: + return + } + + // Divergent hash at newHeight: a reorg onto a shorter, + // higher-work chain. handleReorg builds Disconnected up + // to our old cached tip (covering the orphaned blocks + // above newHeight) and Connected up to newHeight. + t.handleReorg(newHeight, newTipHash) + + return + + case newHeight == oldHeight: + // Same-height: query the hash at this height (which + // the BlockHashByHeight endpoint resolves directly). + // If it differs from the cached hash at the same + // height we have a same-height reorg; if it matches + // the chain has not moved. + newTipHash, err := t.esplora.GetBlockHashByHeight( + context.Background(), newHeight, + ) + if err != nil { + t.log.WarnS( + context.Background(), + "Tip poller same-height hash check failed", + err, + slog.Int("height", int(newHeight)), + ) + + return + } + + if newTipHash == oldHash { + return + } + + t.handleReorg(newHeight, newTipHash) + return + + default: + // newHeight > oldHeight: forward advance, possibly with + // a deeper reorg if the first new height's predecessor + // is not the cached tip hash. + t.advance(oldHeight, newHeight) } +} +// advance walks the chain forward from oldHeight+1 to newHeight, +// detecting reorgs along the way. The very first new height's raw +// header is consulted to verify its PrevBlock matches the cached +// tip hash; mismatch triggers a walk-back to the fork point and a +// reorg emission before any new TipBlock events fire. +func (t *TipPoller) advance(oldHeight, newHeight int32) { t.log.DebugS(context.Background(), "Tip poller advancing", slog.Int("old_height", int(oldHeight)), slog.Int("new_height", int(newHeight)), @@ -349,44 +730,397 @@ func (t *TipPoller) poll() { return } + // On the FIRST new height, verify continuity against + // the cached tip. We use the raw 80-byte header (which + // carries PrevBlock) because the JSON header endpoint + // does not include the previous-block hash. If + // PrevBlock does not match the cached tip hash a reorg + // crossed the boundary; walk back to the fork point + // and emit a ReorgEvent before continuing. + if height == oldHeight+1 { + rawHdr, err := t.esplora.GetRawBlockHeader( + context.Background(), hash, + ) + if err != nil { + // Without the raw header we cannot verify + // that the new block connects to the cached + // tip. Abort the cycle without broadcasting + // or updating the cached tip; the next tick + // will retry. Optimistically advancing here + // would permanently hide a reorg that + // crossed the oldHeight boundary if the + // raw-header fetch flaked exactly when the + // reorg happened. + t.log.WarnS( + context.Background(), + "Tip poller raw header fetch "+ + "failed; aborting cycle", + err, + slog.String("hash", hash.String()), + ) + + return + } + + t.mu.Lock() + cachedOldHash, ok := t.hashAtHeightLocked( + oldHeight, + ) + t.mu.Unlock() + + if ok && rawHdr.PrevBlock != cachedOldHash { + // Deeper reorg: handle it as a new-tip + // reorg from oldHeight, pointing at the + // actual new tip at newHeight. handleReorg + // emits ReorgEvent + TipBlock events + // itself. + tipHash, tipErr := t.esplora. + GetBlockHashByHeight( + context.Background(), newHeight, + ) + if tipErr != nil { + t.log.WarnS( + context.Background(), + "Tip poller "+ + "reorg tip "+ + "fetch failed", + tipErr, + slog.Int( + "height", + int(newHeight), + ), + ) + + return + } + + t.handleReorg(newHeight, tipHash) + + return + } + } + event := &TipBlock{ Height: height, Hash: hash, Header: header, } - // Hold t.mu across the {update tip + SendUpdate} - // pair so BestBlockAndSubscribe can serialize against - // it: a subscriber that acquires t.mu before us reads - // the OLD tip and is guaranteed to receive THIS event - // once it Subscribes (subscribe.Server's handler is - // single-threaded over Subscribe and SendUpdate, so - // our SendUpdate enqueues behind their Subscribe). A - // subscriber that acquires t.mu after us reads the - // NEW tip and will see only events strictly newer - // than this one. Without holding t.mu here a tip - // reader+subscriber pair has a small window where it - // can read the new tip but miss this event entirely. + if !t.broadcastTipBlock(event) { + return + } + } +} + +// broadcastTipBlock updates the cached tip, records the hash in the +// history map, and fans the TipBlock out to subscribers. Returns +// false if the send failed (server shutting down), telling the +// caller to abort the cycle so the cached tip is not advanced past +// an event subscribers did not receive. +func (t *TipPoller) broadcastTipBlock(event *TipBlock) bool { + // Hold t.mu across the {update tip + SendUpdate} pair so + // BestBlockAndSubscribe can serialize against it: a subscriber + // that acquires t.mu before us reads the OLD tip and is + // guaranteed to receive THIS event once it Subscribes + // (subscribe.Server's handler is single-threaded over + // Subscribe and SendUpdate, so our SendUpdate enqueues behind + // their Subscribe). A subscriber that acquires t.mu after us + // reads the NEW tip and will see only events strictly newer + // than this one. + t.mu.Lock() + t.tipHeight = event.Height + t.tipHash = event.Hash + if event.Header != nil { + t.tipTime = time.Unix(event.Header.Timestamp, 0) + } + t.recordHashLocked(event.Height, event.Hash) + tipErr := t.events.SendUpdate(event) + chainErr := t.chain.SendUpdate(&ChainEvent{Tip: event}) + t.mu.Unlock() + + if tipErr != nil { + t.log.WarnS( + context.Background(), + "Tip poller send update failed", + tipErr, + slog.Int("height", int(event.Height)), + ) + + return false + } + + if chainErr != nil { + t.log.WarnS( + context.Background(), + "Tip poller chain stream send failed", + chainErr, + slog.Int("height", int(event.Height)), + ) + + return false + } + + return true +} + +// handleReorg walks back through the recent-hash history map until it +// finds the fork point between the cached chain and the new chain +// whose tip is (newTipHeight, newTipHash). It then constructs a +// ReorgEvent listing the disconnected hashes and connected blocks, +// broadcasts the reorg, and finally fans out the connected blocks +// as TipBlock events in ascending height order so consumers can +// re-check registrations against each new block. +func (t *TipPoller) handleReorg(newTipHeight int32, newTipHash chainhash.Hash) { + t.log.InfoS(context.Background(), "Tip poller detected reorg", + slog.Int("new_tip_height", int(newTipHeight)), + slog.String("new_tip_hash", newTipHash.String()), + ) + + // Walk back to find the fork point. We probe heights from + // newTipHeight downwards, comparing the live Esplora hash at + // each height to the cached hash. The first height at which + // they agree is the fork point. We bound the search by the + // retained history depth so a misbehaving Esplora cannot drag + // us into an unbounded loop. + t.mu.Lock() + cachedTipHeight := t.tipHeight + cutoff := cachedTipHeight - int32(t.historySize) + t.mu.Unlock() + + if cutoff < 0 { + cutoff = 0 + } + + // connectedByHeight collects new-chain blocks discovered + // during walk-back so we can re-broadcast them in ascending + // order after the fork point is found. + connectedByHeight := make(map[int32]chainhash.Hash) + connectedByHeight[newTipHeight] = newTipHash + + forkHeight := int32(-1) + probeHeight := newTipHeight + for probeHeight > cutoff { + probeHeight-- + + // Heights strictly above our cached tip cannot be a fork + // point: we never broadcast a block there, so the live + // block is a NEW connected block on a longer chain (a + // forward reorg that both reorganized old blocks AND + // extended past our tip). Record it and keep walking + // down toward the real fork. Without this, the + // !haveCached branch below would mistake the first such + // height for the fork point and compute a forkHeight + // above the cached tip, underflowing the Disconnected + // slice capacity. + if probeHeight > cachedTipHeight { + liveHash, err := t.esplora.GetBlockHashByHeight( + context.Background(), probeHeight, + ) + if err != nil { + t.log.WarnS( + context.Background(), + "Tip poller reorg walk-back failed", + err, + slog.Int("height", int(probeHeight)), + ) + + return + } + + connectedByHeight[probeHeight] = liveHash + + continue + } + t.mu.Lock() - t.tipHeight = height - t.tipHash = hash - t.tipTime = time.Unix(header.Timestamp, 0) - sendErr := t.events.SendUpdate(event) + cachedHash, haveCached := t.hashAtHeightLocked(probeHeight) t.mu.Unlock() - // SendUpdate failures only happen when the embedded - // subscribe.Server is shutting down; log and exit so - // we do not advance the cached tip past an event we - // failed to fan out. - if sendErr != nil { + if !haveCached { + // We never broadcast a block at this height + // (typically because it's older than our + // retained history's start, e.g. on a fresh + // poller that only ever cached the initial + // tip). Treat probeHeight as the fork point: + // anything at or below it is not part of our + // old broadcast set, so it cannot be + // "disconnected" from a downstream consumer's + // point of view. The live hash at this height + // is not a new connected block we owe + // subscribers either; only blocks strictly + // above probeHeight that we previously + // broadcast (and that have now changed) form + // the reorg boundary. + forkHeight = probeHeight + + break + } + + liveHash, err := t.esplora.GetBlockHashByHeight( + context.Background(), probeHeight, + ) + if err != nil { t.log.WarnS( context.Background(), - "Tip poller send update failed", - sendErr, - slog.Int("height", int(height)), + "Tip poller reorg walk-back failed", + err, + slog.Int("height", int(probeHeight)), + ) + + return + } + + if liveHash == cachedHash { + // Genuine fork point: the new chain agrees with + // our cached canonical chain at this height. + forkHeight = probeHeight + + break + } + + // haveCached && liveHash != cachedHash: this height is + // part of the reorg. Record the live hash so we can + // rebroadcast it after the fork point is found. + connectedByHeight[probeHeight] = liveHash + } + + if forkHeight < 0 { + // We exhausted the retained history without finding a + // fork point. Be conservative: log and bail. A future + // tick will re-attempt with the now-pruned history; in + // practice a reorg deeper than DefaultHashHistorySize + // is a developer / operator problem, not a + // productionally recoverable condition. + t.log.WarnS( + context.Background(), + "Tip poller reorg deeper than retained history; "+ + "giving up walk-back", + fmt.Errorf("history exhausted"), + slog.Int("new_tip_height", int(newTipHeight)), + slog.Int("history_size", t.historySize), + ) + + return + } + + // Build the Disconnected slice from cached hashes between + // forkHeight+1 and the cached tipHeight in one critical + // section. The walk is already in ascending order so no sort + // is needed; we cap at the history size to bound the worst + // case under t.mu. + t.mu.Lock() + cachedTipHeight = t.tipHeight + disconnected := make( + []chainhash.Hash, 0, int(cachedTipHeight-forkHeight), + ) + for h := forkHeight + 1; h <= cachedTipHeight; h++ { + if hash, ok := t.hashAtHeightLocked(h); ok { + disconnected = append(disconnected, hash) + } + } + t.mu.Unlock() + + // Build the Connected slice. Heights range from forkHeight+1 + // to newTipHeight; for each, fetch the header so the + // TipBlock carries a populated Header field for downstream + // consumers. + connectedHeights := make([]int32, 0, len(connectedByHeight)) + for h := range connectedByHeight { + if h <= forkHeight { + continue + } + connectedHeights = append(connectedHeights, h) + } + sort.Slice(connectedHeights, func(i, j int) bool { + return connectedHeights[i] < connectedHeights[j] + }) + + connected := make([]*TipBlock, 0, len(connectedHeights)) + for _, h := range connectedHeights { + hash := connectedByHeight[h] + header, err := t.esplora.GetBlockHeader( + context.Background(), hash, + ) + if err != nil { + t.log.WarnS( + context.Background(), + "Tip poller reorg header fetch failed", + err, + slog.String("hash", hash.String()), ) return } + + connected = append(connected, &TipBlock{ + Height: h, + Hash: hash, + Header: header, + }) + } + + // Prune the now-stale hashes from the history before we + // broadcast the reorg so a subscriber that immediately calls + // back into BestBlock sees the new tip. + t.mu.Lock() + for h := forkHeight + 1; h <= cachedTipHeight; h++ { + delete(t.recentHashes, h) + } + t.mu.Unlock() + + reorgEvent := &ReorgEvent{ + ForkHeight: forkHeight, + Disconnected: disconnected, + Connected: connected, + } + + t.log.InfoS(context.Background(), "Tip poller emitting reorg", + slog.Int("fork_height", int(forkHeight)), + slog.Int("disconnected", len(disconnected)), + slog.Int("connected", len(connected)), + ) + + if err := t.reorgs.SendUpdate(reorgEvent); err != nil { + t.log.WarnS( + context.Background(), + "Tip poller reorg send failed", + err, + ) + + return + } + + // Also emit the reorg on the unified chain stream BEFORE any + // connected-block tip events land on it, so a single consumer + // reading the chain subscription sees Reorged before any of + // the replacement Connected blocks. This is the load-bearing + // ordering property the chain.Interface adapter needs to emit + // BlockDisconnected before BlockConnected, and the chainsource + // backend needs to reset registrations before block-epoch + // driven re-checks run on the replacement chain. + if err := t.chain.SendUpdate( + &ChainEvent{Reorg: reorgEvent}, + ); err != nil { + + t.log.WarnS( + context.Background(), + "Tip poller chain reorg send failed", + err, + ) + + return + } + + // Finally, fan the connected blocks out on the standard + // TipBlock stream so existing consumers re-check on each + // new block exactly as they would on a non-reorg advance. + // broadcastTipBlock also pushes each block onto the unified + // chain stream so the single-channel consumer observes the + // connected blocks immediately after the reorg event in the + // same producer-ordered sequence. + for _, block := range connected { + if !t.broadcastTipBlock(block) { + return + } } } diff --git a/lwwallet/tip_poller_reorg_test.go b/lwwallet/tip_poller_reorg_test.go new file mode 100644 index 000000000..0b9dd2cbc --- /dev/null +++ b/lwwallet/tip_poller_reorg_test.go @@ -0,0 +1,274 @@ +package lwwallet + +import ( + "fmt" + "testing" + "time" + + "github.com/btcsuite/btcd/chainhash/v2" + "github.com/btcsuite/btcd/wire/v2" + "github.com/btcsuite/btclog/v2" + "github.com/stretchr/testify/require" +) + +// mintStubHeaderGen is like mintStubHeader but mixes a fork generation +// into the salt so re-minting the same height on a forked chain yields +// a distinct BlockHash. This lets a test rebuild a height range on a +// different chain whose hashes provably diverge from the original. +func mintStubHeaderGen(height, gen int32, + prev chainhash.Hash) *wire.BlockHeader { + + salt := chainhash.HashH( + fmt.Appendf(nil, "stub-block-%d-gen-%d", height, gen), + ) + + return &wire.BlockHeader{ + Version: 1, + PrevBlock: prev, + MerkleRoot: chainhash.HashH( + fmt.Appendf(nil, "merkle-%d-gen-%d", height, gen), + ), + Timestamp: time.Unix(int64(height)*600, 0), + Bits: 0x207fffff, + Nonce: uint32(salt[0])<<24 | uint32(salt[1])<<16 | + uint32(salt[2])<<8 | uint32(salt[3]), + } +} + +// reorgTo rewrites the chain onto a fork that branches at forkHeight and +// extends to newTip. newTip may be lower than the current tip (a shorter +// but higher-work chain), equal to it (a same-height hash replacement), +// or higher (a deeper forward reorg). Heights above forkHeight are +// replaced with generation-salted blocks whose hashes diverge from the +// old chain; any old heights above newTip are orphaned (removed) so the +// stubbed backend no longer serves them. +func (c *stubChain) reorgTo(t *testing.T, forkHeight, newTip, gen int32) { + t.Helper() + + c.mu.Lock() + defer c.mu.Unlock() + + // Drop every height above the fork point; the new chain rebuilds + // forkHeight+1 .. newTip and orphans anything beyond newTip. + for h := range c.hashAt { + if h > forkHeight { + delete(c.hashAt, h) + delete(c.blocks, h) + } + } + + prev := c.hashAt[forkHeight] + for h := forkHeight + 1; h <= newTip; h++ { + hdr := mintStubHeaderGen(h, gen, prev) + c.blocks[h] = hdr + hash := hdr.BlockHash() + c.hashAt[h] = hash + prev = hash + } + + c.tipHeight = newTip +} + +// setTipHeight lowers (or raises) the reported tip height WITHOUT +// touching any cached hashes. It models a transient indexer hiccup +// where the remote momentarily reports fewer blocks than it served a +// moment ago, but is still on the same chain. +func (c *stubChain) setTipHeight(h int32) { + c.mu.Lock() + defer c.mu.Unlock() + + c.tipHeight = h +} + +// requireTipEventually blocks until the poller's BestBlock height +// reaches want, failing the test if it does not within the deadline. +func requireTipEventually(t *testing.T, tp *TipPoller, want int32) { + t.Helper() + + require.Eventually(t, func() bool { + h, _, _ := tp.BestBlock() + + return h == want + }, 2*time.Second, 5*time.Millisecond, + "poller never reached tip height %d", want) +} + +// newReorgTestPoller spins up a TipPoller over a stubChain seeded at the +// given height and returns both so a test can drive reorgs. +func newReorgTestPoller(t *testing.T, + seedHeight int32) (*TipPoller, *stubChain) { + + t.Helper() + + chain := newStubChain(seedHeight) + srv := mockEsploraServer(t, stubEsploraHandler(t, chain)) + + tp := NewTipPoller( + NewEsploraClient(srv.URL, btclog.Disabled), 10*time.Millisecond, + btclog.Disabled, + ) + require.NoError(t, tp.Start()) + t.Cleanup(tp.Stop) + + return tp, chain +} + +// TestTipPollerShorterChainReorg covers the case Roasbeef flagged: a +// reorg onto a SHORTER but higher-work chain, where blocks above the new +// tip are orphaned. The poller must detect this even though the remote +// reports a lower height than the cached tip, and roll the tip back to +// the shorter chain's tip. +func TestTipPollerShorterChainReorg(t *testing.T) { + t.Parallel() + + tp, chain := newReorgTestPoller(t, 100) + + reorgSub, err := tp.SubscribeReorgs() + require.NoError(t, err) + defer reorgSub.Cancel() + + // Grow to 110 so the poller caches 101..110, then confirm it + // observed the advance before we reorg out from under it. + chain.advance(t, 10) + requireTipEventually(t, tp, 110) + + // Fork at 105 onto a shorter chain that tips at 108 (< 110). + chain.reorgTo(t, 105, 108, 1) + + select { + case ev := <-reorgSub.Updates(): + require.NotNil(t, ev) + require.Equal(t, int32(105), ev.ForkHeight) + + // Old chain 106..110 disconnects (5 blocks), even though + // the new chain only carries 106..108. + require.Len(t, ev.Disconnected, 5) + + // New chain connects 106..108 (3 blocks) in ascending + // order, tipping at 108. + require.Len(t, ev.Connected, 3) + require.Equal( + t, int32(106), ev.Connected[0].Height, + ) + require.Equal( + t, int32(108), ev.Connected[len(ev.Connected)-1].Height, + ) + + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for shorter-chain reorg event") + } + + // The poller's tip must now be the shorter chain's tip. + requireTipEventually(t, tp, 108) +} + +// TestTipPollerShorterHeightSameChainNoReorg verifies the no-op half of +// the shorter-height branch: a transient indexer hiccup where the remote +// reports fewer blocks but is still on the same chain (the hash at the +// reported height matches our cache). No reorg must fire and the cached +// tip must NOT roll back. +func TestTipPollerShorterHeightSameChainNoReorg(t *testing.T) { + t.Parallel() + + tp, chain := newReorgTestPoller(t, 100) + + reorgSub, err := tp.SubscribeReorgs() + require.NoError(t, err) + defer reorgSub.Cancel() + + chain.advance(t, 10) + requireTipEventually(t, tp, 110) + + // Remote briefly reports height 108 with the SAME hash history + // (no fork). This is lag, not a reorg. + chain.setTipHeight(108) + + select { + case ev := <-reorgSub.Updates(): + t.Fatalf("unexpected reorg on transient lag: %+v", ev) + + case <-time.After(300 * time.Millisecond): + // No reorg fired: correct. + } + + // The tip must remain at 110: we never roll back on a lagging + // remote that is still on our chain. + h, _, _ := tp.BestBlock() + require.Equal(t, int32(110), h) +} + +// TestTipPollerSameHeightReorg covers a same-height hash replacement: the +// tip height does not change but the block at the tip is replaced by a +// different block on a competing chain. +func TestTipPollerSameHeightReorg(t *testing.T) { + t.Parallel() + + tp, chain := newReorgTestPoller(t, 100) + + reorgSub, err := tp.SubscribeReorgs() + require.NoError(t, err) + defer reorgSub.Cancel() + + chain.advance(t, 10) + requireTipEventually(t, tp, 110) + + oldHash := chain.hashAt[110] + + // Replace block 110 with a different block at the same height. + chain.reorgTo(t, 109, 110, 1) + + select { + case ev := <-reorgSub.Updates(): + require.NotNil(t, ev) + require.Equal(t, int32(109), ev.ForkHeight) + require.Len(t, ev.Disconnected, 1) + require.Equal(t, oldHash, ev.Disconnected[0]) + require.Len(t, ev.Connected, 1) + require.Equal(t, int32(110), ev.Connected[0].Height) + + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for same-height reorg event") + } + + requireTipEventually(t, tp, 110) +} + +// TestTipPollerDeeperForwardReorg covers a reorg that also extends the +// chain: the new chain forks below the old tip yet ends higher than it. +// The forward-advance path must notice the broken PrevBlock continuity +// at the old tip boundary and emit a reorg before the new tip events. +func TestTipPollerDeeperForwardReorg(t *testing.T) { + t.Parallel() + + tp, chain := newReorgTestPoller(t, 100) + + reorgSub, err := tp.SubscribeReorgs() + require.NoError(t, err) + defer reorgSub.Cancel() + + chain.advance(t, 10) + requireTipEventually(t, tp, 110) + + // Fork at 107 onto a longer chain that tips at 113. + chain.reorgTo(t, 107, 113, 1) + + select { + case ev := <-reorgSub.Updates(): + require.NotNil(t, ev) + require.Equal(t, int32(107), ev.ForkHeight) + + // Old chain 108..110 disconnects (3 blocks). + require.Len(t, ev.Disconnected, 3) + + // New chain connects 108..113 (6 blocks). + require.Len(t, ev.Connected, 6) + require.Equal( + t, int32(113), ev.Connected[len(ev.Connected)-1].Height, + ) + + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for deeper forward reorg event") + } + + requireTipEventually(t, tp, 113) +} diff --git a/lwwallet/tip_poller_test.go b/lwwallet/tip_poller_test.go index e4aed4a9c..cf75fa16b 100644 --- a/lwwallet/tip_poller_test.go +++ b/lwwallet/tip_poller_test.go @@ -1,6 +1,8 @@ package lwwallet import ( + "bytes" + "encoding/hex" "fmt" "net/http" "sync" @@ -9,6 +11,7 @@ import ( "time" "github.com/btcsuite/btcd/chainhash/v2" + "github.com/btcsuite/btcd/wire/v2" "github.com/btcsuite/btclog/v2" "github.com/stretchr/testify/require" ) @@ -16,28 +19,59 @@ import ( // stubChain is a tiny test fixture that simulates an Esplora chain // where the tip height can be advanced under test control. It is // independent of the larger mockEsploraServer helper so tip-poller -// tests can drive the chain forward synchronously. +// tests can drive the chain forward synchronously. Each height holds +// a real wire.BlockHeader whose PrevBlock chains back to the previous +// height; the poller's PrevBlock continuity check on advance therefore +// passes naturally rather than aborting. type stubChain struct { mu sync.Mutex tipHeight int32 - // hashAt[h] is the hash for height h. We pre-populate as the - // tip advances so GetBlockHashByHeight resolves consistently. + // blocks[h] holds the wire.BlockHeader for height h. Pre- + // populated through 0..tipHeight at construction and grown + // monotonically by advance. + blocks map[int32]*wire.BlockHeader + + // hashAt[h] is the chainhash.Hash for height h. hashAt map[int32]chainhash.Hash } +// mintStubHeader builds a deterministic wire.BlockHeader for a given +// height, chaining off the supplied previous hash. The salted nonce +// makes the resulting BlockHash stable per (height, prev) and unique +// across heights. +func mintStubHeader(height int32, prev chainhash.Hash) *wire.BlockHeader { + salt := chainhash.HashH([]byte(fmt.Sprintf("stub-block-%d", height))) + + return &wire.BlockHeader{ + Version: 1, + PrevBlock: prev, + MerkleRoot: chainhash.HashH( + []byte( + fmt.Sprintf("merkle-%d", height), + ), + ), + Timestamp: time.Unix(int64(height)*600, 0), + Bits: 0x207fffff, + Nonce: uint32(salt[0])<<24 | uint32(salt[1])<<16 | + uint32(salt[2])<<8 | uint32(salt[3]), + } +} + func newStubChain(tipHeight int32) *stubChain { c := &stubChain{ tipHeight: tipHeight, + blocks: make(map[int32]*wire.BlockHeader), hashAt: make(map[int32]chainhash.Hash), } + var prev chainhash.Hash for h := int32(0); h <= tipHeight; h++ { - c.hashAt[h] = chainhash.HashH( - []byte( - fmt.Sprintf("block-%d", h), - ), - ) + hdr := mintStubHeader(h, prev) + c.blocks[h] = hdr + hash := hdr.BlockHash() + c.hashAt[h] = hash + prev = hash } return c @@ -49,13 +83,14 @@ func (c *stubChain) advance(t *testing.T, n int32) { c.mu.Lock() defer c.mu.Unlock() + prev := c.hashAt[c.tipHeight] for i := int32(1); i <= n; i++ { h := c.tipHeight + i - c.hashAt[h] = chainhash.HashH( - []byte( - fmt.Sprintf("block-%d", h), - ), - ) + hdr := mintStubHeader(h, prev) + c.blocks[h] = hdr + hash := hdr.BlockHash() + c.hashAt[h] = hash + prev = hash } c.tipHeight += n @@ -147,19 +182,29 @@ func stubEsploraHandler(t *testing.T, chain *stubChain) http.HandlerFunc { h.String(), height, int64(height)*600) + case "/header": + // Raw 80-byte header, hex-encoded. + chain.mu.Lock() + hdr := chain.blocks[height] + chain.mu.Unlock() + if hdr == nil { + http.Error( + w, "not found", + http.StatusNotFound, + ) + + return + } + var buf bytes.Buffer + require.NoError(t, hdr.Serialize(&buf)) + _, _ = fmt.Fprint( + w, + hex.EncodeToString( + buf.Bytes(), + ), + ) + default: - // Raw header / raw block — synthesize a - // header whose serialized bytes hash to - // h. We take the simple route of using - // h's bytes themselves: the header - // hash-verifier compares header.BlockHash() - // to the requested h, so any header that - // happens to round-trip works. Synthesizing - // such a header from a target hash is - // effectively impossible, so for these - // suffix variants we return 501 — tests - // that need them must use the cache - // pre-fill path. http.Error( w, "not implemented", http.StatusNotImplemented, From b21d34ea92188b1fa88ce3abe5243b73ec747e0d Mon Sep 17 00:00:00 2001 From: Elle Mouton Date: Mon, 13 Jul 2026 09:01:14 -0700 Subject: [PATCH 5/8] btcwbackend: reorg-aware Neutrino chain notifier forwarding Forward Neutrino chain-notifier reorg signals into the reorg-aware chainsource lifecycle. --- btcwbackend/chain_backend.go | 171 ++++++++--- btcwbackend/chain_backend_reorg_test.go | 359 ++++++++++++++++++++++++ 2 files changed, 493 insertions(+), 37 deletions(-) create mode 100644 btcwbackend/chain_backend_reorg_test.go diff --git a/btcwbackend/chain_backend.go b/btcwbackend/chain_backend.go index c93dcc5bf..6d63efcfe 100644 --- a/btcwbackend/chain_backend.go +++ b/btcwbackend/chain_backend.go @@ -38,8 +38,13 @@ type ChainBackend struct { neutrinoCS *neutrino.ChainService // notifier provides chain notification services backed by - // neutrino's compact block filter scanning. - notifier *neutrinonotify.NeutrinoNotifier + // neutrino's compact block filter scanning. Stored under the + // chainntnfs interface (concrete type is + // *neutrinonotify.NeutrinoNotifier in production) so reorg-aware + // forwarder tests can substitute a stub that drives the full + // Confirmed / NegativeConf / Done lifecycle without spinning up a + // neutrino chain service. + notifier chainntnfs.ChainNotifier // feeEstimator provides fee estimation from a web API since // neutrino has no mempool visibility. @@ -571,40 +576,89 @@ func (b *ChainBackend) RegisterConf(ctx context.Context, txid *chainhash.Hash, cancelOnce.Do(event.Cancel) } + // Create channels to convert neutrino's confirmation lifecycle + // (which already drives chainntnfs's NegativeConf/Done channels + // directly — neutrino emits these natively from its compact- + // block-filter scanner) into our backend-agnostic types. + // NegativeConf carries a reorg depth that the cross-backend + // chainsource layer intentionally drops, so we forward a bare + // struct{} signal on reorgChan instead. confChan := make(chan *chainsource.TxConfirmation, 1) + reorgChan := make(chan uint64, 1) + doneChan := make(chan struct{}, 1) go func() { + // Defers run in LIFO order. event.Cancel() must run first + // so the upstream notifier stops writing to its internal + // channels before we cancel notifyCtx (which any in-flight + // downstream sends are still using) and finally close the + // outgoing chans. Reversing this order would race the + // upstream notifier against closed channels. defer close(confChan) + defer close(reorgChan) + defer close(doneChan) defer cancel() defer safeCancel() - select { - case lndConf, ok := <-event.Confirmed: - if !ok { - return - } + for { + select { + case lndConf, ok := <-event.Confirmed: + if !ok { + return + } - conf := &chainsource.TxConfirmation{ - BlockHash: lndConf.BlockHash, - BlockHeight: lndConf.BlockHeight, - TxIndex: lndConf.TxIndex, - Tx: lndConf.Tx, - Block: lndConf.Block, - } + conf := &chainsource.TxConfirmation{ + BlockHash: lndConf.BlockHash, + BlockHeight: lndConf.BlockHeight, + TxIndex: lndConf.TxIndex, + Tx: lndConf.Tx, + Block: lndConf.Block, + } + + select { + case confChan <- conf: + case <-notifyCtx.Done(): + return + } + + case _, ok := <-event.NegativeConf: + if !ok { + event.NegativeConf = nil + + continue + } + + select { + case reorgChan <- uint64(0): + case <-notifyCtx.Done(): + return + } + + case _, ok := <-event.Done: + if !ok { + event.Done = nil + + continue + } + + select { + case doneChan <- struct{}{}: + case <-notifyCtx.Done(): + return + } + + return - select { - case confChan <- conf: case <-notifyCtx.Done(): return } - - case <-notifyCtx.Done(): - return } }() return &chainsource.ConfRegistration{ Confirmed: confChan, + Reorged: reorgChan, + Done: doneChan, Cancel: func() { cancel() safeCancel() @@ -641,40 +695,83 @@ func (b *ChainBackend) RegisterSpend(ctx context.Context, cancelOnce.Do(event.Cancel) } + // Create channels to convert neutrino's spend lifecycle into our + // backend-agnostic types. neutrino's Reorg carries no payload, so + // we forward a bare struct{} signal. spendChan := make(chan *chainsource.SpendDetail, 1) + reorgChan := make(chan uint64, 1) + doneChan := make(chan struct{}, 1) go func() { + // LIFO defer: event.Cancel() first so the upstream notifier + // stops writing, then cancel notifyCtx so in-flight downstream + // sends unblock, and finally close the outgoing chans. defer close(spendChan) + defer close(reorgChan) + defer close(doneChan) defer cancel() defer safeCancel() - select { - case lndSpend, ok := <-event.Spend: - if !ok { - return - } + for { + select { + case lndSpend, ok := <-event.Spend: + if !ok { + return + } - spend := &chainsource.SpendDetail{ - SpentOutPoint: lndSpend.SpentOutPoint, - SpenderTxHash: lndSpend.SpenderTxHash, - SpendingTx: lndSpend.SpendingTx, - SpenderInputIndex: lndSpend.SpenderInputIndex, - SpendingHeight: lndSpend.SpendingHeight, - } + spend := &chainsource.SpendDetail{ + SpentOutPoint: lndSpend.SpentOutPoint, + SpenderTxHash: lndSpend.SpenderTxHash, + SpendingTx: lndSpend.SpendingTx, + SpenderInputIndex: lndSpend. + SpenderInputIndex, + SpendingHeight: lndSpend.SpendingHeight, + } + + select { + case spendChan <- spend: + case <-notifyCtx.Done(): + return + } + + case _, ok := <-event.Reorg: + if !ok { + event.Reorg = nil + + continue + } + + select { + case reorgChan <- uint64(0): + case <-notifyCtx.Done(): + return + } + + case _, ok := <-event.Done: + if !ok { + event.Done = nil + + continue + } + + select { + case doneChan <- struct{}{}: + case <-notifyCtx.Done(): + return + } + + return - select { - case spendChan <- spend: case <-notifyCtx.Done(): return } - - case <-notifyCtx.Done(): - return } }() return &chainsource.SpendRegistration{ - Spend: spendChan, + Spend: spendChan, + Reorged: reorgChan, + Done: doneChan, Cancel: func() { cancel() safeCancel() diff --git a/btcwbackend/chain_backend_reorg_test.go b/btcwbackend/chain_backend_reorg_test.go new file mode 100644 index 000000000..da705b44f --- /dev/null +++ b/btcwbackend/chain_backend_reorg_test.go @@ -0,0 +1,359 @@ +package btcwbackend + +import ( + "testing" + "time" + + "github.com/btcsuite/btcd/chainhash/v2" + "github.com/btcsuite/btcd/wire/v2" + "github.com/lightninglabs/darepo-client/chainsource" + "github.com/lightningnetwork/lnd/chainntnfs" + "github.com/stretchr/testify/require" +) + +// reorgWaitTimeout is the per-step deadline used by the btcwbackend +// forwarder reorg tests. The forwarder is a tight goroutine, so the +// timeout exists only to make a hang surface as a fast failure on +// slow CI. +const reorgWaitTimeout = 2 * time.Second + +// fakeNotifier is a minimal chainntnfs.ChainNotifier whose Register* +// methods hand back a caller-supplied event struct. The neutrino-side +// of the real chainntnfs implementation produces events with all four +// channels populated (Confirmed/NegativeConf/Done for conf, +// Spend/Reorg/Done for spend), so the stub lets us drive a synthetic +// upstream lifecycle through the chainbackend forwarder without +// standing up a real neutrino chain service. +type fakeNotifier struct { + confEvent *chainntnfs.ConfirmationEvent + spendEvent *chainntnfs.SpendEvent +} + +// RegisterConfirmationsNtfn returns the stub's pre-built confirmation +// event. +func (n *fakeNotifier) RegisterConfirmationsNtfn(_ *chainhash.Hash, _ []byte, _, + _ uint32, _ ...chainntnfs.NotifierOption) ( + *chainntnfs.ConfirmationEvent, error) { + + return n.confEvent, nil +} + +// RegisterSpendNtfn returns the stub's pre-built spend event. +func (n *fakeNotifier) RegisterSpendNtfn(_ *wire.OutPoint, _ []byte, _ uint32) ( + *chainntnfs.SpendEvent, error) { + + return n.spendEvent, nil +} + +// RegisterBlockEpochNtfn returns an empty block-epoch event. The +// reorg lifecycle tests do not exercise block epochs, so the channels +// are intentionally never written to. +func (n *fakeNotifier) RegisterBlockEpochNtfn(_ *chainntnfs.BlockEpoch) ( + *chainntnfs.BlockEpochEvent, error) { + + return &chainntnfs.BlockEpochEvent{ + Epochs: make(chan *chainntnfs.BlockEpoch), + Cancel: func() {}, + }, nil +} + +// Start satisfies the ChainNotifier interface; the stub has no real +// lifecycle. +func (n *fakeNotifier) Start() error { return nil } + +// Started satisfies the ChainNotifier interface; the stub always +// reports started. +func (n *fakeNotifier) Started() bool { return true } + +// Stop satisfies the ChainNotifier interface; the stub has no real +// lifecycle. +func (n *fakeNotifier) Stop() error { return nil } + +// TestRegisterConfForwardsReorgAndDone drives the full confirmation +// lifecycle through neutrino's chainntnfs notifier into the btcwbackend +// forwarder and asserts each event arrives on the matching chainsource +// registration channel. The lifecycle is: +// +// Confirmed -> NegativeConf -> Confirmed -> Done +// +// and the test additionally verifies the forwarder exits after Done +// by observing that the chainsource channels close. +func TestRegisterConfForwardsReorgAndDone(t *testing.T) { + t.Parallel() + + confChan := make(chan *chainntnfs.TxConfirmation, 2) + negChan := make(chan int32, 1) + doneChan := make(chan struct{}, 1) + notifier := &fakeNotifier{ + confEvent: &chainntnfs.ConfirmationEvent{ + Confirmed: confChan, + NegativeConf: negChan, + Done: doneChan, + Cancel: func() {}, + }, + } + backend := &ChainBackend{notifier: notifier} + + reg, err := backend.RegisterConf( + t.Context(), &chainhash.Hash{0x42}, []byte{0x51}, 1, 100, false, + ) + require.NoError(t, err) + + // 1. First confirmation crosses the forwarder. + hash1 := chainhash.Hash{0xaa} + confChan <- &chainntnfs.TxConfirmation{ + BlockHash: &hash1, + BlockHeight: 123, + Tx: wire.NewMsgTx(2), + } + + conf1 := awaitConfForward(t, reg.Confirmed) + require.Equal(t, uint32(123), conf1.BlockHeight) + require.Equal(t, hash1, *conf1.BlockHash) + + // 2. Reorg ping is forwarded as a single struct{} on Reorged. The + // depth value carried on NegativeConf is intentionally dropped at + // this layer so neutrino and LND backends present the same wire + // shape to chainsource consumers. + negChan <- 1 + + awaitSeq(t, reg.Reorged, "Reorged forward") + + // 3. Transaction re-confirms in a different block on the new tip. + hash2 := chainhash.Hash{0xbb} + confChan <- &chainntnfs.TxConfirmation{ + BlockHash: &hash2, + BlockHeight: 124, + Tx: wire.NewMsgTx(2), + } + + conf2 := awaitConfForward(t, reg.Confirmed) + require.Equal(t, uint32(124), conf2.BlockHeight) + require.Equal(t, hash2, *conf2.BlockHash) + + // 4. Done signal is forwarded; the forwarder then exits and the + // chainsource channels close. + doneChan <- struct{}{} + + awaitStruct(t, reg.Done, "Done forward") + + // All three forwarded channels must close once the forwarder + // exits. + requireConfClosedSoon(t, reg.Confirmed) + requireSeqClosedSoon(t, reg.Reorged) + requireStructClosedSoon(t, reg.Done) +} + +// TestRegisterSpendForwardsReorgAndDone is the spend-side equivalent of +// the confirmation lifecycle test above. +func TestRegisterSpendForwardsReorgAndDone(t *testing.T) { + t.Parallel() + + spendChan := make(chan *chainntnfs.SpendDetail, 2) + reorgChan := make(chan struct{}, 1) + doneChan := make(chan struct{}, 1) + notifier := &fakeNotifier{ + spendEvent: &chainntnfs.SpendEvent{ + Spend: spendChan, + Reorg: reorgChan, + Done: doneChan, + Cancel: func() {}, + }, + } + backend := &ChainBackend{notifier: notifier} + + outpoint := &wire.OutPoint{Index: 1} + reg, err := backend.RegisterSpend( + t.Context(), outpoint, []byte{0x51}, 100, + ) + require.NoError(t, err) + + // 1. First spend. + hash1 := chainhash.Hash{0x10} + spendChan <- &chainntnfs.SpendDetail{ + SpentOutPoint: outpoint, + SpenderTxHash: &hash1, + SpendingTx: wire.NewMsgTx(2), + SpendingHeight: 150, + } + + spend1 := awaitSpendForward(t, reg.Spend) + require.Equal(t, int32(150), spend1.SpendingHeight) + require.Equal(t, hash1, *spend1.SpenderTxHash) + + // 2. Reorg evicts the spend. + reorgChan <- struct{}{} + + awaitSeq(t, reg.Reorged, "spend Reorged forward") + + // 3. A different spender wins the new chain. + hash2 := chainhash.Hash{0x20} + spendChan <- &chainntnfs.SpendDetail{ + SpentOutPoint: outpoint, + SpenderTxHash: &hash2, + SpendingTx: wire.NewMsgTx(2), + SpendingHeight: 151, + } + + spend2 := awaitSpendForward(t, reg.Spend) + require.Equal(t, int32(151), spend2.SpendingHeight) + require.Equal(t, hash2, *spend2.SpenderTxHash) + + // 4. Done. + doneChan <- struct{}{} + + awaitStruct(t, reg.Done, "spend Done forward") + + requireSpendClosedSoon(t, reg.Spend) + requireSeqClosedSoon(t, reg.Reorged) + requireStructClosedSoon(t, reg.Done) +} + +// awaitConfForward reads a single confirmation off the forwarded +// channel with a deadline, failing the test on timeout or unexpected +// close. +func awaitConfForward(t *testing.T, + ch <-chan *chainsource.TxConfirmation) *chainsource.TxConfirmation { + + t.Helper() + + select { + case conf, ok := <-ch: + if !ok { + t.Fatal("conf channel closed before delivery") + } + + return conf + + case <-time.After(reorgWaitTimeout): + t.Fatal("timeout waiting for conf forward") + + return nil + } +} + +// awaitSpendForward reads a single spend off the forwarded channel +// with a deadline, failing the test on timeout or unexpected close. +func awaitSpendForward(t *testing.T, + ch <-chan *chainsource.SpendDetail) *chainsource.SpendDetail { + + t.Helper() + + select { + case spend, ok := <-ch: + if !ok { + t.Fatal("spend channel closed before delivery") + } + + return spend + + case <-time.After(reorgWaitTimeout): + t.Fatal("timeout waiting for spend forward") + + return nil + } +} + +// awaitStruct reads a single struct{} off the forwarded channel with a +// deadline. Used for the Reorged and Done channels. +func awaitStruct(t *testing.T, ch <-chan struct{}, label string) { + t.Helper() + + select { + case _, ok := <-ch: + if !ok { + t.Fatalf("%s channel closed before delivery", label) + } + + case <-time.After(reorgWaitTimeout): + t.Fatalf("timeout waiting for %s", label) + } +} + +// requireConfClosedSoon asserts the confirmation channel closes within +// the reorg wait timeout. Used to verify the forwarder's defer-close +// chain runs after Done is delivered. +func requireConfClosedSoon(t *testing.T, + ch <-chan *chainsource.TxConfirmation) { + + t.Helper() + + require.Eventually(t, func() bool { + select { + case _, ok := <-ch: + return !ok + + default: + return false + } + }, reorgWaitTimeout, 10*time.Millisecond, + "conf channel did not close after Done") +} + +// requireSpendClosedSoon asserts the spend channel closes within the +// reorg wait timeout. +func requireSpendClosedSoon(t *testing.T, ch <-chan *chainsource.SpendDetail) { + t.Helper() + + require.Eventually(t, func() bool { + select { + case _, ok := <-ch: + return !ok + + default: + return false + } + }, reorgWaitTimeout, 10*time.Millisecond, + "spend channel did not close after Done") +} + +// requireStructClosedSoon asserts that a struct{} signal channel +// closes within the reorg wait timeout. +func requireStructClosedSoon(t *testing.T, ch <-chan struct{}) { + t.Helper() + + require.Eventually(t, func() bool { + select { + case _, ok := <-ch: + return !ok + + default: + return false + } + }, reorgWaitTimeout, 10*time.Millisecond, + "struct channel did not close after Done") +} + +// awaitSeq reads a single sequence value off the forwarded Reorged +// channel with a deadline. chainsource retyped Reorged from struct{} +// to the ordering sequence; this backend always sends 0. +func awaitSeq(t *testing.T, ch <-chan uint64, label string) { + t.Helper() + + select { + case _, ok := <-ch: + if !ok { + t.Fatalf("%s channel closed before delivery", label) + } + + case <-time.After(reorgWaitTimeout): + t.Fatalf("timeout waiting for %s", label) + } +} + +// requireSeqClosedSoon asserts that a sequence-carrying signal channel +// closes within the reorg wait timeout. +func requireSeqClosedSoon(t *testing.T, ch <-chan uint64) { + t.Helper() + + require.Eventually(t, func() bool { + select { + case _, ok := <-ch: + return !ok + + default: + return false + } + }, reorgWaitTimeout, 10*time.Millisecond, + "seq channel did not close after Done") +} From d63bd67fcd124172d58d98c9c180b3cb706ea163 Mon Sep 17 00:00:00 2001 From: Elle Mouton Date: Mon, 13 Jul 2026 09:01:14 -0700 Subject: [PATCH 6/8] multi: wire reorg-aware lifecycle into wallet, harness, systests Consume the reorg-aware substrate at the edges: make the wallet boarding sweep reorg-aware, enable height-based finality on the darepod chainsource actor, add the harness GetRawTransaction / SignedV3Tx helpers, and add the end-to-end reorg systests for chainsource and txconfirm. --- darepod/server.go | 7 + harness/harness.go | 169 ++++++++++++++++ systest/reorg_test.go | 244 +++++++++++++++++++++++ systest/systest.go | 7 + systest/txconfirm_reorg_test.go | 221 +++++++++++++++++++++ wallet/boarding_sweep_actor.go | 294 +++++++++++++++++++++++----- wallet/boarding_sweep_actor_test.go | 271 +++++++++++++++++++++++-- 7 files changed, 1158 insertions(+), 55 deletions(-) create mode 100644 systest/reorg_test.go create mode 100644 systest/txconfirm_reorg_test.go diff --git a/darepod/server.go b/darepod/server.go index 58b0082c9..c671f4c0a 100644 --- a/darepod/server.go +++ b/darepod/server.go @@ -2082,6 +2082,13 @@ func (s *Server) registerChainSourceActor( chainsource.ChainSourceConfig{ Backend: s.chainBackend, System: s.actorSystem, + // Enable height-based Done synthesis at the default + // safety depth. Without this, conf/spend sub-actors + // driven through lndclient (whose Done channel is + // allocated-but-never-written) would never receive a + // finality signal, and reorg-aware consumers like + // txconfirm would leak watch state forever. + FinalityDepth: chainsource.DefaultFinalityDepth, }, ) diff --git a/harness/harness.go b/harness/harness.go index a5c9a3cd5..12c5f4258 100644 --- a/harness/harness.go +++ b/harness/harness.go @@ -9,6 +9,7 @@ import ( "bytes" "context" crand "crypto/rand" + "encoding/hex" "encoding/json" "flag" "fmt" @@ -26,8 +27,13 @@ import ( "testing" "time" + btcaddr "github.com/btcsuite/btcd/address/v2" "github.com/btcsuite/btcd/btcutil/v2" + "github.com/btcsuite/btcd/chaincfg/v2" + "github.com/btcsuite/btcd/chainhash/v2" "github.com/btcsuite/btcd/rpcclient" + "github.com/btcsuite/btcd/txscript/v2" + "github.com/btcsuite/btcd/wire/v2" "github.com/lightninglabs/darepo-client/chain" "github.com/lightninglabs/lndclient" "github.com/lightninglabs/taproot-assets/taprpc" @@ -1773,6 +1779,169 @@ func (h *Harness) BlockHeader(hash string) BlockHeader { return hdr } +// GetRawTransaction fetches the raw transaction body for txid from +// bitcoind and deserializes it into a wire.MsgTx. The transaction must +// exist either in the current chain or in bitcoind's mempool; +// `getrawtransaction` returns it via the wallet's lookup path in either +// case for a faucet-sent tx, so callers do not need to mine first. +func (h *Harness) GetRawTransaction(txid string) *wire.MsgTx { + h.T.Helper() + + res, err := h.bitcoinRPCCall("getrawtransaction", txid) + require.NoError(h.T, err, "getrawtransaction rpc failed") + + var hexStr string + err = json.Unmarshal(res, &hexStr) + require.NoError(h.T, err, "getrawtransaction unmarshal failed") + + rawBytes, err := hex.DecodeString(hexStr) + require.NoError(h.T, err, "decode raw tx hex failed") + + tx := wire.NewMsgTx(2) + err = tx.Deserialize(bytes.NewReader(rawBytes)) + require.NoError(h.T, err, "deserialize raw tx failed") + + return tx +} + +// SignedV3Tx builds a TRUC (v3) transaction that sends `amount` to +// `destPkScript`, uses one of bitcoind's wallet UTXOs as input, and +// returns it fully signed but NOT broadcast. The change output goes +// back to a fresh address from bitcoind's wallet. +// +// This helper exists for reorg systests that need to feed a v3 tx into +// txconfirm: bitcoind's `sendtoaddress` faucet path produces v2 txs, +// and txconfirm's CPFP broadcaster enforces v3/TRUC at the version +// gate. Building the v3 tx ourselves and letting txconfirm broadcast +// it is the cleanest way to exercise the full tracked-tx lifecycle on +// a freshly-mined chain entry. +// +// Fee rate is fixed at 5 sat/vB — well above any regtest min-relay-fee +// floor and small enough that the leftover change is reusable. +func (h *Harness) SignedV3Tx(destPkScript []byte, + amount btcutil.Amount) *wire.MsgTx { + + h.T.Helper() + + h.bitcoindEnsureWallet() + + // Find a confirmed wallet UTXO with enough value to cover the + // destination + change + a generous fee. + utxoTxid, utxoVout, utxoValueBTC := h.bitcoindFirstSpendableUTXO() + utxoValue := btcutil.Amount(utxoValueBTC * btcutil.SatoshiPerBitcoin) + + feeSat := btcutil.Amount(2_000) // ~5 sat/vB on a ~400 vB tx. + require.Greater( + h.T, utxoValue, amount+feeSat, + "selected UTXO not large enough for destination + fee", + ) + changeSat := utxoValue - amount - feeSat + + // Fresh change address. + changeAddrRes, err := h.bitcoinRPCCall("getnewaddress") + require.NoError(h.T, err, "getnewaddress for change failed") + var changeAddrStr string + require.NoError( + h.T, json.Unmarshal(changeAddrRes, &changeAddrStr), + "getnewaddress unmarshal failed", + ) + changeAddr, err := btcaddr.DecodeAddress( + changeAddrStr, &chaincfg.RegressionNetParams, + ) + require.NoError(h.T, err, "decode change address failed") + changePkScript, err := txscript.PayToAddrScript(changeAddr) + require.NoError(h.T, err, "derive change pkScript failed") + + // Build the unsigned v3 tx. + tx := wire.NewMsgTx(3) + prevHash, err := chainhash.NewHashFromStr(utxoTxid) + require.NoError(h.T, err, "parse selected UTXO txid") + tx.AddTxIn(&wire.TxIn{ + PreviousOutPoint: wire.OutPoint{ + Hash: *prevHash, Index: utxoVout, + }, + Sequence: wire.MaxTxInSequenceNum - 1, + }) + tx.AddTxOut(&wire.TxOut{ + Value: int64(amount), + PkScript: destPkScript, + }) + tx.AddTxOut(&wire.TxOut{ + Value: int64(changeSat), + PkScript: changePkScript, + }) + + // Hex-encode and ask bitcoind to sign. + var unsignedBuf bytes.Buffer + require.NoError( + h.T, tx.Serialize(&unsignedBuf), + "serialize unsigned v3 tx", + ) + unsignedHex := hex.EncodeToString(unsignedBuf.Bytes()) + + signRes, err := h.bitcoinRPCCall( + "signrawtransactionwithwallet", unsignedHex, + ) + require.NoError(h.T, err, "signrawtransactionwithwallet failed") + + var signResult struct { + Hex string `json:"hex"` + Complete bool `json:"complete"` + } + require.NoError( + h.T, json.Unmarshal(signRes, &signResult), + "signrawtransactionwithwallet unmarshal failed", + ) + require.True( + h.T, signResult.Complete, "tx signing incomplete: %s", + signResult.Hex, + ) + + signedBytes, err := hex.DecodeString(signResult.Hex) + require.NoError(h.T, err, "decode signed v3 tx hex failed") + + signed := wire.NewMsgTx(3) + require.NoError( + h.T, + signed.Deserialize( + bytes.NewReader(signedBytes), + ), + "deserialize signed v3 tx failed", + ) + require.Equal( + h.T, int32(3), signed.Version, + "signing must preserve v3 version", + ) + + return signed +} + +// bitcoindFirstSpendableUTXO picks the first confirmed wallet UTXO via +// `listunspent` and returns its txid, vout, and amount in BTC. +func (h *Harness) bitcoindFirstSpendableUTXO() (string, uint32, float64) { + h.T.Helper() + + // Restrict to confirmed and spendable; bitcoind defaults are + // already minconf=1, maxconf=9999999. + res, err := h.bitcoinRPCCall("listunspent") + require.NoError(h.T, err, "listunspent rpc failed") + + var utxos []struct { + Txid string `json:"txid"` + Vout uint32 `json:"vout"` + Amount float64 `json:"amount"` + } + require.NoError( + h.T, json.Unmarshal(res, &utxos), + "listunspent unmarshal failed", + ) + require.NotEmpty(h.T, utxos, "no spendable wallet UTXOs") + + first := utxos[0] + + return first.Txid, first.Vout, first.Amount +} + // Faucet funds a test address by sending the specified amount from bitcoind's // default wallet, creating unconfirmed UTXOs for tests to spend. This mimics // external funding without requiring manual transaction construction. diff --git a/systest/reorg_test.go b/systest/reorg_test.go new file mode 100644 index 000000000..bdb19a64b --- /dev/null +++ b/systest/reorg_test.go @@ -0,0 +1,244 @@ +//go:build systest + +package systest + +import ( + "crypto/sha256" + "testing" + "time" + + btcaddr "github.com/btcsuite/btcd/address/v2" + "github.com/btcsuite/btcd/btcutil/v2" + "github.com/btcsuite/btcd/chaincfg/v2" + "github.com/btcsuite/btcd/chainhash/v2" + "github.com/btcsuite/btcd/txscript/v2" + "github.com/lightninglabs/darepo-client/baselib/actor" + "github.com/lightninglabs/darepo-client/chainsource" + "github.com/lightningnetwork/lnd/fn/v2" + "github.com/stretchr/testify/require" +) + +// reorgSystestEventTimeout is the per-step deadline used by the +// chainsource reorg systests. The chain-notification pipeline runs +// over gRPC to a real lnd instance, so the timeout is generous enough +// to absorb container startup variance and notifier wake-up latency. +const reorgSystestEventTimeout = 30 * time.Second + +// TestChainSourceConfReorgRoundTrip drives a real bitcoind reorg +// through the full chainsource pipeline: +// +// lnd chainntnfs (in-process) +// -> lndclient gRPC (WithReOrgChan) +// -> chainbackends.LndClientChainNotifier (bridge) +// -> chainbackends.LNDBackend (multi-shot forwarder) +// -> chainsource.ConfActor (reorg-aware mode) +// -> test actor refs +// +// The flow is: +// +// 1. Register a reorg-aware confirmation watch on a synthetic P2WPKH +// pkScript whose txid we know once we faucet to it. +// 2. Faucet + mine one block. Assert ConfirmationEvent arrives with +// the expected (txid, blockHeight, blockHash). +// 3. Drive a 1-block reorg via the harness helper, which invalidates +// the confirmation block and mines a strictly longer (2-block) +// replacement branch. Bitcoind preserves the original tx in its +// mempool, so the transaction re-confirms in the new chain at a +// different block hash (and potentially the same height). +// 4. Assert ConfReorgedEvent arrives. +// 5. Assert a fresh ConfirmationEvent arrives with the new chain's +// block hash (NOT the original one), demonstrating that lnd's +// chainntnfs dispatched a re-confirmation after the reorg and +// that every layer above it propagated the multi-shot signal +// correctly. +// +// This is the systest-level oracle for "the reorg-aware pipeline +// actually works over the real gRPC transport". The unit tests in +// chainsource/reorg_test.go and chainbackends/lnd_reorg_test.go prove +// the same lifecycle against mocks, but they cannot prove that +// lndclient.WithReOrgChan fires when wired to real lnd. Only this +// test does. +func TestChainSourceConfReorgRoundTrip(t *testing.T) { + ParallelN(t) + + h := NewSysTestHarness(t) + ctx := h.Context() + + // Spawn a real chainsource actor over the harness's LND. + chainSource := h.NewChainSourceActor() + + // Build a synthetic P2WPKH address from a deterministic per-test + // pubkey hash. The address does not need a controllable key (we + // never spend it in this test); we just need a known pkScript so + // we can register a confirmation watch. + pubKeyHash := sha256.Sum256([]byte(t.Name())) + addr, err := btcaddr.NewAddressWitnessPubKeyHash( + pubKeyHash[:20], &chaincfg.RegressionNetParams, + ) + require.NoError(t, err, "build synthetic P2WPKH address") + pkScript, err := txscript.PayToAddrScript(addr) + require.NoError(t, err, "derive pkScript for synthetic address") + + // Wire test refs for each event variant. The Reorged ref also + // has to be set for NotifyDone-or-NotifyReorged admission to flip + // the sub-actor into multi-shot mode; we leave Done unwired + // because the lndclient transport does not synthesize a Done + // signal so it would never fire over a real lnd run anyway. + confRef := actor.NewChannelTellOnlyRef[chainsource.ConfirmationEvent]( + "systest-conf", 8, + ) + reorgRef := actor.NewChannelTellOnlyRef[chainsource.ConfReorgedEvent]( + "systest-conf-reorged", 8, + ) + + // Register the watch BEFORE the tx hits the mempool so we + // exercise the live-detection path rather than the + // historical-backfill path. + heightHint := h.Harness.BlockCount() + amount := btcutil.Amount(btcutil.SatoshiPerBitcoin / 100) + + // We need the txid up front, which means faucet first, then + // register, then mine. The watch is on the txid + pkScript pair + // so the ordering between mempool entry and registration is OK; + // what must NOT happen is that we mine the block before + // registering, because lnd's notifier would then dispatch + // historical confirmation state and our test would be racing two + // delivery paths. + txidStr := h.Harness.Faucet(addr.String(), amount) + txid, err := chainhash.NewHashFromStr(txidStr) + require.NoError(t, err, "parse faucet txid") + + confNotify := actor.TellOnlyRef[chainsource.ConfirmationEvent]( + confRef, + ) + reorgNotify := actor.TellOnlyRef[chainsource.ConfReorgedEvent]( + reorgRef, + ) + + regResp := chainSource.Ask(ctx, &chainsource.RegisterConfRequest{ + CallerID: "test-reorg-conf-" + txidStr, + Txid: txid, + PkScript: pkScript, + TargetConfs: 1, + HeightHint: heightHint, + NotifyActor: fn.Some(confNotify), + NotifyReorged: fn.Some(reorgNotify), + }).Await(ctx) + require.True(t, regResp.IsOk(), "register reorg-aware conf watch") + resp, err := regResp.Unpack() + require.NoError(t, err) + _, ok := resp.(*chainsource.RegisterConfResponse) + require.True(t, ok, "unexpected register response type") + + // 1. Mine the block that confirms the faucet tx. + originalBlocks := h.Harness.Generate(1) + require.Len(t, originalBlocks, 1) + originalBlock := originalBlocks[0] + + originalHash, err := chainhash.NewHashFromStr(originalBlock.Hash) + require.NoError(t, err, "parse original block hash") + + // 2. Assert the first ConfirmationEvent. + firstConf := awaitConfEvent(t, confRef) + require.Equal(t, *txid, firstConf.Txid, "first conf txid mismatch") + require.Equal( + t, int32(originalBlock.Height), firstConf.BlockHeight, + "first conf block height should match the mined block", + ) + require.Equal( + t, *originalHash, firstConf.BlockHash, + "first conf block hash should match the mined block", + ) + t.Logf( + "first ConfirmationEvent: txid=%s height=%d hash=%s", + firstConf.Txid, firstConf.BlockHeight, firstConf.BlockHash, + ) + + // 3. Drive a reorg: invalidate the conf block, mine a strictly + // longer replacement branch. The harness Reorg helper waits for + // lnd's chain sync to catch up before returning. + reorg := h.Harness.Reorg(1, 2) + require.Equal( + t, originalBlock.Hash, reorg.Disconnected[0].Hash, + "the reorg should have disconnected the conf block", + ) + require.Len(t, reorg.Connected, 2) + t.Logf( + "reorg: disconnected=%d connected=%d fork=%d", + len(reorg.Disconnected), len(reorg.Connected), + reorg.ForkPoint.Height, + ) + + // 4. Assert the ConfReorgedEvent. Lnd's chainntnfs notifier + // dispatches NegativeConf for the disconnected confirmation + // asynchronously after processing the disconnect; the gRPC + // transport adds a further hop, so this can take longer than + // the initial confirmation event. + reorgEvt := awaitReorgEvent(t, reorgRef) + require.Equal(t, *txid, reorgEvt.Txid, "reorg event txid mismatch") + t.Logf("ConfReorgedEvent: txid=%s", reorgEvt.Txid) + + // 5. Assert a fresh ConfirmationEvent for the same tx, now in + // the replacement chain. Bitcoind keeps the tx in mempool across + // the invalidate, so generatetoaddress picks it back up on the + // first new block. The new block hash MUST differ from the + // original; the height may or may not match depending on where + // the tx landed in the replacement branch. + secondConf := awaitConfEvent(t, confRef) + require.Equal(t, *txid, secondConf.Txid, "re-conf txid mismatch") + require.NotEqual( + t, firstConf.BlockHash, secondConf.BlockHash, + "re-confirmation must arrive in a new block", + ) + t.Logf( + "second ConfirmationEvent: txid=%s height=%d hash=%s", + secondConf.Txid, secondConf.BlockHeight, secondConf.BlockHash, + ) + + // Sanity: the new block hash must be one of the harness-reported + // connected blocks. chainhash.Hash.String() renders the + // canonical big-endian hex form that bitcoind's RPCs emit, so + // the strings can be compared directly. + connectedHashes := make(map[string]struct{}, len(reorg.Connected)) + for _, blk := range reorg.Connected { + connectedHashes[blk.Hash] = struct{}{} + } + require.Contains( + t, connectedHashes, secondConf.BlockHash.String(), + "re-confirmation block must belong to the replacement branch", + ) +} + +// awaitConfEvent reads a single ConfirmationEvent from the test ref +// with a generous deadline, failing the test on timeout. +func awaitConfEvent(t *testing.T, + ref *actor.ChannelTellOnlyRef[chainsource.ConfirmationEvent], +) chainsource.ConfirmationEvent { + + t.Helper() + + evt, ok := ref.AwaitMessage(reorgSystestEventTimeout) + require.True( + t, ok, "timeout waiting for ConfirmationEvent (%s)", + reorgSystestEventTimeout, + ) + + return evt +} + +// awaitReorgEvent reads a single ConfReorgedEvent from the test ref +// with a generous deadline, failing the test on timeout. +func awaitReorgEvent(t *testing.T, + ref *actor.ChannelTellOnlyRef[chainsource.ConfReorgedEvent], +) chainsource.ConfReorgedEvent { + + t.Helper() + + evt, ok := ref.AwaitMessage(reorgSystestEventTimeout) + require.True( + t, ok, "timeout waiting for ConfReorgedEvent (%s)", + reorgSystestEventTimeout, + ) + + return evt +} diff --git a/systest/systest.go b/systest/systest.go index bfdc46a03..c2f264876 100644 --- a/systest/systest.go +++ b/systest/systest.go @@ -212,6 +212,13 @@ func (h *SysTestHarness) NewChainSourceActor() actor.ActorRef[ chainsource.ChainSourceConfig{ Backend: backend, System: h.actorSystem, + // Mirror the production wiring so systests that + // register reorg-aware conf/spend watches do not + // leak per-watch sub-actors past test teardown: + // the lndclient backend never writes the upstream + // Done channel, so without height-based synthesis + // reorg-aware watches would stay open forever. + FinalityDepth: chainsource.DefaultFinalityDepth, }.WithLogger( h.SubLogger(chainsource.Subsystem), ), diff --git a/systest/txconfirm_reorg_test.go b/systest/txconfirm_reorg_test.go new file mode 100644 index 000000000..e52bbae20 --- /dev/null +++ b/systest/txconfirm_reorg_test.go @@ -0,0 +1,221 @@ +//go:build systest + +package systest + +import ( + "context" + "crypto/sha256" + "testing" + "time" + + btcaddr "github.com/btcsuite/btcd/address/v2" + "github.com/btcsuite/btcd/btcutil/v2" + "github.com/btcsuite/btcd/chaincfg/v2" + "github.com/btcsuite/btcd/txscript/v2" + "github.com/lightninglabs/darepo-client/baselib/actor" + "github.com/lightninglabs/darepo-client/txconfirm" + "github.com/stretchr/testify/require" +) + +// txConfirmSystestEventTimeout is the per-step deadline used by the +// txconfirm reorg systest. Generous because the chain-notification +// pipeline runs over gRPC to a real lnd instance plus this test waits +// for txconfirm's tracked-tx FSM to forward each event. +const txConfirmSystestEventTimeout = 30 * time.Second + +// recordingNotificationRef captures every Notification delivered to a +// txconfirm subscriber so the test can assert on the order and shape +// of the lifecycle without racing concurrent delivery. +type recordingNotificationRef struct { + id string + msgs chan txconfirm.Notification +} + +func newRecordingNotificationRef(id string) *recordingNotificationRef { + return &recordingNotificationRef{ + id: id, + msgs: make(chan txconfirm.Notification, 16), + } +} + +// ID returns the subscriber identifier. +func (r *recordingNotificationRef) ID() string { + return r.id +} + +// Tell records the inbound notification on the channel for the test to +// consume. +func (r *recordingNotificationRef) Tell(_ context.Context, + msg txconfirm.Notification) error { + + r.msgs <- msg + + return nil +} + +// await pulls the next notification or fails the test on timeout. +func (r *recordingNotificationRef) await(t *testing.T) txconfirm.Notification { + t.Helper() + + select { + case msg := <-r.msgs: + return msg + + case <-time.After(txConfirmSystestEventTimeout): + t.Fatalf("timeout waiting for txconfirm notification (%s)", + txConfirmSystestEventTimeout) + + return nil + } +} + +// TestTxConfirmReorgRoundTrip drives a real bitcoind reorg through +// the full txconfirm pipeline: +// +// lnd chainntnfs (in-process) +// -> lndclient gRPC (WithReOrgChan) +// -> chainbackends.LndClientChainNotifier (bridge) +// -> chainbackends.LNDBackend (multi-shot forwarder) +// -> chainsource.ConfActor (reorg-aware mode + finality synth) +// -> txconfirm.TxBroadcasterActor (tracked-tx FSM) +// -> recording subscriber +// +// This is the systest-level oracle for the layer that unroll consumes. +// The chainsource-level systest (TestChainSourceConfReorgRoundTrip) +// proves the chain-event plumbing; this one proves the tracked-tx FSM +// transitions Confirmed -> AwaitingConfirmation -> Confirmed correctly +// on real reorgs and that subscribers see TxConfirmed -> TxReorged -> +// TxConfirmed in order, with TxFinalized arriving once the +// height-based safety depth is reached. +// +// Full daemon-level end-to-end coverage (the VTXOUnrollActor walking +// a real proof through a real wallet under a real reorg) belongs in +// itest; the unroll FSM's reducer behavior on these events is already +// covered by unit tests in unroll/reorg_safety_test.go. +func TestTxConfirmReorgRoundTrip(t *testing.T) { + ParallelN(t) + + h := NewSysTestHarness(t) + ctx := h.Context() + + chainSource := h.NewChainSourceActor() + + // Spawn a txconfirm actor over the real chainsource. Wallet is + // nil because the faucet tx is non-anchor and we never trigger + // CPFP fee-input selection in this test. + txconfBehavior := txconfirm.NewTxBroadcasterActor(txconfirm.Config{ + ChainSource: chainSource, + }) + txconfInstance := actor.NewActor(actor.ActorConfig[ + txconfirm.Msg, txconfirm.Resp, + ]{ + ID: "txconfirm-systest", + Behavior: txconfBehavior, + MailboxSize: 64, + }) + txconfBehavior.SetSelfRef(txconfInstance.TellRef()) + txconfInstance.Start() + t.Cleanup(txconfInstance.Stop) + + // Synthetic watched address: deterministic P2WPKH derived from + // the test name. We faucet to it so we have a known txid and + // pkScript for txconfirm to track. The address is never spent. + pubKeyHash := sha256.Sum256([]byte(t.Name())) + addr, err := btcaddr.NewAddressWitnessPubKeyHash( + pubKeyHash[:20], &chaincfg.RegressionNetParams, + ) + require.NoError(t, err, "build synthetic P2WPKH address") + pkScript, err := txscript.PayToAddrScript(addr) + require.NoError(t, err, "derive pkScript for synthetic address") + + heightHint := h.Harness.BlockCount() + + // txconfirm's CPFP broadcaster enforces v3/TRUC at the version + // gate, and the standard bitcoind faucet returns v2 txs. Build a + // v3 tx that pays our synthetic address from bitcoind's wallet, + // sign it, but do NOT broadcast — txconfirm will handle the + // first broadcast on EnsureConfirmedReq. + signedTx := h.Harness.SignedV3Tx( + pkScript, btcutil.Amount(btcutil.SatoshiPerBitcoin/100), + ) + txidVal := signedTx.TxHash() + txid := &txidVal + t.Logf("constructed v3 tx: txid=%s", txid) + + subscriber := newRecordingNotificationRef("txconfirm-sub") + var subRef actor.TellOnlyRef[txconfirm.Notification] = subscriber + + // Register the EnsureConfirmedReq BEFORE mining so we exercise + // the live-detection path and the tracked-tx FSM walks the full + // Broadcasting -> AwaitingConfirmation -> Confirmed transitions. + ensureResp, err := txconfInstance.Ref().Ask( + ctx, &txconfirm.EnsureConfirmedReq{ + Tx: signedTx, + ConfirmationPkScript: pkScript, + Label: "systest-reorg", + HeightHint: heightHint, + TargetConfs: 1, + Subscriber: subRef, + }, + ).Await(ctx).Unpack() + require.NoError(t, err, "EnsureConfirmedReq failed") + require.IsType(t, &txconfirm.EnsureConfirmedResp{}, ensureResp) + + // 1. Mine the block that confirms the faucet tx. + originalBlocks := h.Harness.Generate(1) + require.Len(t, originalBlocks, 1) + originalBlock := originalBlocks[0] + + // 2. Expect TxConfirmed. + first := subscriber.await(t) + firstConfirmed, ok := first.(*txconfirm.TxConfirmed) + require.True( + t, ok, "first notification must be TxConfirmed, got %T", first, + ) + require.Equal(t, *txid, firstConfirmed.Txid) + require.Equal( + t, int32(originalBlock.Height), firstConfirmed.BlockHeight, + "first conf block height should match the mined block", + ) + t.Logf( + "first TxConfirmed: txid=%s height=%d", firstConfirmed.Txid, + firstConfirmed.BlockHeight, + ) + + // 3. Reorg the conf block out, mine a strictly longer + // replacement branch, wait for lnd to chain-sync. + reorg := h.Harness.Reorg(1, 2) + require.Equal( + t, originalBlock.Hash, reorg.Disconnected[0].Hash, + "the reorg should have disconnected the conf block", + ) + require.Len(t, reorg.Connected, 2) + t.Logf( + "reorg: disconnected=%d connected=%d fork=%d", + len(reorg.Disconnected), len(reorg.Connected), + reorg.ForkPoint.Height, + ) + + // 4. Expect TxReorged. + second := subscriber.await(t) + reorgedMsg, ok := second.(*txconfirm.TxReorged) + require.True( + t, ok, "second notification must be TxReorged, got %T", second, + ) + require.Equal(t, *txid, reorgedMsg.Txid) + t.Logf("TxReorged: txid=%s", reorgedMsg.Txid) + + // 5. Expect a fresh TxConfirmed on the replacement chain. The + // faucet tx stays in mempool across the invalidate so it + // re-confirms in the first new block. + third := subscriber.await(t) + secondConfirmed, ok := third.(*txconfirm.TxConfirmed) + require.True( + t, ok, "third notification must be TxConfirmed, got %T", third, + ) + require.Equal(t, *txid, secondConfirmed.Txid) + t.Logf( + "second TxConfirmed: txid=%s height=%d", secondConfirmed.Txid, + secondConfirmed.BlockHeight, + ) +} diff --git a/wallet/boarding_sweep_actor.go b/wallet/boarding_sweep_actor.go index 255e54fab..2620ee413 100644 --- a/wallet/boarding_sweep_actor.go +++ b/wallet/boarding_sweep_actor.go @@ -233,29 +233,154 @@ func (m BoardingSweepSpendNotification) MessageType() string { func (m BoardingSweepSpendNotification) walletMsgSealed() {} -// BoardingSweepTxNotification is a Tell carrying a txconfirm terminal -// notification (confirmation or failure) for a tracked sweep tx, -// re-wrapped from txconfirm.TxConfirmed / txconfirm.TxFailed via +// BoardingSweepTxStatus identifies which point in the txconfirm +// reorg-aware lifecycle drove this notification. Splitting the status +// out (instead of a single `Confirmed bool`) makes it impossible for +// the handler to confuse a reorg-out with a terminal failure: txconfirm +// fan-outs a TxReorged whenever a previously delivered TxConfirmed is +// rolled back on chain, and we must NOT treat that as a sweep failure. +type BoardingSweepTxStatus int + +const ( + // BoardingSweepTxStatusUnknown is the zero value; receiving it + // indicates a programmer error (MapNotification missed a kind). + BoardingSweepTxStatusUnknown BoardingSweepTxStatus = iota + + // BoardingSweepTxStatusConfirmed reports that the sweep + // transaction has been observed on the canonical chain at + // BlockHeight with NumConfs confirmations. The observation is + // provisional until BoardingSweepTxStatusFinalized arrives — a + // reorg can roll it back via BoardingSweepTxStatusReorged. + BoardingSweepTxStatusConfirmed + + // BoardingSweepTxStatusReorged reports that a previously + // delivered TxConfirmed for this sweep was reorged out. The + // handler must NOT call MarkBoardingSweepFailed; instead it + // leaves the spend watches and pending state armed so the next + // TxConfirmed on the new canonical chain (or an eventual + // TxFailed) drives the terminal decision. + BoardingSweepTxStatusReorged + + // BoardingSweepTxStatusFinalized reports that the sweep + // confirmation is past the chainsource backend's reorg-safety + // depth and is no longer reversible. The handler can release + // any reorg-recovery bookkeeping for this sweep. + BoardingSweepTxStatusFinalized + + // BoardingSweepTxStatusFailed reports a terminal failure from + // txconfirm (broadcast rejected, retry budget exhausted, etc.). + // The handler marks the sweep failed in the store and cleans up + // pending state. + BoardingSweepTxStatusFailed +) + +// classifyTxconfirmNotificationForBoardingSweep maps one event from +// the txconfirm reorg-aware lifecycle into the wallet-domain +// BoardingSweepTxNotification shape consumed by +// handleSweepTxNotification. Pulled out of submitSweepConfirmer so the +// systest (TestBoardingSweepReorgRoundTrip) can reuse the exact same +// classifier the production wiring uses; if the production wiring +// drifts from the systest classifier, the test stops validating +// production behavior. +func classifyTxconfirmNotificationForBoardingSweep( + n txconfirm.Notification) BoardingSweepTxNotification { + + switch ev := n.(type) { + case *txconfirm.TxConfirmed: + return BoardingSweepTxNotification{ + Status: BoardingSweepTxStatusConfirmed, + + Txid: ev.Txid, + BlockHeight: ev.BlockHeight, + NumConfs: ev.NumConfs, + } + + case *txconfirm.TxReorged: + // Reorg-out: do NOT mark the sweep as failed in the + // handler. Leave spend watches and pending state armed so + // the next TxConfirmed on the new canonical chain (or an + // eventual TxFailed) drives the terminal decision. + return BoardingSweepTxNotification{ + Status: BoardingSweepTxStatusReorged, + Txid: ev.Txid, + } + + case *txconfirm.TxFinalized: + // Reorg-safety horizon reached. The sweep observation is + // no longer reversible; the handler may release reorg- + // recovery bookkeeping. Audit/ledger emission already + // fired on the original Confirmed (idempotent on replay). + return BoardingSweepTxNotification{ + Status: BoardingSweepTxStatusFinalized, + + Txid: ev.Txid, + BlockHeight: ev.BlockHeight, + NumConfs: ev.NumConfs, + } + + case *txconfirm.TxFailed: + return BoardingSweepTxNotification{ + Status: BoardingSweepTxStatusFailed, + Txid: ev.Txid, + Reason: ev.Reason, + } + } + + return BoardingSweepTxNotification{} +} + +// NewBoardingSweepTxconfirmSubscriber wires the production boarding- +// sweep classifier onto a txconfirm.MapNotification anchored at +// selfRef. The returned ref can be set as the Subscriber field on a +// txconfirm.EnsureConfirmedReq; every TxConfirmed / TxReorged / +// TxFinalized / TxFailed delivered for that sweep will be classified +// into a BoardingSweepTxNotification and Tell'd to selfRef as a +// WalletMsg, which the wallet actor's Receive arm dispatches to +// handleSweepTxNotification. +// +// Exported so the systest can construct the exact same subscriber +// chain that submitSweepConfirmer uses in production. +func NewBoardingSweepTxconfirmSubscriber( + selfRef actor.TellOnlyRef[WalletMsg], +) actor.TellOnlyRef[txconfirm.Notification] { + + walletNotif := actor.NewMapInputRef[ + BoardingSweepTxNotification, WalletMsg, + ]( + selfRef, + func(n BoardingSweepTxNotification) WalletMsg { + return n + }, + ) + + return txconfirm.MapNotification( + walletNotif, classifyTxconfirmNotificationForBoardingSweep, + ) +} + +// BoardingSweepTxNotification is a Tell carrying one event from the +// txconfirm reorg-aware lifecycle for a tracked sweep tx, re-wrapped +// from txconfirm.{TxConfirmed,TxReorged,TxFinalized,TxFailed} via // txconfirm.MapNotification. type BoardingSweepTxNotification struct { actor.BaseMessage - // Confirmed is true when the underlying txconfirm.TxConfirmed event - // fired; false when it was txconfirm.TxFailed. - Confirmed bool + // Status identifies which lifecycle event this notification + // carries. See BoardingSweepTxStatus. + Status BoardingSweepTxStatus // Txid identifies the tracked sweep transaction. Txid chainhash.Hash // BlockHeight is the height at which the sweep confirmed when - // Confirmed=true; zero otherwise. + // Status=Confirmed or Finalized; zero otherwise. BlockHeight int32 - // NumConfs is the confirmation count when Confirmed=true; zero - // otherwise. + // NumConfs is the confirmation count when Status=Confirmed or + // Finalized; zero otherwise. NumConfs uint32 - // Reason is the human-readable failure reason when Confirmed=false. + // Reason is the human-readable failure reason when Status=Failed. Reason string } @@ -430,6 +555,16 @@ func (a *Ark) loadSweepCandidates(ctx context.Context, return intents, nil } +// isTerminalSuccessSweepStatus reports whether a persisted boarding-sweep +// status represents a resolved sweep whose accounting is already booked and +// must not be rolled back to failed: confirmed (our sweep landed) or +// external_resolved (the input was spent by another path). A spurious +// TxFailed for such a sweep is ignored. +func isTerminalSuccessSweepStatus(status string) bool { + return status == BoardingSweepStatusConfirmed || + status == BoardingSweepStatusExternalResolved +} + // defaultBoardingSweepStatuses are the boarding-intent statuses considered // candidates for an aggregate timeout sweep when no outpoint set is // supplied. @@ -868,37 +1003,11 @@ func (a *Ark) cancelSweepSpendWatches(ctx context.Context, func (a *Ark) submitSweepConfirmer(ctx context.Context, tx *wire.MsgTx, pkScript []byte, heightHint uint32) error { - walletNotif := actor.NewMapInputRef[ - BoardingSweepTxNotification, WalletMsg, - ]( - a.selfRef, - func(n BoardingSweepTxNotification) WalletMsg { - return n - }, - ) - - subscriber := txconfirm.MapNotification(walletNotif, - func(n txconfirm.Notification) BoardingSweepTxNotification { - switch ev := n.(type) { - case *txconfirm.TxConfirmed: - return BoardingSweepTxNotification{ - Confirmed: true, - Txid: ev.Txid, - BlockHeight: ev.BlockHeight, - NumConfs: ev.NumConfs, - } - - case *txconfirm.TxFailed: - return BoardingSweepTxNotification{ - Confirmed: false, - Txid: ev.Txid, - Reason: ev.Reason, - } - } + if a.actorSystem == nil { + return fmt.Errorf("actor system unavailable") + } - return BoardingSweepTxNotification{} - }, - ) + subscriber := NewBoardingSweepTxconfirmSubscriber(a.selfRef) return a.submitSweepToConfirm( ctx, tx, pkScript, heightHint, boardingSweepBroadcastLabel, @@ -1061,8 +1170,8 @@ func (a *Ark) handleSweepTxNotification(ctx context.Context, return fn.Ok[WalletResp](&BoardingSweepNotificationAck{}) } - switch { - case notif.Confirmed: + switch notif.Status { + case BoardingSweepTxStatusConfirmed: a.logger(ctx).DebugS( ctx, "Boarding sweep confirmation observed by broadcaster", @@ -1074,7 +1183,74 @@ func (a *Ark) handleSweepTxNotification(ctx context.Context, a.emitSweepConfirmedLedger(ctx, notif) - default: + case BoardingSweepTxStatusReorged: + // A previously delivered TxConfirmed for this sweep was + // reorged out. Do NOT mark the sweep as failed and do NOT + // tear down the pending entry: txconfirm keeps the watch + // alive, so a subsequent TxConfirmed on the new canonical + // chain will re-run reconcileSweepInputsOnConfirm with the + // new height. MarkBoardingSweepInputSpent is idempotent on + // (outpoint, txid), and the ledger emissions use txid-keyed + // idempotency, so the second confirmation will not produce + // duplicate audit/balance entries. + // + // Best-effort backstop only: the per-input chainsource + // spend watch fires handleSweepSpendNotification if some + // other party spends an input. That spend-watch path is + // not itself reorg-symmetric — MapSpendEvent collapses the + // chainsource SpendEvent lifecycle to a single + // BoardingSweepSpendNotification without surfacing + // Reorged / Done — so a reorged-out external spender will + // leave the input row marked external_spent until manual + // reconciliation. Closing that gap is tracked separately. + a.logger(ctx).WarnS( + ctx, + "Boarding sweep confirmation reorged out; waiting "+ + "for re-confirmation or external spend", + nil, + slog.String("txid", notif.Txid.String()), + ) + + case BoardingSweepTxStatusFinalized: + // Confirmation is past the backend's reorg-safety depth. + // No further reversible events will fire for this sweep — + // reorg-recovery bookkeeping can drop. Audit/ledger + // emission already ran on Confirmed and is idempotent on + // replay, so we do not re-emit here. + a.logger(ctx).DebugS( + ctx, + "Boarding sweep confirmation finalized", + nil, + slog.String("txid", notif.Txid.String()), + slog.Int("block_height", int(notif.BlockHeight)), + ) + + case BoardingSweepTxStatusFailed: + // Defensive guard against a failure arriving for a sweep that + // already confirmed. txconfirm's contract is that TxFailed + // never follows a real TxConfirmed, but the confirmed sweep's + // ledger legs (fee + per-input + destination) are irreversible + // and txid-keyed, so rolling the intents back to failed here + // would diverge the store from the ledger. If the persisted + // record shows the sweep already reached a terminal-success + // status, ignore the failure rather than undo a booked sweep. + if rec, ok := a.lookupSweepRecord(ctx, notif.Txid); ok && + isTerminalSuccessSweepStatus(rec.Status) { + + a.logger(ctx).WarnS( + ctx, + "Ignoring boarding sweep failure for an "+ + "already-resolved sweep", + errors.New(notif.Reason), + slog.String("txid", notif.Txid.String()), + slog.String("status", rec.Status), + ) + + return fn.Ok[WalletResp]( + &BoardingSweepNotificationAck{}, + ) + } + a.logger(ctx).WarnS( ctx, "Boarding sweep broadcaster reported failure", @@ -1104,6 +1280,14 @@ func (a *Ark) handleSweepTxNotification(ctx context.Context, if pending != nil { a.cancelSweepSpendWatches(ctx, pending) } + + default: + a.logger(ctx).WarnS( + ctx, + "Boarding sweep tx notification with unknown status", + fmt.Errorf("status=%d", notif.Status), + slog.String("txid", notif.Txid.String()), + ) } return fn.Ok[WalletResp](&BoardingSweepNotificationAck{}) @@ -1116,6 +1300,18 @@ func (a *Ark) handleSweepTxNotification(ctx context.Context, // the spending txid is the sweep's own txid. MarkBoardingSweepInputSpent // is idempotent, so inputs already resolved via the spend-notification path // are left untouched. +// +// Under the reorg-aware lifecycle a second TxConfirmed can land after a +// TxReorged / re-confirmation cycle. The store's status guard rejects +// the redundant transition with sql.ErrNoRows; that signals "input row +// already advanced past pending/published" and is treated as a benign +// no-op, mirroring how handleSweepSpendNotification classifies the +// same error. Other errors still log at warn because they indicate a +// real persistence problem. Note: the input row's confirmed_height +// stays pinned to the FIRST Confirmed observation — a deliberate +// first-observation-wins audit policy that survives reorg-reconfirm +// at a different height; the reorg-safety horizon is reported +// separately via the Finalized lifecycle event. func (a *Ark) reconcileSweepInputsOnConfirm(ctx context.Context, notif BoardingSweepTxNotification) { @@ -1128,7 +1324,17 @@ func (a *Ark) reconcileSweepInputsOnConfirm(ctx context.Context, _, err := a.sweepStore.MarkBoardingSweepInputSpent( ctx, op, notif.Txid, notif.BlockHeight, ) - if err != nil { + switch { + case err == nil: + // Success. + + case errors.Is(err, sql.ErrNoRows): + // Row already past pending/published — likely + // resolved via handleSweepSpendNotification or a + // previous Confirmed in a reorg-reconfirm cycle. + // Idempotent no-op. + + default: a.logger(ctx).WarnS( ctx, "Failed to mark sweep input spent on confirm", diff --git a/wallet/boarding_sweep_actor_test.go b/wallet/boarding_sweep_actor_test.go index a1030f27f..a8d103eae 100644 --- a/wallet/boarding_sweep_actor_test.go +++ b/wallet/boarding_sweep_actor_test.go @@ -599,7 +599,7 @@ func TestSweepTxNotificationConfirmedEmitsLedger(t *testing.T) { result := a.handleSweepTxNotification( t.Context(), BoardingSweepTxNotification{ - Confirmed: true, + Status: BoardingSweepTxStatusConfirmed, Txid: swept, BlockHeight: 800_650, NumConfs: 1, @@ -719,7 +719,7 @@ func TestSweepTxNotificationConfirmedExternalDestSkipsCreated(t *testing.T) { result := a.handleSweepTxNotification( t.Context(), BoardingSweepTxNotification{ - Confirmed: true, + Status: BoardingSweepTxStatusConfirmed, Txid: swept, BlockHeight: 800_700, }, @@ -854,12 +854,13 @@ func TestSweepLedgerClearingNetsToZero(t *testing.T) { ), ) + notif := BoardingSweepTxNotification{ + Status: BoardingSweepTxStatusConfirmed, + Txid: swept, + BlockHeight: 800_800, + } result := a.handleSweepTxNotification( - t.Context(), BoardingSweepTxNotification{ - Confirmed: true, - Txid: swept, - BlockHeight: 800_800, - }, + t.Context(), notif, ) require.True(t, result.IsOk()) @@ -934,7 +935,7 @@ func TestSweepTxNotificationMissingTxSkipsLegs(t *testing.T) { result := a.handleSweepTxNotification( t.Context(), BoardingSweepTxNotification{ - Confirmed: true, + Status: BoardingSweepTxStatusConfirmed, Txid: swept, BlockHeight: 800_900, }, @@ -962,6 +963,12 @@ func TestSweepTxNotificationFailedMarksFailed(t *testing.T) { mock.Anything, ).Return(nil) + // The Failed arm looks the sweep up first to ignore failures for an + // already-resolved sweep; absent record means the failure proceeds. + store.On( + "GetBoardingSweep", mock.Anything, failedTxid, + ).Return(nil, nil) + a := newSweepTestArk(t, store, nil, 0, 0) a.pendingSweeps[failedTxid] = &pendingSweepState{ txid: failedTxid, @@ -970,9 +977,9 @@ func TestSweepTxNotificationFailedMarksFailed(t *testing.T) { result := a.handleSweepTxNotification( t.Context(), BoardingSweepTxNotification{ - Confirmed: false, - Txid: failedTxid, - Reason: "test failure", + Status: BoardingSweepTxStatusFailed, + Txid: failedTxid, + Reason: "test failure", }, ) require.True(t, result.IsOk()) @@ -980,3 +987,245 @@ func TestSweepTxNotificationFailedMarksFailed(t *testing.T) { store.AssertExpectations(t) } + +// TestSweepTxNotificationReorgedDoesNotMarkFailed verifies that a +// TxReorged event from txconfirm — which arrives whenever a previously +// observed confirmation is rolled back on chain — must NOT be treated +// as a sweep failure. The handler should leave pendingSweeps and the +// persistent sweep record intact so that the next TxConfirmed on the +// new canonical chain (or, in the worst case, a chainsource spend +// notification for some other spender of the inputs) drives the +// terminal decision. +func TestSweepTxNotificationReorgedDoesNotMarkFailed(t *testing.T) { + t.Parallel() + + reorgedTxid := chainhash.Hash{0xa1} + store := &MockBoardingSweepStore{} + // CRITICAL: MarkBoardingSweepFailed must NOT be called on reorg. + // testify/mock will fail the test if any unexpected method is + // invoked, so we simply do not register MarkBoardingSweepFailed + // here and rely on AssertExpectations to verify the negative. + + a := newSweepTestArk(t, store, nil, 0, 0) + pending := &pendingSweepState{ + txid: reorgedTxid, + inputs: map[wire.OutPoint]string{}, + } + a.pendingSweeps[reorgedTxid] = pending + + result := a.handleSweepTxNotification( + t.Context(), BoardingSweepTxNotification{ + Status: BoardingSweepTxStatusReorged, + Txid: reorgedTxid, + }, + ) + require.True(t, result.IsOk()) + + // Pending state must remain intact: a re-confirmation on the new + // canonical chain has to find the same entry to drive its + // reconcileSweepInputsOnConfirm pass. + require.Same( + t, pending, a.pendingSweeps[reorgedTxid], + "reorg must not evict the pending sweep entry", + ) + + // Mock did not register MarkBoardingSweepFailed; AssertExpectations + // passes vacuously, but any call would have failed the mock. + store.AssertExpectations(t) +} + +// TestSweepTxNotificationFinalizedIsBenign verifies that a TxFinalized +// event (the chainsource reorg-safety horizon is reached) does not +// mark the sweep failed, does not re-emit ledger entries, and does +// not tear down pending state. Pending state remains because the +// terminal release path is the same as the legacy confirmation path — +// post-finalization cleanup is a no-op since reconcileSweepInputsOnConfirm +// already ran on the original TxConfirmed. +func TestSweepTxNotificationFinalizedIsBenign(t *testing.T) { + t.Parallel() + + finalizedTxid := chainhash.Hash{0xa2} + store := &MockBoardingSweepStore{} + // No expectations registered — finalized must not call any + // store mutation method. + + a := newSweepTestArk(t, store, nil, 0, 0) + pending := &pendingSweepState{ + txid: finalizedTxid, + inputs: map[wire.OutPoint]string{}, + } + a.pendingSweeps[finalizedTxid] = pending + + result := a.handleSweepTxNotification( + t.Context(), BoardingSweepTxNotification{ + Status: BoardingSweepTxStatusFinalized, + Txid: finalizedTxid, + BlockHeight: 800_750, + NumConfs: 6, + }, + ) + require.True(t, result.IsOk()) + + // Pending state should still be present; finalized is informational. + require.Same(t, pending, a.pendingSweeps[finalizedTxid]) + + store.AssertExpectations(t) +} + +// TestSweepTxNotificationReorgedAfterPendingCleared verifies the +// Reorged handler arm survives a missing pendingSweeps entry (which +// can happen when every per-input spend notification has already +// resolved and the entry was cleaned up by handleSweepSpendNotification +// before the tx-level reorg notification arrives). +func TestSweepTxNotificationReorgedAfterPendingCleared(t *testing.T) { + t.Parallel() + + reorgedTxid := chainhash.Hash{0xa3} + store := &MockBoardingSweepStore{} + // No store expectations — reorg with no pending entry must touch + // nothing. + + a := newSweepTestArk(t, store, nil, 0, 0) + // Note: pendingSweeps is intentionally empty. + + result := a.handleSweepTxNotification( + t.Context(), BoardingSweepTxNotification{ + Status: BoardingSweepTxStatusReorged, + Txid: reorgedTxid, + }, + ) + require.True(t, result.IsOk()) + require.Empty(t, a.pendingSweeps) + + store.AssertExpectations(t) +} + +// TestSweepTxNotificationFailedAfterReorgedStillTerminates verifies +// the Reorged arm does NOT suppress a subsequent terminal Failed +// notification: a sequence of Reorged then Failed must still call +// MarkBoardingSweepFailed and drop the pending entry, otherwise a +// reorg followed by a hard broadcast failure would be silently +// stranded. +func TestSweepTxNotificationFailedAfterReorgedStillTerminates(t *testing.T) { + t.Parallel() + + txid := chainhash.Hash{0xa4} + store := &MockBoardingSweepStore{} + store.On( + "MarkBoardingSweepFailed", mock.Anything, txid, + mock.Anything, + ).Return(nil) + + // The Failed arm now looks the sweep up first to ignore a spurious + // failure for an already-resolved sweep. Here the record is absent + // (not terminal-success), so the failure path proceeds as before. + store.On( + "GetBoardingSweep", mock.Anything, txid, + ).Return(nil, nil) + + a := newSweepTestArk(t, store, nil, 0, 0) + pending := &pendingSweepState{ + txid: txid, + inputs: map[wire.OutPoint]string{}, + } + a.pendingSweeps[txid] = pending + + // Step 1: Reorged — pending should remain, store should not be + // touched. + reorgResult := a.handleSweepTxNotification( + t.Context(), BoardingSweepTxNotification{ + Status: BoardingSweepTxStatusReorged, + Txid: txid, + }, + ) + require.True(t, reorgResult.IsOk()) + require.Same(t, pending, a.pendingSweeps[txid]) + + // Step 2: Failed — terminal path must still fire even though we + // passed through Reorged first. + failResult := a.handleSweepTxNotification( + t.Context(), BoardingSweepTxNotification{ + Status: BoardingSweepTxStatusFailed, + Txid: txid, + Reason: "post-reorg broadcast failure", + }, + ) + require.True(t, failResult.IsOk()) + require.Empty( + t, a.pendingSweeps, + "Failed after Reorged must still tear down pendingSweeps", + ) + + store.AssertExpectations(t) +} + +// TestSweepTxNotificationFailedIgnoredForConfirmedSweep verifies the +// defensive guard on the Failed arm: a spurious Failed notification for a +// sweep whose persisted record already shows a terminal-success status +// (confirmed) must NOT roll the sweep back to failed, because the +// confirmed sweep's ledger legs are irreversible and txid-keyed. +func TestSweepTxNotificationFailedIgnoredForConfirmedSweep(t *testing.T) { + t.Parallel() + + txid := chainhash.Hash{0xa6} + store := &MockBoardingSweepStore{} + + // The record is already confirmed, so the guard must short-circuit + // before MarkBoardingSweepFailed; no expectation is set for it, so a + // call would fail the test. + store.On( + "GetBoardingSweep", mock.Anything, txid, + ).Return(&BoardingSweepRecord{ + Status: BoardingSweepStatusConfirmed, + }, nil) + + a := newSweepTestArk(t, store, nil, 0, 0) + pending := &pendingSweepState{ + txid: txid, + inputs: map[wire.OutPoint]string{}, + } + a.pendingSweeps[txid] = pending + + failResult := a.handleSweepTxNotification( + t.Context(), BoardingSweepTxNotification{ + Status: BoardingSweepTxStatusFailed, + Txid: txid, + Reason: "spurious failure after confirmation", + }, + ) + require.True(t, failResult.IsOk()) + + store.AssertNotCalled(t, "MarkBoardingSweepFailed") + store.AssertExpectations(t) +} + +// TestSweepTxNotificationUnknownStatusIsBenign verifies the default +// arm of handleSweepTxNotification handles an unrecognised status +// without touching the store or pendingSweeps. This guards against a +// future txconfirm lifecycle event being added without a matching +// MapNotification arm. +func TestSweepTxNotificationUnknownStatusIsBenign(t *testing.T) { + t.Parallel() + + txid := chainhash.Hash{0xa5} + store := &MockBoardingSweepStore{} + // No expectations — unknown must touch nothing. + + a := newSweepTestArk(t, store, nil, 0, 0) + pending := &pendingSweepState{ + txid: txid, + inputs: map[wire.OutPoint]string{}, + } + a.pendingSweeps[txid] = pending + + result := a.handleSweepTxNotification( + t.Context(), BoardingSweepTxNotification{ + Status: BoardingSweepTxStatusUnknown, + Txid: txid, + }, + ) + require.True(t, result.IsOk()) + require.Same(t, pending, a.pendingSweeps[txid]) + + store.AssertExpectations(t) +} From 7ab9547a14b25d2fb03f24e1f74d83d406371b37 Mon Sep 17 00:00:00 2001 From: Elle Mouton Date: Tue, 14 Jul 2026 14:03:12 -0700 Subject: [PATCH 7/8] mod: pin lnd + lndclient forks for notifier reorg depth/Done Temporary replaces onto the lnd and lndclient branches that add the reorg depth and Done events to the chain notifier, now rebased onto btcd v2 so they resolve against the current module graph. Allow-listed in the gomoddirectives linter config. Drop all three once those changes land upstream and tag (tracked in darepo-client#827). --- .golangci.yml | 5 +++++ go.mod | 4 ++++ go.sum | 8 ++++---- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 71f2dd318..6761daa0e 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -83,6 +83,11 @@ linters: # Allow btcwallet to follow the 0.21 lnd/lndclient stack while # taproot-assets still requires the released v0.16.17 tag. - github.com/btcsuite/btcwallet + # Temporary forks that surface the chain-notifier reorg depth and + # Done events over gRPC. Drop once these land upstream and tag + # (tracked in darepo-client#827). + - github.com/lightningnetwork/lnd + - github.com/lightninglabs/lndclient disable: # We instead use our own custom line length linter called `ll` since # then we can ignore log lines. diff --git a/go.mod b/go.mod index 098ad76dd..11cb33b9e 100644 --- a/go.mod +++ b/go.mod @@ -236,3 +236,7 @@ replace google.golang.org/protobuf => github.com/lightninglabs/protobuf-go-hex-d // Use local baselib for development and CI. replace github.com/lightninglabs/darepo-client/baselib => ./baselib + +replace github.com/lightningnetwork/lnd => github.com/ellemouton/lnd v0.8.0-beta-rc3.0.20260701213016-8b4da077a431 + +replace github.com/lightninglabs/lndclient => github.com/ellemouton/lndclient v1.0.1-0.20260701213152-a3baa8dfda6c diff --git a/go.sum b/go.sum index be7e4b95b..503bc3c1f 100644 --- a/go.sum +++ b/go.sum @@ -757,6 +757,10 @@ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ellemouton/lnd v0.8.0-beta-rc3.0.20260701213016-8b4da077a431 h1:KOLxRQYXb++mDgkqXhudfh5M2rZ/GG1USFsyb44LBSU= +github.com/ellemouton/lnd v0.8.0-beta-rc3.0.20260701213016-8b4da077a431/go.mod h1:LWfQaNNXlZghUJqIIh+JsgfxiVtC+ecZkrIe5bm9o4U= +github.com/ellemouton/lndclient v1.0.1-0.20260701213152-a3baa8dfda6c h1:v1aM7KEH0l4cR8FzYK9WwNQsFyj10Qm904aTbffGJZc= +github.com/ellemouton/lndclient v1.0.1-0.20260701213152-a3baa8dfda6c/go.mod h1:QlUlKGfYeQ+6lR8BpWDe4qBM91OGdiOl09esQMTEq8k= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -1046,8 +1050,6 @@ github.com/lightninglabs/gozmq v0.0.0-20191113021534-d20a764486bf h1:HZKvJUHlcXI github.com/lightninglabs/gozmq v0.0.0-20191113021534-d20a764486bf/go.mod h1:vxmQPeIQxPf6Jf9rM8R+B4rKBqLA2AjttNxkFBL2Plk= github.com/lightninglabs/lightning-node-connect/hashmailrpc v1.0.4-0.20250610182311-2f1d46ef18b7 h1:373o5lNr1udAdhcf5+zq/0dYpRtkvYLl8Lk6wG7I0DY= github.com/lightninglabs/lightning-node-connect/hashmailrpc v1.0.4-0.20250610182311-2f1d46ef18b7/go.mod h1:bDnEKRN1u13NFBuy/C+bFLhxA5bfd3clT25y76QY0AM= -github.com/lightninglabs/lndclient v1.0.1-0.20260701212139-09c54915cfba h1:PbtOHmA9P41mjWt7HDZDDzYshv8fmyhIBPkdOf8wKws= -github.com/lightninglabs/lndclient v1.0.1-0.20260701212139-09c54915cfba/go.mod h1:DtZg/wz7YFXVCwOwYOJyWKQBnUp1NFg4RQdax3MSYbc= github.com/lightninglabs/loop v0.33.0-beta h1:cfs+JrnmKw+ANa09XFS/2Jjm9wxHq3CfAzh9B1ym3e0= github.com/lightninglabs/loop v0.33.0-beta/go.mod h1:uTcjRmEC4ni4lWrOdnIvI5pyY8XJ2wXSXSE12vwFgzk= github.com/lightninglabs/migrate/v4 v4.18.2-9023d66a-fork-pr-2 h1:eFjp1dIB2BhhQp/THKrjLdlYuPugO9UU4kDqu91OX/Q= @@ -1064,8 +1066,6 @@ github.com/lightninglabs/taproot-assets/taprpc v1.1.1-0.20260706193822-2adfadc58 github.com/lightninglabs/taproot-assets/taprpc v1.1.1-0.20260706193822-2adfadc58e3c/go.mod h1:nOjUfSstCfHYY0QNnxlhN+N915Vcl5ziyHiOD8hVoJI= github.com/lightningnetwork/lightning-onion v1.4.0 h1:qWE1icOH4AKXRcq1KCzt6P/TesqptgBTP++V7wowTc0= github.com/lightningnetwork/lightning-onion v1.4.0/go.mod h1:YDPkvVTVQ6FBBE6Yj93tDd7zA3iTSrryi9xq46i7bKE= -github.com/lightningnetwork/lnd v0.21.0-beta.rc2.0.20260630214209-40c64f9db30d h1:mDcB9mwuwJXRbDmxps6mBgGs318Iz5ZbOrt0ieEx0os= -github.com/lightningnetwork/lnd v0.21.0-beta.rc2.0.20260630214209-40c64f9db30d/go.mod h1:LWfQaNNXlZghUJqIIh+JsgfxiVtC+ecZkrIe5bm9o4U= github.com/lightningnetwork/lnd/actor v0.0.6 h1:Ge8N2wivARG+27qJBwTlB0vwsypStZYZy8vk4Zl38sU= github.com/lightningnetwork/lnd/actor v0.0.6/go.mod h1:YAsoniSbY/cAM9HTVNfZLvt7RI6swDxy6wzPspTcMZg= github.com/lightningnetwork/lnd/cert v1.2.2 h1:71YK6hogeJtxSxw2teq3eGeuy4rHGKcFf0d0Uy4qBjI= From 246992ea37c1e882d23091dc311a06c0cbd5022b Mon Sep 17 00:00:00 2001 From: Elle Mouton Date: Tue, 30 Jun 2026 14:12:43 -0700 Subject: [PATCH 8/8] chainbackends: consume real reorg depth and Done from lndclient The lndclient adapter previously forwarded a sentinel reorg depth of 0 on NegativeConf and never wrote the Done channel, because lnd's gRPC notifier dropped the depth and surfaced its past-reorg-safety-depth signal only by closing the stream. With the forked lndclient now exposing both via WithReOrgDepthChan and WithDoneChan, forward the real reorg depth through the ordered confirmation bridge and wire the terminal Done straight through to the ConfirmationEvent/SpendEvent Done channels. The spend path has no depth (lnd does not track one) so its bare reorg ping is unchanged. --- chainbackends/lndclient_adapters.go | 106 +++++++++++------- chainbackends/lndclient_ordering_test.go | 136 +++++++++++++++++++++++ 2 files changed, 203 insertions(+), 39 deletions(-) create mode 100644 chainbackends/lndclient_ordering_test.go diff --git a/chainbackends/lndclient_adapters.go b/chainbackends/lndclient_adapters.go index 38cab8ee4..3bbdae686 100644 --- a/chainbackends/lndclient_adapters.go +++ b/chainbackends/lndclient_adapters.go @@ -142,14 +142,19 @@ func (n *LndClientChainNotifier) RegisterConfirmationsNtfn(txid *chainhash.Hash, } // Ask lndclient to keep the confirmation stream alive past the first - // Confirmed event and forward any subsequent reorg signal on a - // dedicated channel. Without WithReOrgChan, lndclient's receive - // loop tears the stream down after one delivery and any later - // reorg is silently dropped at the gRPC layer. - reorgPing := make(chan struct{}, 1) + // Confirmed event and forward any subsequent reorg signal (with depth) + // and the terminal Done signal on dedicated channels. Without these the + // receive loop tears the stream down after one delivery and any later + // reorg is silently dropped at the gRPC layer. WithReOrgDepthChan + // carries both the reorg signal and its depth, and WithDoneChan + // delivers lnd's "past reorg-safety depth" signal that this transport + // previously could not surface. + reorgDepth := make(chan int32, 1) + doneChan := make(chan struct{}, 1) lndOpts := []lndclient.NotifierOption{ - lndclient.WithReOrgChan(reorgPing), + lndclient.WithReOrgDepthChan(reorgDepth), + lndclient.WithDoneChan(doneChan), } if notifierOpts.IncludeBlock { lndOpts = append(lndOpts, lndclient.WithIncludeBlock()) @@ -231,9 +236,7 @@ func (n *LndClientChainNotifier) RegisterConfirmationsNtfn(txid *chainhash.Hash, // makes the ConfActor observe exactly the forwarder's (lndclient's) // order. // - // lndclient does not preserve the reorg depth across the gRPC boundary, - // so a sentinel value of 0 is forwarded on NegativeConf; callers over - // this transport must not rely on the integer value. + // lndclient now forwards the real reorg depth on NegativeConf. orderedConfirmed := make(chan *chainntnfs.TxConfirmation, 1) negativeConf := make(chan int32, 1) go func() { @@ -241,52 +244,72 @@ func (n *LndClientChainNotifier) RegisterConfirmationsNtfn(txid *chainhash.Hash, defer close(negativeConf) forwardOrderedReorg( - ctx, reorgPing, confChan, orderedConfirmed, + ctx, reorgDepth, confChan, orderedConfirmed, negativeConf, ) }() - // Done is allocated but never written to because lnd's internal - // "past reorg-safety depth" signal is not surfaced through the - // lndclient gRPC transport. Consumers needing such a gate must - // compute it themselves from block height. + // Done is now driven by lnd's "past reorg-safety depth" signal, which + // lndclient surfaces via WithDoneChan. It is a terminal, post-finality + // signal so it is forwarded directly rather than through the ordered + // confirmation forwarder. return &chainntnfs.ConfirmationEvent{ Confirmed: orderedConfirmed, NegativeConf: negativeConf, Cancel: cancel, - Done: make(chan struct{}, 1), + Done: doneChan, }, nil } // forwardOrderedReorg copies confirmations and reorg pings from lndclient's two -// source channels onto the downstream Confirmed / NegativeConf channels, -// forwarding each event in the order this single goroutine observes it. It does -// not bias either channel: the authoritative lifecycle ordering is -// re-established downstream by the per-registration sequence number the -// LNDBackend forwarder stamps (see chainsource.TxConfirmation.Seq), so the -// consumer applies highest-seq-wins regardless of how near-simultaneous events -// interleave here. lndclient's two-channel split makes a perfectly ordered -// merge impossible at this layer, so forwarding in natural arrival order keeps -// the stamped sequence faithful to what was actually observed rather than -// injecting an artificial reorg-first bias. lndclient does not preserve the -// reorg depth across the gRPC boundary, so a sentinel value of 0 is forwarded -// on NegativeConf; callers over this transport must not rely on the value. -func forwardOrderedReorg(ctx context.Context, reorgPing <-chan struct{}, +// source channels onto the downstream Confirmed / NegativeConf channels in +// lndclient's emission order. A pending reorg is drained with priority before +// any confirmation on each iteration, and every forwarded event is handed off +// with a blocking send, so the single downstream consumer observes the events +// strictly in order (it never has two of our channels ready at once). This +// preserves the reorg-before-reconfirmation ordering lndclient guarantees from +// its single ordered receive loop, which the two-channel split would otherwise +// lose at the consumer's select. +func forwardOrderedReorg(ctx context.Context, reorgDepth <-chan int32, confChan <-chan *chainntnfs.TxConfirmation, outConfirmed chan<- *chainntnfs.TxConfirmation, outNegConf chan<- int32) { - for confChan != nil || reorgPing != nil { + for confChan != nil || reorgDepth != nil { + // Priority: forward a reorg that is already pending before + // looking at confirmations, so a reorg lndclient wrote before + // the replacement Confirmed is delivered first. + if reorgDepth != nil { + select { + case depth, ok := <-reorgDepth: + if !ok { + reorgDepth = nil + + continue + } + + select { + case outNegConf <- depth: + case <-ctx.Done(): + return + } + + continue + + default: + } + } + select { - case _, ok := <-reorgPing: + case depth, ok := <-reorgDepth: if !ok { - reorgPing = nil + reorgDepth = nil continue } select { - case outNegConf <- 0: + case outNegConf <- depth: case <-ctx.Done(): return } @@ -316,10 +339,13 @@ func (n *LndClientChainNotifier) RegisterSpendNtfn(outpoint *wire.OutPoint, pkScript []byte, heightHint uint32) (*chainntnfs.SpendEvent, error) { // Ask lndclient to keep the spend stream alive past the first - // Spend event so it can forward reorg pings. Without WithReOrgChan - // the stream is torn down after the first delivery and any later - // spend reorg is silently dropped at the gRPC layer. + // Spend event so it can forward reorg pings and the terminal Done + // signal. Without these the stream is torn down after the first + // delivery and any later spend reorg is silently dropped at the gRPC + // layer. The spend notifier does not track a reorg depth, so the bare + // WithReOrgChan signal is sufficient on this path. reorgPing := make(chan struct{}, 1) + doneChan := make(chan struct{}, 1) ctx, cancel := context.WithCancel(context.Background()) @@ -336,6 +362,7 @@ func (n *LndClientChainNotifier) RegisterSpendNtfn(outpoint *wire.OutPoint, sc, ec, err := n.cfg.LND.ChainNotifier.RegisterSpendNtfn( ctx, outpoint, pkScript, int32(heightHint), lndclient.WithReOrgChan(reorgPing), + lndclient.WithDoneChan(doneChan), ) resultCh <- regResult{sc, ec, err} }() @@ -395,13 +422,14 @@ func (n *LndClientChainNotifier) RegisterSpendNtfn(outpoint *wire.OutPoint, ) }() - // Done is allocated but never written to because lnd's "past - // reorg-safety depth" signal is not surfaced through the lndclient - // gRPC transport. + // Done is now driven by lnd's "past reorg-safety depth" signal, which + // lndclient surfaces via WithDoneChan. It is a terminal, post-finality + // signal so it is forwarded directly rather than through the ordered + // spend forwarder. return &chainntnfs.SpendEvent{ Spend: orderedSpend, Reorg: reorgChan, - Done: make(chan struct{}, 1), + Done: doneChan, Cancel: cancel, }, nil } diff --git a/chainbackends/lndclient_ordering_test.go b/chainbackends/lndclient_ordering_test.go new file mode 100644 index 000000000..d6e0aab5b --- /dev/null +++ b/chainbackends/lndclient_ordering_test.go @@ -0,0 +1,136 @@ +package chainbackends + +import ( + "testing" + "time" + + "github.com/lightningnetwork/lnd/chainntnfs" + "github.com/stretchr/testify/require" +) + +// orderingWaitTimeout bounds each step of the ordered-forwarder tests so a +// hang (a forwarder that never delivers) surfaces as a fast failure. +const orderingWaitTimeout = 2 * time.Second + +// TestForwardOrderedReorgPrioritizesReorg pins the cross-channel ordering +// guarantee of the conf-path forwarder: when a reorg ping and a confirmation +// are BOTH already pending (the exact race the two-goroutine split created), +// the forwarder must deliver the NegativeConf (reorg) before the Confirmed, +// preserving lndclient's reorg-before-reconfirmation order so the downstream +// ConfActor cannot reset confirmHeight after a re-confirmation and strand the +// watch. +func TestForwardOrderedReorgPrioritizesReorg(t *testing.T) { + t.Parallel() + + ctx := t.Context() + + // Both sources ready up front: lndclient wrote the reorg before the + // replacement confirmation, and both are buffered by the time the + // forwarder runs. The reorg carries its depth (3). + reorgDepth := make(chan int32, 1) + confChan := make(chan *chainntnfs.TxConfirmation, 1) + reorgDepth <- 3 + confChan <- &chainntnfs.TxConfirmation{BlockHeight: 100} + + // The downstream channels are unbuffered so the forwarder cannot run + // ahead: it must block handing off the reorg before it can forward the + // confirmation, exactly as the single in-order consumer it feeds in + // production behaves. Buffering these would let both hand-offs complete + // before the assertion runs and make the ordering check race. + outConfirmed := make(chan *chainntnfs.TxConfirmation) + outNegConf := make(chan int32) + + go forwardOrderedReorg( + ctx, reorgDepth, confChan, outConfirmed, outNegConf, + ) + + // The reorg must arrive first, carrying the forwarded depth. + select { + case depth := <-outNegConf: + require.EqualValues(t, 3, depth) + + case <-outConfirmed: + t.Fatal("confirmation delivered before reorg") + + case <-time.After(orderingWaitTimeout): + t.Fatal("timeout waiting for the reorg") + } + + // Then the confirmation. + select { + case c := <-outConfirmed: + require.EqualValues(t, 100, c.BlockHeight) + + case <-time.After(orderingWaitTimeout): + t.Fatal("timeout waiting for the confirmation") + } +} + +// TestForwardOrderedSpendReorgPrioritizesReorg is the spend-path analogue: +// a pending reorg must be delivered on Reorg before a pending re-Spend. +func TestForwardOrderedSpendReorgPrioritizesReorg(t *testing.T) { + t.Parallel() + + ctx := t.Context() + + reorgPing := make(chan struct{}, 1) + spendChan := make(chan *chainntnfs.SpendDetail, 1) + reorgPing <- struct{}{} + spendChan <- &chainntnfs.SpendDetail{SpendingHeight: 100} + + // Unbuffered downstream channels so the forwarder cannot run ahead of + // the in-order consumer; see the conf-path test for the rationale. + outSpend := make(chan *chainntnfs.SpendDetail) + outReorg := make(chan struct{}) + + go forwardOrderedSpendReorg( + ctx, reorgPing, spendChan, outSpend, outReorg, + ) + + select { + case <-outReorg: + case <-outSpend: + t.Fatal("spend delivered before reorg") + + case <-time.After(orderingWaitTimeout): + t.Fatal("timeout waiting for the reorg") + } + + select { + case sp := <-outSpend: + require.EqualValues(t, 100, sp.SpendingHeight) + + case <-time.After(orderingWaitTimeout): + t.Fatal("timeout waiting for the spend") + } +} + +// TestForwardOrderedReorgForwardsConfirmationAlone verifies the common path: +// with no reorg pending, a confirmation is forwarded unchanged. +func TestForwardOrderedReorgForwardsConfirmationAlone(t *testing.T) { + t.Parallel() + + ctx := t.Context() + + reorgDepth := make(chan int32, 1) + confChan := make(chan *chainntnfs.TxConfirmation, 1) + confChan <- &chainntnfs.TxConfirmation{BlockHeight: 42} + + outConfirmed := make(chan *chainntnfs.TxConfirmation, 1) + outNegConf := make(chan int32, 1) + + go forwardOrderedReorg( + ctx, reorgDepth, confChan, outConfirmed, outNegConf, + ) + + select { + case c := <-outConfirmed: + require.EqualValues(t, 42, c.BlockHeight) + + case <-outNegConf: + t.Fatal("unexpected reorg with none pending") + + case <-time.After(orderingWaitTimeout): + t.Fatal("timeout waiting for the confirmation") + } +}