diff --git a/.golangci.yml b/.golangci.yml index 71f2dd318..6761daa0e 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -83,6 +83,11 @@ linters: # Allow btcwallet to follow the 0.21 lnd/lndclient stack while # taproot-assets still requires the released v0.16.17 tag. - github.com/btcsuite/btcwallet + # Temporary forks that surface the chain-notifier reorg depth and + # Done events over gRPC. Drop once these land upstream and tag + # (tracked in darepo-client#827). + - github.com/lightningnetwork/lnd + - github.com/lightninglabs/lndclient disable: # We instead use our own custom line length linter called `ll` since # then we can ignore log lines. diff --git a/btcwbackend/chain_backend.go b/btcwbackend/chain_backend.go index c93dcc5bf..6d63efcfe 100644 --- a/btcwbackend/chain_backend.go +++ b/btcwbackend/chain_backend.go @@ -38,8 +38,13 @@ type ChainBackend struct { neutrinoCS *neutrino.ChainService // notifier provides chain notification services backed by - // neutrino's compact block filter scanning. - notifier *neutrinonotify.NeutrinoNotifier + // neutrino's compact block filter scanning. Stored under the + // chainntnfs interface (concrete type is + // *neutrinonotify.NeutrinoNotifier in production) so reorg-aware + // forwarder tests can substitute a stub that drives the full + // Confirmed / NegativeConf / Done lifecycle without spinning up a + // neutrino chain service. + notifier chainntnfs.ChainNotifier // feeEstimator provides fee estimation from a web API since // neutrino has no mempool visibility. @@ -571,40 +576,89 @@ func (b *ChainBackend) RegisterConf(ctx context.Context, txid *chainhash.Hash, cancelOnce.Do(event.Cancel) } + // Create channels to convert neutrino's confirmation lifecycle + // (which already drives chainntnfs's NegativeConf/Done channels + // directly — neutrino emits these natively from its compact- + // block-filter scanner) into our backend-agnostic types. + // NegativeConf carries a reorg depth that the cross-backend + // chainsource layer intentionally drops, so we forward a bare + // struct{} signal on reorgChan instead. confChan := make(chan *chainsource.TxConfirmation, 1) + reorgChan := make(chan uint64, 1) + doneChan := make(chan struct{}, 1) go func() { + // Defers run in LIFO order. event.Cancel() must run first + // so the upstream notifier stops writing to its internal + // channels before we cancel notifyCtx (which any in-flight + // downstream sends are still using) and finally close the + // outgoing chans. Reversing this order would race the + // upstream notifier against closed channels. defer close(confChan) + defer close(reorgChan) + defer close(doneChan) defer cancel() defer safeCancel() - select { - case lndConf, ok := <-event.Confirmed: - if !ok { - return - } + for { + select { + case lndConf, ok := <-event.Confirmed: + if !ok { + return + } - conf := &chainsource.TxConfirmation{ - BlockHash: lndConf.BlockHash, - BlockHeight: lndConf.BlockHeight, - TxIndex: lndConf.TxIndex, - Tx: lndConf.Tx, - Block: lndConf.Block, - } + conf := &chainsource.TxConfirmation{ + BlockHash: lndConf.BlockHash, + BlockHeight: lndConf.BlockHeight, + TxIndex: lndConf.TxIndex, + Tx: lndConf.Tx, + Block: lndConf.Block, + } + + select { + case confChan <- conf: + case <-notifyCtx.Done(): + return + } + + case _, ok := <-event.NegativeConf: + if !ok { + event.NegativeConf = nil + + continue + } + + select { + case reorgChan <- uint64(0): + case <-notifyCtx.Done(): + return + } + + case _, ok := <-event.Done: + if !ok { + event.Done = nil + + continue + } + + select { + case doneChan <- struct{}{}: + case <-notifyCtx.Done(): + return + } + + return - select { - case confChan <- conf: case <-notifyCtx.Done(): return } - - case <-notifyCtx.Done(): - return } }() return &chainsource.ConfRegistration{ Confirmed: confChan, + Reorged: reorgChan, + Done: doneChan, Cancel: func() { cancel() safeCancel() @@ -641,40 +695,83 @@ func (b *ChainBackend) RegisterSpend(ctx context.Context, cancelOnce.Do(event.Cancel) } + // Create channels to convert neutrino's spend lifecycle into our + // backend-agnostic types. neutrino's Reorg carries no payload, so + // we forward a bare struct{} signal. spendChan := make(chan *chainsource.SpendDetail, 1) + reorgChan := make(chan uint64, 1) + doneChan := make(chan struct{}, 1) go func() { + // LIFO defer: event.Cancel() first so the upstream notifier + // stops writing, then cancel notifyCtx so in-flight downstream + // sends unblock, and finally close the outgoing chans. defer close(spendChan) + defer close(reorgChan) + defer close(doneChan) defer cancel() defer safeCancel() - select { - case lndSpend, ok := <-event.Spend: - if !ok { - return - } + for { + select { + case lndSpend, ok := <-event.Spend: + if !ok { + return + } - spend := &chainsource.SpendDetail{ - SpentOutPoint: lndSpend.SpentOutPoint, - SpenderTxHash: lndSpend.SpenderTxHash, - SpendingTx: lndSpend.SpendingTx, - SpenderInputIndex: lndSpend.SpenderInputIndex, - SpendingHeight: lndSpend.SpendingHeight, - } + spend := &chainsource.SpendDetail{ + SpentOutPoint: lndSpend.SpentOutPoint, + SpenderTxHash: lndSpend.SpenderTxHash, + SpendingTx: lndSpend.SpendingTx, + SpenderInputIndex: lndSpend. + SpenderInputIndex, + SpendingHeight: lndSpend.SpendingHeight, + } + + select { + case spendChan <- spend: + case <-notifyCtx.Done(): + return + } + + case _, ok := <-event.Reorg: + if !ok { + event.Reorg = nil + + continue + } + + select { + case reorgChan <- uint64(0): + case <-notifyCtx.Done(): + return + } + + case _, ok := <-event.Done: + if !ok { + event.Done = nil + + continue + } + + select { + case doneChan <- struct{}{}: + case <-notifyCtx.Done(): + return + } + + return - select { - case spendChan <- spend: case <-notifyCtx.Done(): return } - - case <-notifyCtx.Done(): - return } }() return &chainsource.SpendRegistration{ - Spend: spendChan, + Spend: spendChan, + Reorged: reorgChan, + Done: doneChan, Cancel: func() { cancel() safeCancel() diff --git a/btcwbackend/chain_backend_reorg_test.go b/btcwbackend/chain_backend_reorg_test.go new file mode 100644 index 000000000..da705b44f --- /dev/null +++ b/btcwbackend/chain_backend_reorg_test.go @@ -0,0 +1,359 @@ +package btcwbackend + +import ( + "testing" + "time" + + "github.com/btcsuite/btcd/chainhash/v2" + "github.com/btcsuite/btcd/wire/v2" + "github.com/lightninglabs/darepo-client/chainsource" + "github.com/lightningnetwork/lnd/chainntnfs" + "github.com/stretchr/testify/require" +) + +// reorgWaitTimeout is the per-step deadline used by the btcwbackend +// forwarder reorg tests. The forwarder is a tight goroutine, so the +// timeout exists only to make a hang surface as a fast failure on +// slow CI. +const reorgWaitTimeout = 2 * time.Second + +// fakeNotifier is a minimal chainntnfs.ChainNotifier whose Register* +// methods hand back a caller-supplied event struct. The neutrino-side +// of the real chainntnfs implementation produces events with all four +// channels populated (Confirmed/NegativeConf/Done for conf, +// Spend/Reorg/Done for spend), so the stub lets us drive a synthetic +// upstream lifecycle through the chainbackend forwarder without +// standing up a real neutrino chain service. +type fakeNotifier struct { + confEvent *chainntnfs.ConfirmationEvent + spendEvent *chainntnfs.SpendEvent +} + +// RegisterConfirmationsNtfn returns the stub's pre-built confirmation +// event. +func (n *fakeNotifier) RegisterConfirmationsNtfn(_ *chainhash.Hash, _ []byte, _, + _ uint32, _ ...chainntnfs.NotifierOption) ( + *chainntnfs.ConfirmationEvent, error) { + + return n.confEvent, nil +} + +// RegisterSpendNtfn returns the stub's pre-built spend event. +func (n *fakeNotifier) RegisterSpendNtfn(_ *wire.OutPoint, _ []byte, _ uint32) ( + *chainntnfs.SpendEvent, error) { + + return n.spendEvent, nil +} + +// RegisterBlockEpochNtfn returns an empty block-epoch event. The +// reorg lifecycle tests do not exercise block epochs, so the channels +// are intentionally never written to. +func (n *fakeNotifier) RegisterBlockEpochNtfn(_ *chainntnfs.BlockEpoch) ( + *chainntnfs.BlockEpochEvent, error) { + + return &chainntnfs.BlockEpochEvent{ + Epochs: make(chan *chainntnfs.BlockEpoch), + Cancel: func() {}, + }, nil +} + +// Start satisfies the ChainNotifier interface; the stub has no real +// lifecycle. +func (n *fakeNotifier) Start() error { return nil } + +// Started satisfies the ChainNotifier interface; the stub always +// reports started. +func (n *fakeNotifier) Started() bool { return true } + +// Stop satisfies the ChainNotifier interface; the stub has no real +// lifecycle. +func (n *fakeNotifier) Stop() error { return nil } + +// TestRegisterConfForwardsReorgAndDone drives the full confirmation +// lifecycle through neutrino's chainntnfs notifier into the btcwbackend +// forwarder and asserts each event arrives on the matching chainsource +// registration channel. The lifecycle is: +// +// Confirmed -> NegativeConf -> Confirmed -> Done +// +// and the test additionally verifies the forwarder exits after Done +// by observing that the chainsource channels close. +func TestRegisterConfForwardsReorgAndDone(t *testing.T) { + t.Parallel() + + confChan := make(chan *chainntnfs.TxConfirmation, 2) + negChan := make(chan int32, 1) + doneChan := make(chan struct{}, 1) + notifier := &fakeNotifier{ + confEvent: &chainntnfs.ConfirmationEvent{ + Confirmed: confChan, + NegativeConf: negChan, + Done: doneChan, + Cancel: func() {}, + }, + } + backend := &ChainBackend{notifier: notifier} + + reg, err := backend.RegisterConf( + t.Context(), &chainhash.Hash{0x42}, []byte{0x51}, 1, 100, false, + ) + require.NoError(t, err) + + // 1. First confirmation crosses the forwarder. + hash1 := chainhash.Hash{0xaa} + confChan <- &chainntnfs.TxConfirmation{ + BlockHash: &hash1, + BlockHeight: 123, + Tx: wire.NewMsgTx(2), + } + + conf1 := awaitConfForward(t, reg.Confirmed) + require.Equal(t, uint32(123), conf1.BlockHeight) + require.Equal(t, hash1, *conf1.BlockHash) + + // 2. Reorg ping is forwarded as a single struct{} on Reorged. The + // depth value carried on NegativeConf is intentionally dropped at + // this layer so neutrino and LND backends present the same wire + // shape to chainsource consumers. + negChan <- 1 + + awaitSeq(t, reg.Reorged, "Reorged forward") + + // 3. Transaction re-confirms in a different block on the new tip. + hash2 := chainhash.Hash{0xbb} + confChan <- &chainntnfs.TxConfirmation{ + BlockHash: &hash2, + BlockHeight: 124, + Tx: wire.NewMsgTx(2), + } + + conf2 := awaitConfForward(t, reg.Confirmed) + require.Equal(t, uint32(124), conf2.BlockHeight) + require.Equal(t, hash2, *conf2.BlockHash) + + // 4. Done signal is forwarded; the forwarder then exits and the + // chainsource channels close. + doneChan <- struct{}{} + + awaitStruct(t, reg.Done, "Done forward") + + // All three forwarded channels must close once the forwarder + // exits. + requireConfClosedSoon(t, reg.Confirmed) + requireSeqClosedSoon(t, reg.Reorged) + requireStructClosedSoon(t, reg.Done) +} + +// TestRegisterSpendForwardsReorgAndDone is the spend-side equivalent of +// the confirmation lifecycle test above. +func TestRegisterSpendForwardsReorgAndDone(t *testing.T) { + t.Parallel() + + spendChan := make(chan *chainntnfs.SpendDetail, 2) + reorgChan := make(chan struct{}, 1) + doneChan := make(chan struct{}, 1) + notifier := &fakeNotifier{ + spendEvent: &chainntnfs.SpendEvent{ + Spend: spendChan, + Reorg: reorgChan, + Done: doneChan, + Cancel: func() {}, + }, + } + backend := &ChainBackend{notifier: notifier} + + outpoint := &wire.OutPoint{Index: 1} + reg, err := backend.RegisterSpend( + t.Context(), outpoint, []byte{0x51}, 100, + ) + require.NoError(t, err) + + // 1. First spend. + hash1 := chainhash.Hash{0x10} + spendChan <- &chainntnfs.SpendDetail{ + SpentOutPoint: outpoint, + SpenderTxHash: &hash1, + SpendingTx: wire.NewMsgTx(2), + SpendingHeight: 150, + } + + spend1 := awaitSpendForward(t, reg.Spend) + require.Equal(t, int32(150), spend1.SpendingHeight) + require.Equal(t, hash1, *spend1.SpenderTxHash) + + // 2. Reorg evicts the spend. + reorgChan <- struct{}{} + + awaitSeq(t, reg.Reorged, "spend Reorged forward") + + // 3. A different spender wins the new chain. + hash2 := chainhash.Hash{0x20} + spendChan <- &chainntnfs.SpendDetail{ + SpentOutPoint: outpoint, + SpenderTxHash: &hash2, + SpendingTx: wire.NewMsgTx(2), + SpendingHeight: 151, + } + + spend2 := awaitSpendForward(t, reg.Spend) + require.Equal(t, int32(151), spend2.SpendingHeight) + require.Equal(t, hash2, *spend2.SpenderTxHash) + + // 4. Done. + doneChan <- struct{}{} + + awaitStruct(t, reg.Done, "spend Done forward") + + requireSpendClosedSoon(t, reg.Spend) + requireSeqClosedSoon(t, reg.Reorged) + requireStructClosedSoon(t, reg.Done) +} + +// awaitConfForward reads a single confirmation off the forwarded +// channel with a deadline, failing the test on timeout or unexpected +// close. +func awaitConfForward(t *testing.T, + ch <-chan *chainsource.TxConfirmation) *chainsource.TxConfirmation { + + t.Helper() + + select { + case conf, ok := <-ch: + if !ok { + t.Fatal("conf channel closed before delivery") + } + + return conf + + case <-time.After(reorgWaitTimeout): + t.Fatal("timeout waiting for conf forward") + + return nil + } +} + +// awaitSpendForward reads a single spend off the forwarded channel +// with a deadline, failing the test on timeout or unexpected close. +func awaitSpendForward(t *testing.T, + ch <-chan *chainsource.SpendDetail) *chainsource.SpendDetail { + + t.Helper() + + select { + case spend, ok := <-ch: + if !ok { + t.Fatal("spend channel closed before delivery") + } + + return spend + + case <-time.After(reorgWaitTimeout): + t.Fatal("timeout waiting for spend forward") + + return nil + } +} + +// awaitStruct reads a single struct{} off the forwarded channel with a +// deadline. Used for the Reorged and Done channels. +func awaitStruct(t *testing.T, ch <-chan struct{}, label string) { + t.Helper() + + select { + case _, ok := <-ch: + if !ok { + t.Fatalf("%s channel closed before delivery", label) + } + + case <-time.After(reorgWaitTimeout): + t.Fatalf("timeout waiting for %s", label) + } +} + +// requireConfClosedSoon asserts the confirmation channel closes within +// the reorg wait timeout. Used to verify the forwarder's defer-close +// chain runs after Done is delivered. +func requireConfClosedSoon(t *testing.T, + ch <-chan *chainsource.TxConfirmation) { + + t.Helper() + + require.Eventually(t, func() bool { + select { + case _, ok := <-ch: + return !ok + + default: + return false + } + }, reorgWaitTimeout, 10*time.Millisecond, + "conf channel did not close after Done") +} + +// requireSpendClosedSoon asserts the spend channel closes within the +// reorg wait timeout. +func requireSpendClosedSoon(t *testing.T, ch <-chan *chainsource.SpendDetail) { + t.Helper() + + require.Eventually(t, func() bool { + select { + case _, ok := <-ch: + return !ok + + default: + return false + } + }, reorgWaitTimeout, 10*time.Millisecond, + "spend channel did not close after Done") +} + +// requireStructClosedSoon asserts that a struct{} signal channel +// closes within the reorg wait timeout. +func requireStructClosedSoon(t *testing.T, ch <-chan struct{}) { + t.Helper() + + require.Eventually(t, func() bool { + select { + case _, ok := <-ch: + return !ok + + default: + return false + } + }, reorgWaitTimeout, 10*time.Millisecond, + "struct channel did not close after Done") +} + +// awaitSeq reads a single sequence value off the forwarded Reorged +// channel with a deadline. chainsource retyped Reorged from struct{} +// to the ordering sequence; this backend always sends 0. +func awaitSeq(t *testing.T, ch <-chan uint64, label string) { + t.Helper() + + select { + case _, ok := <-ch: + if !ok { + t.Fatalf("%s channel closed before delivery", label) + } + + case <-time.After(reorgWaitTimeout): + t.Fatalf("timeout waiting for %s", label) + } +} + +// requireSeqClosedSoon asserts that a sequence-carrying signal channel +// closes within the reorg wait timeout. +func requireSeqClosedSoon(t *testing.T, ch <-chan uint64) { + t.Helper() + + require.Eventually(t, func() bool { + select { + case _, ok := <-ch: + return !ok + + default: + return false + } + }, reorgWaitTimeout, 10*time.Millisecond, + "seq channel did not close after Done") +} diff --git a/chainbackends/lnd.go b/chainbackends/lnd.go index 43fab8519..61c9015ca 100644 --- a/chainbackends/lnd.go +++ b/chainbackends/lnd.go @@ -27,6 +27,20 @@ import ( // unexported there and so must be matched by value. const lndNeutrinoBroadcastMsg = "broadcast-unverified" +// reorgSignalBufferSize bounds the buffered reorg notifications forwarded +// from lnd's notifier to a chainsource registration. The forwarder sends +// to this channel with a blocking send, so a full buffer head-of-line +// blocks delivery of unrelated Confirmed / Done events on the same +// forwarder goroutine. A reorg burst (e.g. a multi-block reorg emitting +// one NegativeConf per disconnected block) can produce several signals +// back-to-back; sizing the buffer to absorb a typical reorg depth keeps +// the forwarder moving. The signal is coalescing — the consumer +// re-queries chain state on any reorg — so exact depth need not be +// preserved; the buffer only needs to be deep enough to avoid stalling, +// not to record every event. Eight comfortably covers realistic reorg +// depths while staying negligible in memory. +const reorgSignalBufferSize = 8 + // TxBroadcaster is a minimal interface for broadcasting transactions. This // allows LNDBackend to work with both lnwallet.WalletController (in-process // lnd) and lndclient wrappers (remote lnd via gRPC). @@ -330,37 +344,98 @@ func (b *LNDBackend) RegisterConf(ctx context.Context, txid *chainhash.Hash, // context is released. notifyCtx, cancel := context.WithCancel(context.Background()) - // Create a channel to convert lnd's TxConfirmation to our type. + // Create channels to convert lnd's confirmation lifecycle to our + // backend-agnostic types. NegativeConf carries a reorg depth that + // the lndclient gRPC transport cannot preserve, so we forward the + // forwarder's sequence number instead (see below). confChan := make(chan *chainsource.TxConfirmation, 1) + reorgChan := make(chan uint64, reorgSignalBufferSize) + doneChan := make(chan struct{}, 1) go func() { + // seq is a per-registration monotonic counter stamped onto + // every Confirmed and Reorged signal in the order this single + // forwarder goroutine observes them. Confirmed and Reorged + // leave on separate channels, so a select over both at any + // downstream hop (here, and again in the chainsource conf + // actor) cannot recover their order; the shared sequence lets + // the final consumer apply highest-seq-wins and discard a + // stale signal that lost a cross-channel race. This goroutine + // is the single authoritative ordering point — whatever order + // it reads lnd's channels in is the order the consumer honors. + var seq uint64 + // Defers run in LIFO order. event.Cancel() must run first so + // the upstream notifier stops writing to its internal + // channels before we cancel notifyCtx (which any in-flight + // downstream sends are still using) and finally close the + // outgoing chans. Reversing this order would race the + // upstream notifier against closed channels. defer close(confChan) + defer close(reorgChan) + defer close(doneChan) defer cancel() defer event.Cancel() - select { - case lndConf, ok := <-event.Confirmed: - if !ok { - return - } + for { + select { + case lndConf, ok := <-event.Confirmed: + if !ok { + return + } - conf := &chainsource.TxConfirmation{ - BlockHash: lndConf.BlockHash, - BlockHeight: lndConf.BlockHeight, - TxIndex: lndConf.TxIndex, - Tx: lndConf.Tx, - Block: lndConf.Block, - } + seq++ + conf := &chainsource.TxConfirmation{ + BlockHash: lndConf.BlockHash, + BlockHeight: lndConf.BlockHeight, + TxIndex: lndConf.TxIndex, + Tx: lndConf.Tx, + Block: lndConf.Block, + Seq: seq, + } + + select { + case confChan <- conf: + case <-notifyCtx.Done(): + return + } - confChan <- conf + case _, ok := <-event.NegativeConf: + if !ok { + event.NegativeConf = nil + continue + } - case <-notifyCtx.Done(): - return + seq++ + select { + case reorgChan <- seq: + case <-notifyCtx.Done(): + return + } + + case _, ok := <-event.Done: + if !ok { + event.Done = nil + continue + } + + select { + case doneChan <- struct{}{}: + case <-notifyCtx.Done(): + return + } + + return + + case <-notifyCtx.Done(): + return + } } }() return &chainsource.ConfRegistration{ Confirmed: confChan, + Reorged: reorgChan, + Done: doneChan, Cancel: func() { cancel() event.Cancel() @@ -393,39 +468,97 @@ func (b *LNDBackend) RegisterSpend(ctx context.Context, outpoint *wire.OutPoint, // Keep spend delivery alive independently of the actor request context. notifyCtx, cancel := context.WithCancel(context.Background()) - // Create a channel to convert lnd's SpendDetail to our type. + // Create channels to convert lnd's spend lifecycle to our + // backend-agnostic types. lnd's spend Reorg carries no payload, so + // we forward the forwarder's sequence number instead (see below). spendChan := make(chan *chainsource.SpendDetail, 1) + reorgChan := make(chan uint64, reorgSignalBufferSize) + doneChan := make(chan struct{}, 1) - // Start a goroutine to convert and forward the spend. + // Start a goroutine to convert and forward spend lifecycle events. go func() { + // seq is a per-registration monotonic counter stamped onto + // every Spend and Reorged signal in the order this single + // forwarder observes them. Spend and Reorged leave on separate + // channels, so a downstream select cannot recover their order; + // the shared sequence lets the final consumer apply + // highest-seq-wins and discard a stale signal that lost a + // cross-channel race. This goroutine is the single + // authoritative ordering point. + var seq uint64 + // Defer order is LIFO: event.Cancel() stops the upstream + // notifier first, then cancel() ends notifyCtx, and only + // then are the outgoing channels closed. This avoids a race + // where the upstream notifier writes to a freshly-closed + // downstream channel. defer close(spendChan) + defer close(reorgChan) + defer close(doneChan) defer cancel() defer event.Cancel() - select { - case lndSpend, ok := <-event.Spend: - if !ok { - return - } + for { + select { + case lndSpend, ok := <-event.Spend: + if !ok { + return + } - // Convert to our type. - spend := &chainsource.SpendDetail{ - SpentOutPoint: lndSpend.SpentOutPoint, - SpenderTxHash: lndSpend.SpenderTxHash, - SpendingTx: lndSpend.SpendingTx, - SpenderInputIndex: lndSpend.SpenderInputIndex, - SpendingHeight: lndSpend.SpendingHeight, - } + // Convert to our type. + seq++ + spend := &chainsource.SpendDetail{ + SpentOutPoint: lndSpend.SpentOutPoint, + SpenderTxHash: lndSpend.SpenderTxHash, + SpendingTx: lndSpend.SpendingTx, + SpenderInputIndex: lndSpend. + SpenderInputIndex, + SpendingHeight: lndSpend.SpendingHeight, + Seq: seq, + } + + select { + case spendChan <- spend: + case <-notifyCtx.Done(): + return + } + + case _, ok := <-event.Reorg: + if !ok { + event.Reorg = nil + continue + } + + seq++ + select { + case reorgChan <- seq: + case <-notifyCtx.Done(): + return + } + + case _, ok := <-event.Done: + if !ok { + event.Done = nil + continue + } + + select { + case doneChan <- struct{}{}: + case <-notifyCtx.Done(): + return + } - spendChan <- spend + return - case <-notifyCtx.Done(): - return + case <-notifyCtx.Done(): + return + } } }() return &chainsource.SpendRegistration{ - Spend: spendChan, + Spend: spendChan, + Reorged: reorgChan, + Done: doneChan, Cancel: func() { cancel() event.Cancel() diff --git a/chainbackends/lnd_reorg_test.go b/chainbackends/lnd_reorg_test.go new file mode 100644 index 000000000..622f8516d --- /dev/null +++ b/chainbackends/lnd_reorg_test.go @@ -0,0 +1,308 @@ +package chainbackends + +import ( + "testing" + "time" + + "github.com/btcsuite/btcd/chainhash/v2" + "github.com/btcsuite/btcd/wire/v2" + "github.com/lightninglabs/darepo-client/chainsource" + "github.com/lightningnetwork/lnd/chainntnfs" + "github.com/stretchr/testify/require" +) + +// reorgWaitTimeout is the per-step deadline used by the LNDBackend +// forwarder reorg tests. The forwarder is a tight goroutine, so the +// timeout exists only to make a hang surface as a fast failure on slow CI. +const reorgWaitTimeout = 2 * time.Second + +// TestRegisterConfForwardsReorgAndDone drives the full confirmation +// lifecycle through the chainntnfs notifier into the LNDBackend forwarder +// and asserts each event arrives on the matching chainsource registration +// channel. The lifecycle is: +// +// Confirmed -> NegativeConf -> Confirmed -> Done +// +// and the test additionally verifies the forwarder exits after Done by +// observing that the chainsource channels close. +func TestRegisterConfForwardsReorgAndDone(t *testing.T) { + t.Parallel() + + confChan := make(chan *chainntnfs.TxConfirmation, 2) + negChan := make(chan int32, 1) + doneChan := make(chan struct{}, 1) + notifier := &stubNotifier{ + confEvent: &chainntnfs.ConfirmationEvent{ + Confirmed: confChan, + NegativeConf: negChan, + Done: doneChan, + Cancel: func() {}, + }, + } + backend := NewLNDBackend( + notifier, &stubFeeEstimator{}, &stubBroadcaster{}, + ) + + reg, err := backend.RegisterConf( + t.Context(), &chainhash.Hash{0x42}, []byte{0x51}, 1, 100, false, + ) + require.NoError(t, err) + + // 1. First confirmation crosses the forwarder. + hash1 := chainhash.Hash{0xaa} + confChan <- &chainntnfs.TxConfirmation{ + BlockHash: &hash1, + BlockHeight: 123, + Tx: wire.NewMsgTx(2), + } + + conf1 := awaitConfForward(t, reg.Confirmed) + require.Equal(t, uint32(123), conf1.BlockHeight) + require.Equal(t, hash1, *conf1.BlockHash) + + // 2. Reorg ping is forwarded as a single struct{} on Reorged. The + // depth value carried on NegativeConf is intentionally dropped at + // this layer; consumers must not rely on it. + negChan <- 1 + + awaitSeq(t, reg.Reorged, "Reorged forward") + + // 3. Transaction re-confirms in a different block on the new tip. + hash2 := chainhash.Hash{0xbb} + confChan <- &chainntnfs.TxConfirmation{ + BlockHash: &hash2, + BlockHeight: 124, + Tx: wire.NewMsgTx(2), + } + + conf2 := awaitConfForward(t, reg.Confirmed) + require.Equal(t, uint32(124), conf2.BlockHeight) + require.Equal(t, hash2, *conf2.BlockHash) + + // 4. Done signal is forwarded; the forwarder then exits and the + // chainsource channels close. + doneChan <- struct{}{} + + awaitStruct(t, reg.Done, "Done forward") + + // All three forwarded channels must close once the forwarder exits. + requireConfClosedSoon(t, reg.Confirmed) + requireSeqClosedSoon(t, reg.Reorged) + requireStructClosedSoon(t, reg.Done) +} + +// TestRegisterSpendForwardsReorgAndDone is the spend-side equivalent of +// the confirmation lifecycle test above. +func TestRegisterSpendForwardsReorgAndDone(t *testing.T) { + t.Parallel() + + spendChan := make(chan *chainntnfs.SpendDetail, 2) + reorgChan := make(chan struct{}, 1) + doneChan := make(chan struct{}, 1) + notifier := &stubNotifier{ + spendEvent: &chainntnfs.SpendEvent{ + Spend: spendChan, + Reorg: reorgChan, + Done: doneChan, + Cancel: func() {}, + }, + } + backend := NewLNDBackend( + notifier, &stubFeeEstimator{}, &stubBroadcaster{}, + ) + + outpoint := &wire.OutPoint{Index: 1} + reg, err := backend.RegisterSpend( + t.Context(), outpoint, []byte{0x51}, 100, + ) + require.NoError(t, err) + + // 1. First spend. + hash1 := chainhash.Hash{0x10} + spendChan <- &chainntnfs.SpendDetail{ + SpentOutPoint: outpoint, + SpenderTxHash: &hash1, + SpendingTx: wire.NewMsgTx(2), + SpendingHeight: 150, + } + + spend1 := awaitSpendForward(t, reg.Spend) + require.Equal(t, int32(150), spend1.SpendingHeight) + require.Equal(t, hash1, *spend1.SpenderTxHash) + + // 2. Reorg evicts the spend. + reorgChan <- struct{}{} + + awaitSeq(t, reg.Reorged, "spend Reorged forward") + + // 3. A different spender wins the new chain. + hash2 := chainhash.Hash{0x20} + spendChan <- &chainntnfs.SpendDetail{ + SpentOutPoint: outpoint, + SpenderTxHash: &hash2, + SpendingTx: wire.NewMsgTx(2), + SpendingHeight: 151, + } + + spend2 := awaitSpendForward(t, reg.Spend) + require.Equal(t, int32(151), spend2.SpendingHeight) + require.Equal(t, hash2, *spend2.SpenderTxHash) + + // 4. Done. + doneChan <- struct{}{} + + awaitStruct(t, reg.Done, "spend Done forward") + + requireSpendClosedSoon(t, reg.Spend) + requireSeqClosedSoon(t, reg.Reorged) + requireStructClosedSoon(t, reg.Done) +} + +// awaitConfForward reads a single confirmation off the forwarded channel +// with a deadline, failing the test on timeout or unexpected close. +func awaitConfForward(t *testing.T, + ch <-chan *chainsource.TxConfirmation) *chainsource.TxConfirmation { + + t.Helper() + + select { + case conf, ok := <-ch: + if !ok { + t.Fatal("conf channel closed before delivery") + } + + return conf + + case <-time.After(reorgWaitTimeout): + t.Fatal("timeout waiting for conf forward") + + return nil + } +} + +// awaitSpendForward reads a single spend off the forwarded channel with a +// deadline, failing the test on timeout or unexpected close. +func awaitSpendForward(t *testing.T, + ch <-chan *chainsource.SpendDetail) *chainsource.SpendDetail { + + t.Helper() + + select { + case spend, ok := <-ch: + if !ok { + t.Fatal("spend channel closed before delivery") + } + + return spend + + case <-time.After(reorgWaitTimeout): + t.Fatal("timeout waiting for spend forward") + + return nil + } +} + +// awaitStruct reads a single struct{} off the forwarded channel with a +// deadline. Used for the Reorged and Done channels. +func awaitStruct(t *testing.T, ch <-chan struct{}, label string) { + t.Helper() + + select { + case _, ok := <-ch: + if !ok { + t.Fatalf("%s channel closed before delivery", label) + } + + case <-time.After(reorgWaitTimeout): + t.Fatalf("timeout waiting for %s", label) + } +} + +// requireConfClosedSoon asserts the confirmation channel closes within +// the reorg wait timeout. Used to verify the forwarder's defer-close +// chain runs after Done is delivered. +func requireConfClosedSoon(t *testing.T, + ch <-chan *chainsource.TxConfirmation) { + + t.Helper() + + require.Eventually(t, func() bool { + select { + case _, ok := <-ch: + return !ok + + default: + return false + } + }, reorgWaitTimeout, 10*time.Millisecond, + "conf channel did not close after Done") +} + +// requireSpendClosedSoon asserts the spend channel closes within the +// reorg wait timeout. +func requireSpendClosedSoon(t *testing.T, ch <-chan *chainsource.SpendDetail) { + t.Helper() + + require.Eventually(t, func() bool { + select { + case _, ok := <-ch: + return !ok + + default: + return false + } + }, reorgWaitTimeout, 10*time.Millisecond, + "spend channel did not close after Done") +} + +// requireStructClosedSoon asserts that a struct{} signal channel closes +// within the reorg wait timeout. +func requireStructClosedSoon(t *testing.T, ch <-chan struct{}) { + t.Helper() + + require.Eventually(t, func() bool { + select { + case _, ok := <-ch: + return !ok + + default: + return false + } + }, reorgWaitTimeout, 10*time.Millisecond, + "struct channel did not close after Done") +} + +// awaitSeq reads a single sequence number off the forwarded Reorged +// channel with a deadline. The reorg signal now carries the forwarder's +// monotonic sequence (see chainsource.TxConfirmation.Seq) rather than a +// bare struct{}. +func awaitSeq(t *testing.T, ch <-chan uint64, label string) { + t.Helper() + + select { + case _, ok := <-ch: + if !ok { + t.Fatalf("%s channel closed before delivery", label) + } + + case <-time.After(reorgWaitTimeout): + t.Fatalf("timeout waiting for %s", label) + } +} + +// requireSeqClosedSoon asserts that a sequence-carrying signal channel +// closes within the reorg wait timeout. +func requireSeqClosedSoon(t *testing.T, ch <-chan uint64) { + t.Helper() + + require.Eventually(t, func() bool { + select { + case _, ok := <-ch: + return !ok + + default: + return false + } + }, reorgWaitTimeout, 10*time.Millisecond, + "seq channel did not close after Done") +} diff --git a/chainbackends/lndclient_adapters.go b/chainbackends/lndclient_adapters.go index 326a93fb0..3bbdae686 100644 --- a/chainbackends/lndclient_adapters.go +++ b/chainbackends/lndclient_adapters.go @@ -17,6 +17,16 @@ import ( "github.com/lightningnetwork/lnd/lnwallet/chainfee" ) +// lndRegistrationTimeout bounds how long a conf/spend registration call into +// lndclient may block before we give up and return an error. lnd can be slow +// to answer a RegisterConfirmationsNtfn / RegisterSpendNtfn while it is busy +// processing a fresh block, and a registration that hangs forever would pin +// the chainsource sub-actor's Receive call and back-pressure the factory +// actor onto every other in-flight registration. Fifteen seconds is well +// above lnd's normal under-load response time yet short enough that a +// genuinely wedged backend surfaces as an error rather than a silent hang. +const lndRegistrationTimeout = 15 * time.Second + // LndClientTxBroadcaster implements TxBroadcaster using // lndclient.WalletKitClient. type LndClientTxBroadcaster struct { @@ -131,7 +141,21 @@ func (n *LndClientChainNotifier) RegisterConfirmationsNtfn(txid *chainhash.Hash, opt(notifierOpts) } - var lndOpts []lndclient.NotifierOption + // Ask lndclient to keep the confirmation stream alive past the first + // Confirmed event and forward any subsequent reorg signal (with depth) + // and the terminal Done signal on dedicated channels. Without these the + // receive loop tears the stream down after one delivery and any later + // reorg is silently dropped at the gRPC layer. WithReOrgDepthChan + // carries both the reorg signal and its depth, and WithDoneChan + // delivers lnd's "past reorg-safety depth" signal that this transport + // previously could not surface. + reorgDepth := make(chan int32, 1) + doneChan := make(chan struct{}, 1) + + lndOpts := []lndclient.NotifierOption{ + lndclient.WithReOrgDepthChan(reorgDepth), + lndclient.WithDoneChan(doneChan), + } if notifierOpts.IncludeBlock { lndOpts = append(lndOpts, lndclient.WithIncludeBlock()) } @@ -177,11 +201,11 @@ func (n *LndClientChainNotifier) RegisterConfirmationsNtfn(txid *chainhash.Hash, confChan = r.confChan errChan = r.errChan - case <-time.After(15 * time.Second): + case <-time.After(lndRegistrationTimeout): cancel() - return nil, fmt.Errorf("register confirmations timed out " + - "after 15s") + return nil, fmt.Errorf("register confirmations timed out "+ + "after %s", lndRegistrationTimeout) } go func() { @@ -199,26 +223,169 @@ func (n *LndClientChainNotifier) RegisterConfirmationsNtfn(txid *chainhash.Hash, } }() + // Forward the confirmation lifecycle through a SINGLE goroutine so the + // reorg ping and the (re-)confirmation reach the downstream chainntnfs + // channels in lndclient's emission order. lndclient drives both off one + // ordered gRPC receive loop and writes the reorg ping before the + // replacement Confirmed, but it splits them across two channels + // (confChan and reorgPing); if we forwarded each on its own goroutine, + // the downstream ConfActor's select could consume a re-Confirmed before + // the Reorged, reset confirmHeight to 0 with no further Confirmed + // coming, and strand the watch. Draining a pending reorg with priority + // on each iteration, and handing every event off with a blocking send, + // makes the ConfActor observe exactly the forwarder's (lndclient's) + // order. + // + // lndclient now forwards the real reorg depth on NegativeConf. + orderedConfirmed := make(chan *chainntnfs.TxConfirmation, 1) + negativeConf := make(chan int32, 1) + go func() { + defer close(orderedConfirmed) + defer close(negativeConf) + + forwardOrderedReorg( + ctx, reorgDepth, confChan, orderedConfirmed, + negativeConf, + ) + }() + + // Done is now driven by lnd's "past reorg-safety depth" signal, which + // lndclient surfaces via WithDoneChan. It is a terminal, post-finality + // signal so it is forwarded directly rather than through the ordered + // confirmation forwarder. return &chainntnfs.ConfirmationEvent{ - Confirmed: confChan, - Cancel: cancel, - Done: make(chan struct{}, 1), + Confirmed: orderedConfirmed, + NegativeConf: negativeConf, + Cancel: cancel, + Done: doneChan, }, nil } +// forwardOrderedReorg copies confirmations and reorg pings from lndclient's two +// source channels onto the downstream Confirmed / NegativeConf channels in +// lndclient's emission order. A pending reorg is drained with priority before +// any confirmation on each iteration, and every forwarded event is handed off +// with a blocking send, so the single downstream consumer observes the events +// strictly in order (it never has two of our channels ready at once). This +// preserves the reorg-before-reconfirmation ordering lndclient guarantees from +// its single ordered receive loop, which the two-channel split would otherwise +// lose at the consumer's select. +func forwardOrderedReorg(ctx context.Context, reorgDepth <-chan int32, + confChan <-chan *chainntnfs.TxConfirmation, + outConfirmed chan<- *chainntnfs.TxConfirmation, + outNegConf chan<- int32) { + + for confChan != nil || reorgDepth != nil { + // Priority: forward a reorg that is already pending before + // looking at confirmations, so a reorg lndclient wrote before + // the replacement Confirmed is delivered first. + if reorgDepth != nil { + select { + case depth, ok := <-reorgDepth: + if !ok { + reorgDepth = nil + + continue + } + + select { + case outNegConf <- depth: + case <-ctx.Done(): + return + } + + continue + + default: + } + } + + select { + case depth, ok := <-reorgDepth: + if !ok { + reorgDepth = nil + + continue + } + + select { + case outNegConf <- depth: + case <-ctx.Done(): + return + } + + case c, ok := <-confChan: + if !ok { + confChan = nil + + continue + } + + select { + case outConfirmed <- c: + case <-ctx.Done(): + return + } + + case <-ctx.Done(): + return + } + } +} + // RegisterSpendNtfn registers for spend notifications using lndclient's // ChainNotifier. func (n *LndClientChainNotifier) RegisterSpendNtfn(outpoint *wire.OutPoint, pkScript []byte, heightHint uint32) (*chainntnfs.SpendEvent, error) { + // Ask lndclient to keep the spend stream alive past the first + // Spend event so it can forward reorg pings and the terminal Done + // signal. Without these the stream is torn down after the first + // delivery and any later spend reorg is silently dropped at the gRPC + // layer. The spend notifier does not track a reorg depth, so the bare + // WithReOrgChan signal is sufficient on this path. + reorgPing := make(chan struct{}, 1) + doneChan := make(chan struct{}, 1) + ctx, cancel := context.WithCancel(context.Background()) - spendChan, errChan, err := n.cfg.LND.ChainNotifier.RegisterSpendNtfn( - ctx, outpoint, pkScript, int32(heightHint), - ) - if err != nil { + + // Run the registration in a goroutine with a timeout to prevent + // hanging when LND is slow under block load, mirroring the conf path. + type regResult struct { + spendChan chan *chainntnfs.SpendDetail + errChan chan error + err error + } + + resultCh := make(chan regResult, 1) + go func() { + sc, ec, err := n.cfg.LND.ChainNotifier.RegisterSpendNtfn( + ctx, outpoint, pkScript, int32(heightHint), + lndclient.WithReOrgChan(reorgPing), + lndclient.WithDoneChan(doneChan), + ) + resultCh <- regResult{sc, ec, err} + }() + + var spendChan chan *chainntnfs.SpendDetail + var errChan chan error + + select { + case r := <-resultCh: + if r.err != nil { + cancel() + + return nil, fmt.Errorf("register spend: %w", r.err) + } + + spendChan = r.spendChan + errChan = r.errChan + + case <-time.After(lndRegistrationTimeout): cancel() - return nil, fmt.Errorf("register spend: %w", err) + return nil, fmt.Errorf("register spend timed out after %s", + lndRegistrationTimeout) } go func() { @@ -236,14 +403,81 @@ func (n *LndClientChainNotifier) RegisterSpendNtfn(outpoint *wire.OutPoint, } }() + // Forward the spend lifecycle through a SINGLE goroutine so the reorg + // ping and the (re-)spend reach the downstream chainntnfs channels in + // lndclient's emission order, for the same reason as the conf path: the + // two-channel split (spendChan and reorgPing) would otherwise let the + // downstream SpendActor's select consume a re-Spend before the Reorged + // and strand the watch. Draining a pending reorg with priority and + // using blocking hand-offs makes the SpendActor observe lndclient's + // order. + orderedSpend := make(chan *chainntnfs.SpendDetail, 1) + reorgChan := make(chan struct{}, 1) + go func() { + defer close(orderedSpend) + defer close(reorgChan) + + forwardOrderedSpendReorg( + ctx, reorgPing, spendChan, orderedSpend, reorgChan, + ) + }() + + // Done is now driven by lnd's "past reorg-safety depth" signal, which + // lndclient surfaces via WithDoneChan. It is a terminal, post-finality + // signal so it is forwarded directly rather than through the ordered + // spend forwarder. return &chainntnfs.SpendEvent{ - Spend: spendChan, - Reorg: make(chan struct{}, 1), - Done: make(chan struct{}, 1), + Spend: orderedSpend, + Reorg: reorgChan, + Done: doneChan, Cancel: cancel, }, nil } +// forwardOrderedSpendReorg is the spend-path analogue of forwardOrderedReorg: +// it copies spends and reorg pings from lndclient's two source channels onto +// the downstream Spend / Reorg channels in the order this single goroutine +// observes them, without biasing either channel. The authoritative lifecycle +// ordering is re-established downstream by the per-registration sequence number +// the LNDBackend forwarder stamps (see chainsource.SpendDetail.Seq). +func forwardOrderedSpendReorg(ctx context.Context, reorgPing <-chan struct{}, + spendChan <-chan *chainntnfs.SpendDetail, + outSpend chan<- *chainntnfs.SpendDetail, outReorg chan<- struct{}) { + + for spendChan != nil || reorgPing != nil { + select { + case _, ok := <-reorgPing: + if !ok { + reorgPing = nil + + continue + } + + select { + case outReorg <- struct{}{}: + case <-ctx.Done(): + return + } + + case sp, ok := <-spendChan: + if !ok { + spendChan = nil + + continue + } + + select { + case outSpend <- sp: + case <-ctx.Done(): + return + } + + case <-ctx.Done(): + return + } + } +} + // RegisterBlockEpochNtfn registers for block epoch notifications using // lndclient's ChainNotifier. func (n *LndClientChainNotifier) RegisterBlockEpochNtfn( diff --git a/chainbackends/lndclient_ordering_test.go b/chainbackends/lndclient_ordering_test.go new file mode 100644 index 000000000..d6e0aab5b --- /dev/null +++ b/chainbackends/lndclient_ordering_test.go @@ -0,0 +1,136 @@ +package chainbackends + +import ( + "testing" + "time" + + "github.com/lightningnetwork/lnd/chainntnfs" + "github.com/stretchr/testify/require" +) + +// orderingWaitTimeout bounds each step of the ordered-forwarder tests so a +// hang (a forwarder that never delivers) surfaces as a fast failure. +const orderingWaitTimeout = 2 * time.Second + +// TestForwardOrderedReorgPrioritizesReorg pins the cross-channel ordering +// guarantee of the conf-path forwarder: when a reorg ping and a confirmation +// are BOTH already pending (the exact race the two-goroutine split created), +// the forwarder must deliver the NegativeConf (reorg) before the Confirmed, +// preserving lndclient's reorg-before-reconfirmation order so the downstream +// ConfActor cannot reset confirmHeight after a re-confirmation and strand the +// watch. +func TestForwardOrderedReorgPrioritizesReorg(t *testing.T) { + t.Parallel() + + ctx := t.Context() + + // Both sources ready up front: lndclient wrote the reorg before the + // replacement confirmation, and both are buffered by the time the + // forwarder runs. The reorg carries its depth (3). + reorgDepth := make(chan int32, 1) + confChan := make(chan *chainntnfs.TxConfirmation, 1) + reorgDepth <- 3 + confChan <- &chainntnfs.TxConfirmation{BlockHeight: 100} + + // The downstream channels are unbuffered so the forwarder cannot run + // ahead: it must block handing off the reorg before it can forward the + // confirmation, exactly as the single in-order consumer it feeds in + // production behaves. Buffering these would let both hand-offs complete + // before the assertion runs and make the ordering check race. + outConfirmed := make(chan *chainntnfs.TxConfirmation) + outNegConf := make(chan int32) + + go forwardOrderedReorg( + ctx, reorgDepth, confChan, outConfirmed, outNegConf, + ) + + // The reorg must arrive first, carrying the forwarded depth. + select { + case depth := <-outNegConf: + require.EqualValues(t, 3, depth) + + case <-outConfirmed: + t.Fatal("confirmation delivered before reorg") + + case <-time.After(orderingWaitTimeout): + t.Fatal("timeout waiting for the reorg") + } + + // Then the confirmation. + select { + case c := <-outConfirmed: + require.EqualValues(t, 100, c.BlockHeight) + + case <-time.After(orderingWaitTimeout): + t.Fatal("timeout waiting for the confirmation") + } +} + +// TestForwardOrderedSpendReorgPrioritizesReorg is the spend-path analogue: +// a pending reorg must be delivered on Reorg before a pending re-Spend. +func TestForwardOrderedSpendReorgPrioritizesReorg(t *testing.T) { + t.Parallel() + + ctx := t.Context() + + reorgPing := make(chan struct{}, 1) + spendChan := make(chan *chainntnfs.SpendDetail, 1) + reorgPing <- struct{}{} + spendChan <- &chainntnfs.SpendDetail{SpendingHeight: 100} + + // Unbuffered downstream channels so the forwarder cannot run ahead of + // the in-order consumer; see the conf-path test for the rationale. + outSpend := make(chan *chainntnfs.SpendDetail) + outReorg := make(chan struct{}) + + go forwardOrderedSpendReorg( + ctx, reorgPing, spendChan, outSpend, outReorg, + ) + + select { + case <-outReorg: + case <-outSpend: + t.Fatal("spend delivered before reorg") + + case <-time.After(orderingWaitTimeout): + t.Fatal("timeout waiting for the reorg") + } + + select { + case sp := <-outSpend: + require.EqualValues(t, 100, sp.SpendingHeight) + + case <-time.After(orderingWaitTimeout): + t.Fatal("timeout waiting for the spend") + } +} + +// TestForwardOrderedReorgForwardsConfirmationAlone verifies the common path: +// with no reorg pending, a confirmation is forwarded unchanged. +func TestForwardOrderedReorgForwardsConfirmationAlone(t *testing.T) { + t.Parallel() + + ctx := t.Context() + + reorgDepth := make(chan int32, 1) + confChan := make(chan *chainntnfs.TxConfirmation, 1) + confChan <- &chainntnfs.TxConfirmation{BlockHeight: 42} + + outConfirmed := make(chan *chainntnfs.TxConfirmation, 1) + outNegConf := make(chan int32, 1) + + go forwardOrderedReorg( + ctx, reorgDepth, confChan, outConfirmed, outNegConf, + ) + + select { + case c := <-outConfirmed: + require.EqualValues(t, 42, c.BlockHeight) + + case <-outNegConf: + t.Fatal("unexpected reorg with none pending") + + case <-time.After(orderingWaitTimeout): + t.Fatal("timeout waiting for the confirmation") + } +} diff --git a/chainsource/backend.go b/chainsource/backend.go index 3ef13b54e..7a6acfcaa 100644 --- a/chainsource/backend.go +++ b/chainsource/backend.go @@ -137,12 +137,41 @@ type ChainBackend interface { // ConfRegistration encapsulates the channels and control functions for a // confirmation registration. This mirrors lnd's chainntnfs.ConfirmationEvent // structure but provides a backend-agnostic interface. +// +// The registration is reorg-aware: after a confirmation has been delivered +// on Confirmed, a subsequent reorg that buries the original block can cause +// a fresh send on Reorged. Once the transaction re-confirms on the new +// canonical chain another event arrives on Confirmed. The lifecycle is +// therefore Confirmed -> Reorged -> Confirmed -> ... terminated by either a +// send on Done (the confirmation is past the backend's reorg-safety depth) +// or a caller-driven Cancel. Backends that cannot observe reorgs leave +// Reorged and Done as never-firing channels; callers must not assume either +// will ever fire. type ConfRegistration struct { - // Confirmed is a channel that fires once when the transaction reaches - // the target number of confirmations. The channel is buffered and will - // only send a single event. + // Confirmed fires every time the transaction reaches the target + // number of confirmations on the canonical chain. After a Reorged + // event it may fire again when the transaction re-confirms. The + // channel is buffered. Confirmed <-chan *TxConfirmation + // Reorged fires when a previously delivered confirmation is reorged + // out of the canonical chain. The payload is the backend forwarder's + // monotonic sequence number (see TxConfirmation.Seq) rather than + // reorg depth or block identity, which the lndclient gRPC transport + // cannot preserve; the sequence lets the consumer order this signal + // against confirmations sharing the same sequence space even though + // they arrive on a different channel. Backends that cannot observe + // reorgs leave this channel nil-equivalent (allocated but never + // written to); reading from it is always safe. + Reorged <-chan uint64 + + // Done fires once when the confirmation watch is past the backend's + // reorg-safety depth and will receive no further events. Callers + // must still invoke Cancel to release client-side resources. + // Backends that cannot synthesize a safety-depth signal leave this + // channel nil-equivalent (allocated but never written to). + Done <-chan struct{} + // Cancel is a function that can be called to cancel this registration // and clean up resources. After calling Cancel, no more events will be // sent on any channels. @@ -168,17 +197,54 @@ type TxConfirmation struct { // when the confirmation was registered with IncludeBlock=true. This // matches lnd's chainntnfs behavior. Block *wire.MsgBlock + + // Seq is a per-registration monotonic sequence number stamped by the + // backend forwarder in the order it observed lifecycle events. + // Confirmed and Reorged share one sequence space so the consumer can + // order them even though they arrive on separate channels (a select + // over two ready channels picks at random and cannot recover order). + // The consumer applies an event only when its Seq exceeds the highest + // Seq seen so far, discarding a stale event that lost a cross-channel + // race. Zero means the backend does not stamp sequences (it never + // reorgs, so ordering is moot); such events are always applied. + Seq uint64 } // SpendRegistration encapsulates the channels and control functions for a // spend registration. This mirrors lnd's chainntnfs.SpendEvent structure. +// +// The registration is reorg-aware: after a spend has been delivered on +// Spend, a subsequent reorg that buries the spending block can cause a +// fresh send on Reorged. If the outpoint is then re-spent on the new +// canonical chain (by the same or a different transaction) another event +// arrives on Spend. The lifecycle is therefore Spend -> Reorged -> Spend +// -> ... terminated by either a send on Done (the spend is past the +// backend's reorg-safety depth) or a caller-driven Cancel. Backends that +// cannot observe reorgs leave Reorged and Done as never-firing channels. type SpendRegistration struct { - // Spend is a channel that fires when the monitored outpoint is spent. - // The spending transaction must have at least one confirmation. The - // channel is buffered and will send an event for each spend (though - // typically only one unless reorgs occur). + // Spend fires every time the monitored outpoint is spent on the + // canonical chain. After a Reorged event it may fire again when a + // new spender confirms. The channel is buffered. Spend <-chan *SpendDetail + // Reorged fires when a previously delivered spend is reorged out of + // the canonical chain. The payload is the backend forwarder's + // monotonic sequence number (see SpendDetail.Seq) rather than reorg + // depth or block identity, which the lndclient gRPC transport cannot + // preserve; the sequence lets the consumer order this signal against + // spends sharing the same sequence space even though they arrive on a + // different channel. Backends that cannot observe spend reorgs leave + // this channel nil-equivalent (allocated but never written to); + // reading from it is always safe. + Reorged <-chan uint64 + + // Done fires once when the spend watch is past the backend's + // reorg-safety depth and will receive no further events. Callers + // must still invoke Cancel to release client-side resources. + // Backends that cannot synthesize a safety-depth signal leave this + // channel nil-equivalent (allocated but never written to). + Done <-chan struct{} + // Cancel is a function that can be called to cancel this registration // and clean up resources. Cancel func() @@ -203,6 +269,17 @@ type SpendDetail struct { // SpendingHeight is the block height where the spending transaction // was confirmed. SpendingHeight int32 + + // Seq is a per-registration monotonic sequence number stamped by the + // backend forwarder in the order it observed lifecycle events. Spend + // and Reorged share one sequence space so the consumer can order them + // even though they arrive on separate channels (a select over two + // ready channels picks at random and cannot recover order). The + // consumer applies an event only when its Seq exceeds the highest Seq + // seen so far, discarding a stale event that lost a cross-channel + // race. Zero means the backend does not stamp sequences (it never + // reorgs, so ordering is moot); such events are always applied. + Seq uint64 } // BlockRegistration encapsulates the channels and control functions for a diff --git a/chainsource/chainsource.go b/chainsource/chainsource.go index 44c6e7ce6..42f0cdde4 100644 --- a/chainsource/chainsource.go +++ b/chainsource/chainsource.go @@ -18,6 +18,16 @@ const ( // allows for buffering up to 10 blocks in transit, which should cover // normal block arrival patterns without blocking the backend. epochChannelSize = 10 + + // DefaultFinalityDepth is the conventional Bitcoin reorg-safety + // depth. After this many inclusive confirmations the ConfActor + // and SpendActor synthesize their Done events when the backend + // transport (notably lndclient over gRPC) does not surface one + // of its own. Six is the value the wider Lightning stack treats + // as final for the purposes of channel funding / settlement, so + // using it here keeps the unroll subsystem's finality threshold + // aligned with the rest of the daemon's chain assumptions. + DefaultFinalityDepth uint32 = 6 ) // ChainSourceConfig holds configuration for ChainSourceActor. @@ -32,6 +42,14 @@ type ChainSourceConfig struct { // falls back to extracting a logger from context via LoggerFromContext, // or uses btclog.Disabled if no logger is found. Log fn.Option[btclog.Logger] + + // FinalityDepth is forwarded to each spawned sub-actor as the + // number of confirmations past the first observed positive event + // that the actor uses to synthesize a Done signal when the backend + // transport (notably lndclient over gRPC) cannot deliver one. Zero + // disables height-based finality synthesis. See + // ConfActorConfig.FinalityDepth / SpendActorConfig.FinalityDepth. + FinalityDepth uint32 } // WithLogger returns a new config with the given logger set. @@ -318,8 +336,9 @@ func (a *ChainSourceActor) handleRegisterConf(ctx context.Context, ) confCfg := ConfActorConfig{ - Backend: a.cfg.Backend, - Log: fn.Some(a.logger(ctx)), + Backend: a.cfg.Backend, + Log: fn.Some(a.logger(ctx)), + FinalityDepth: a.cfg.FinalityDepth, } confActor := NewConfActor(confCfg) actorRef := serviceKey.Spawn(a.cfg.System, actorID, confActor) @@ -353,8 +372,9 @@ func (a *ChainSourceActor) handleRegisterSpend(ctx context.Context, ) spendCfg := SpendActorConfig{ - Backend: a.cfg.Backend, - Log: fn.Some(a.logger(ctx)), + Backend: a.cfg.Backend, + Log: fn.Some(a.logger(ctx)), + FinalityDepth: a.cfg.FinalityDepth, } spendActor := NewSpendActor(spendCfg) actorRef := serviceKey.Spawn(a.cfg.System, actorID, spendActor) diff --git a/chainsource/chainsource_test.go b/chainsource/chainsource_test.go index 822f4cb92..92b0d89d5 100644 --- a/chainsource/chainsource_test.go +++ b/chainsource/chainsource_test.go @@ -3,6 +3,7 @@ package chainsource import ( "context" "errors" + "sync/atomic" "testing" "time" @@ -17,12 +18,23 @@ import ( // mockBackend implements ChainBackend for testing. type mockBackend struct { - confChan chan *TxConfirmation - spendChan chan *SpendDetail - epochChan chan *BlockEpoch + confChan chan *TxConfirmation + confReorgedChan chan uint64 + confDoneChan chan struct{} + spendChan chan *SpendDetail + spendReorgedChan chan uint64 + spendDoneChan chan struct{} + epochChan chan *BlockEpoch epochCancel chan struct{} + // confCancelled / spendCancelled count Cancel invocations on the + // most recently issued registration. The reorg-aware lifecycle + // tests rely on these to assert that the actor released the + // registration after a Done event. + confCancelled atomic.Int32 + spendCancelled atomic.Int32 + feeRate btcutil.Amount bestHeight int32 bestHash chainhash.Hash @@ -42,12 +54,16 @@ type reconnectBlockBackend struct { // newMockBackend creates a new mock backend for testing. func newMockBackend() *mockBackend { return &mockBackend{ - confChan: make(chan *TxConfirmation, 1), - spendChan: make(chan *SpendDetail, 1), - epochChan: make(chan *BlockEpoch, 10), - epochCancel: make(chan struct{}, 10), - feeRate: 1000, - bestHeight: 100, + confChan: make(chan *TxConfirmation, 1), + confReorgedChan: make(chan uint64, 1), + confDoneChan: make(chan struct{}, 1), + spendChan: make(chan *SpendDetail, 1), + spendReorgedChan: make(chan uint64, 1), + spendDoneChan: make(chan struct{}, 1), + epochChan: make(chan *BlockEpoch, 10), + epochCancel: make(chan struct{}, 10), + feeRate: 1000, + bestHeight: 100, } } @@ -208,7 +224,11 @@ func (m *mockBackend) RegisterConf(ctx context.Context, txid *chainhash.Hash, return &ConfRegistration{ Confirmed: m.confChan, - Cancel: func() {}, + Reorged: m.confReorgedChan, + Done: m.confDoneChan, + Cancel: func() { + m.confCancelled.Add(1) + }, }, nil } @@ -217,8 +237,12 @@ func (m *mockBackend) RegisterSpend(ctx context.Context, *SpendRegistration, error) { return &SpendRegistration{ - Spend: m.spendChan, - Cancel: func() {}, + Spend: m.spendChan, + Reorged: m.spendReorgedChan, + Done: m.spendDoneChan, + Cancel: func() { + m.spendCancelled.Add(1) + }, }, nil } diff --git a/chainsource/conf_actor.go b/chainsource/conf_actor.go index d5d1ecfaa..e9e1b6852 100644 --- a/chainsource/conf_actor.go +++ b/chainsource/conf_actor.go @@ -22,6 +22,20 @@ type ConfActorConfig struct { // falls back to extracting a logger from context via LoggerFromContext, // or uses btclog.Disabled if no logger is found. Log fn.Option[btclog.Logger] + + // FinalityDepth is the number of confirmations past the first + // observed Confirmed event that the actor uses to synthesize a Done + // signal when the backend cannot deliver one. Zero disables + // height-based finality synthesis entirely; in that case the actor + // only fires ConfDoneEvent when the backend's own Done channel + // fires (e.g. an in-process lnd notifier). Non-zero values close + // the lndclient transport gap, where the gRPC layer does not + // surface lnd's internal "past reorg-safety depth" signal. + // + // The depth is counted inclusively (a tx confirmed at height H is + // at depth 1; height-based finality fires once the actor observes + // a block at H + FinalityDepth - 1). The conventional choice is 6. + FinalityDepth uint32 } // WithLogger returns a new config with the given logger set. @@ -67,9 +81,28 @@ type ConfActor struct { // mode. notifyActor fn.Option[actor.TellOnlyRef[ConfirmationEvent]] + // notifyReorged receives negative confirmation events in Actor mode. + notifyReorged fn.Option[actor.TellOnlyRef[ConfReorgedEvent]] + + // notifyDone receives finality events in Actor mode. + notifyDone fn.Option[actor.TellOnlyRef[ConfDoneEvent]] + // registration is the backend registration for this watch. registration *ConfRegistration + // blockReg is the block-epoch subscription used by height-based + // finality synthesis. Allocated lazily after the first Confirmed + // event when FinalityDepth > 0 and the registration is reorg-aware, + // torn down when the actor exits. + blockReg *BlockRegistration + + // confirmHeight records the block height the most recent + // Confirmed event arrived at. Used by the height-based finality + // synthesizer to compute current depth. Zero means there is no + // active confirmation to count from (either we have not yet seen + // one, or the last one was reorged out). + confirmHeight int32 + // ctx is the actor's internal context for cancellation, created from // context.Background() to ensure it outlives any request context. //nolint:containedctx @@ -143,6 +176,13 @@ func (a *ConfActor) handleRegisterConf(actorCtx context.Context, "than zero"), ) } + if req.NotifyActor.IsNone() && + (req.NotifyReorged.IsSome() || req.NotifyDone.IsSome()) { + return fn.Err[ConfResp]( + fmt.Errorf("confirmation reorg/done notifications " + + "require actor-mode NotifyActor"), + ) + } a.txid = req.Txid a.pkScript = req.PkScript @@ -150,6 +190,8 @@ func (a *ConfActor) handleRegisterConf(actorCtx context.Context, a.heightHint = req.HeightHint a.includeBlock = req.IncludeBlock a.notifyActor = req.NotifyActor + a.notifyReorged = req.NotifyReorged + a.notifyDone = req.NotifyDone // We're either in future or iterator mode, set the promise // accordingly. @@ -197,8 +239,8 @@ func (a *ConfActor) handleRegisterConf(actorCtx context.Context, } // monitorConfirmation runs in a background goroutine and waits for the target -// confirmation count to be reached. When reached, it delivers the event and -// exits. +// confirmation count to be reached. Legacy watches exit after confirmation, +// while reorg-aware actor-mode watches remain alive for reorg and done events. func (a *ConfActor) monitorConfirmation() { defer a.wg.Done() defer a.cancel() @@ -215,50 +257,324 @@ func (a *ConfActor) monitorConfirmation() { if a.registration != nil { a.registration.Cancel() } + if a.blockReg != nil { + a.blockReg.Cancel() + } }() - select { - case confDetails, ok := <-a.registration.Confirmed: - if !ok || confDetails == nil { - log.WarnS(a.ctx, "Confirmation subscription closed", - fmt.Errorf("channel closed or nil details"), + var lastEvent *ConfirmationEvent + reorgAware := a.notifyReorged.IsSome() || a.notifyDone.IsSome() + + // lastSeq is the highest backend forwarder sequence applied so far. + // Confirmed and Reorged signals arrive on separate channels and a + // select cannot order two ready channels, so we order them by the + // shared sequence instead: an event whose Seq does not exceed lastSeq + // lost a cross-channel race to a newer signal and is discarded. This + // makes the actor's view correct regardless of delivery interleaving + // — both reorg-then-reconfirm and confirm-then-reorg resolve to the + // highest-sequence outcome. Seq 0 means the backend does not stamp + // sequences (it never reorgs); those events are always applied. + var lastSeq uint64 + + // blockEpochs is rebound when height-based finality synthesis + // arms a block subscription. Until then a nil channel keeps the + // select arm parked. + var blockEpochs <-chan *BlockEpoch + + // blockRegCh hands a finality block subscription from the off-loop + // arming goroutine back to this loop; arming guards against launching + // more than one armer at a time. See armFinalityAsync for why arming + // runs off the select loop. + blockRegCh := make(chan finalityArmResult) + var arming bool + + for { + select { + case confDetails, ok := <-a.registration.Confirmed: + if !ok || confDetails == nil { + log.WarnS( + a.ctx, + "Confirmation subscription closed", + fmt.Errorf("channel closed or nil "+ + "details"), + ) + a.failConfirmation( + fmt.Errorf("confirmation " + + "subscription closed"), + ) + + return + } + + // Discard a confirmation that lost a cross-channel race + // to a newer reorg: an event whose sequence does not + // exceed the highest applied is stale. + if confDetails.Seq != 0 && confDetails.Seq <= lastSeq { + continue + } + if confDetails.Seq > lastSeq { + lastSeq = confDetails.Seq + } + + log.InfoS(a.ctx, "Received confirmation from backend", + "block_height", confDetails.BlockHeight, + "block_hash", confDetails.BlockHash, + ) + + event, err := buildConfirmationEvent(confDetails, a) + if err != nil { + log.WarnS( + a.ctx, + "Failed to build confirmation event", + err, + ) + a.failConfirmation(err) + + return + } + + log.InfoS(a.ctx, "Delivering confirmation event", + "txid", event.Txid, + "block_height", event.BlockHeight, ) - a.failConfirmation( - fmt.Errorf("confirmation subscription closed"), + a.deliverConfirmation(event) + lastEvent = &event + a.confirmHeight = event.BlockHeight + if !reorgAware || a.promise.IsSome() { + return + } + + // Arm height-based finality synthesis on the first + // confirmation if requested, off the select loop so + // the bounded RegisterBlocks retries cannot stall + // delivery of Reorged/Done/ctx.Done on this watch. A + // nil blockEpochs channel keeps the synthesis arm + // parked until the registration is handed back on + // blockRegCh. + if a.cfg.FinalityDepth > 0 && a.blockReg == nil && + !arming { + + arming = true + a.armFinalityAsync(blockRegCh, log) + } + + case armed := <-blockRegCh: + // Finality arming completed. Clear the flag; a nil reg + // only happens when the watch context was cancelled + // (arming otherwise retries until it succeeds). + arming = false + if armed.reg == nil { + continue + } + a.blockReg = armed.reg + blockEpochs = armed.reg.Epochs + + // The block-epoch subscription only delivers FUTURE + // epochs, but the confirmation that armed it may + // already be buried past FinalityDepth (it confirmed + // several blocks ago, or we re-armed after a restart). + // Use the tip observed at arm time to synthesize Done + // at once rather than hang until a fresh block is + // mined. The confirmHeight==0 / FinalityDepth==0 guards + // mirror the epoch handler below. + if a.confirmHeight == 0 || a.cfg.FinalityDepth == 0 { + continue + } + if armed.height-a.confirmHeight+1 < + int32(a.cfg.FinalityDepth) { + + continue + } + + log.InfoS(a.ctx, "Synthesizing confirmation done on arm "+ + "from height-based safety depth", + "confirm_height", a.confirmHeight, + "current_height", armed.height, + "finality_depth", int(a.cfg.FinalityDepth), ) + a.deliverConfDone(lastEvent) return - } - log.InfoS(a.ctx, "Received confirmation from backend", - "block_height", confDetails.BlockHeight, - "block_hash", confDetails.BlockHash, - ) + case seq, ok := <-a.registration.Reorged: + if !ok { + a.registration.Reorged = nil + continue + } + + // Discard a stale reorg that lost a cross-channel race + // to a newer confirmation. + if seq != 0 && seq <= lastSeq { + continue + } + if seq > lastSeq { + lastSeq = seq + } + + a.deliverConfReorged(lastEvent) + + // The previous confirmation is no longer on the + // canonical chain. Clear the cached event so a later + // Done cannot report the reorged-out txid, and reset + // the depth counter so the next re-confirmation starts + // a fresh window. + lastEvent = nil + a.confirmHeight = 0 + + case _, ok := <-a.registration.Done: + if !ok { + a.registration.Done = nil + continue + } + + a.deliverConfDone(lastEvent) - event, err := buildConfirmationEvent(confDetails, a) - if err != nil { - log.WarnS(a.ctx, "Failed to build confirmation event", - err, + return + + case epoch, ok := <-blockEpochs: + if !ok || epoch == nil { + blockEpochs = nil + continue + } + + // Coalesce any epochs already queued behind this one + // and evaluate finality against the most recent + // height only. With rapid-fire blocks (or a backend + // that re-delivers historical epochs) the channel can + // hold several epochs at once; processing them one per + // loop iteration would re-check the same monotonic + // Done condition repeatedly and risk synthesizing + // against a stale height. If the channel closed during + // the drain, park it so we stop selecting on it. + var closed bool + epoch, closed = drainToLatestEpoch(blockEpochs, epoch) + if closed { + blockEpochs = nil + } + + // The confirmHeight==0 guard is load-bearing: a reorg + // resets confirmHeight to 0 (see the Reorged case + // above), and a fresh epoch on the new tip would + // otherwise produce a negative depth (epoch.Height - 0 + // wraps to a large value); the depth comparison is + // meaningful only after a confirmation arms + // confirmHeight. FinalityDepth==0 disables synthesis + // entirely. + if a.confirmHeight == 0 || + a.cfg.FinalityDepth == 0 { + + continue + } + + depth := epoch.Height - a.confirmHeight + 1 + if depth < int32(a.cfg.FinalityDepth) { + continue + } + + log.InfoS(a.ctx, "Synthesizing confirmation done "+ + "from height-based safety depth", + "confirm_height", a.confirmHeight, + "current_height", epoch.Height, + "finality_depth", int(a.cfg.FinalityDepth), ) - a.failConfirmation(err) + + // Finality is terminal for the whole watch: deliver + // Done (a no-op when only NotifyReorged was set) and + // stop. A reorg-only watch therefore intentionally + // stops receiving reorg events once the confirmation is + // buried FinalityDepth deep — past that depth a reorg + // is beyond the safety threshold the watch was created + // to cover, so there is nothing left to observe. + a.deliverConfDone(lastEvent) return - } - log.InfoS(a.ctx, "Delivering confirmation event", - "txid", event.Txid, - "block_height", event.BlockHeight, - ) - a.deliverConfirmation(event) + case <-a.ctx.Done(): + log.InfoS(a.ctx, "ConfActor context cancelled") + a.failConfirmation(a.ctx.Err()) - case <-a.ctx.Done(): - log.InfoS(a.ctx, "ConfActor context cancelled") - a.failConfirmation(a.ctx.Err()) + return + } + } +} + +// drainToLatestEpoch non-blockingly consumes any block epochs already +// queued on ch and returns the most recent non-nil epoch, starting from +// cur. With rapid-fire blocks (or a backend that re-delivers historical +// epochs) several epochs can sit in the channel at once; height-based +// finality synthesis depends only on the highest observed height, so +// collapsing the backlog to the newest epoch avoids re-evaluating the +// same monotonic Done condition once per stale epoch. The returned bool +// reports whether the channel was observed closed during the drain so the +// caller can park its receive on a nil channel. +func drainToLatestEpoch(ch <-chan *BlockEpoch, + cur *BlockEpoch) (*BlockEpoch, bool) { + + latest := cur + for { + select { + case e, ok := <-ch: + if !ok { + return latest, true + } + if e != nil { + latest = e + } + + default: + return latest, false + } } } // deliverConfirmation sends the confirmation event to the subscriber and // completes the promise (Future mode) or sends to the actor (Actor mode). +// armFinalityAsync registers a block-epoch subscription for height-based +// finality synthesis off the actor's select loop. registerBlocksForFinality +// retries with a bounded backoff that can run for tens of seconds; doing it +// inline would block delivery of Reorged/Done/ctx.Done on this watch for the +// whole window. The registration (or nil on failure) is handed back on regCh, +// or cancelled if the actor exits before the loop reads it. The goroutine is +// tracked by the actor's wait group so Stop drains it. +func (a *ConfActor) armFinalityAsync(regCh chan<- finalityArmResult, + log btclog.Logger) { + + a.wg.Go(func() { + reg, err := registerBlocksForFinality(a.ctx, a.cfg.Backend, log) + if err != nil { + log.WarnS(a.ctx, "Giving up on height-based finality "+ + "synthesis; conf sub-actor will rely on "+ + "backend Done", err) + reg = nil + } + + // Capture the tip at arm time so the loop can finalize + // immediately when the arming confirmation is already buried + // past FinalityDepth (the block-epoch sub only delivers future + // epochs). A read failure is non-fatal: height stays zero and + // the loop falls back to waiting for the next epoch. + var height int32 + if reg != nil { + h, _, hErr := a.cfg.Backend.BestBlock(a.ctx) + if hErr != nil { + log.WarnS(a.ctx, "Failed to read best height "+ + "for on-arm finality check; will wait "+ + "for next epoch", hErr) + } else { + height = h + } + } + + select { + case regCh <- finalityArmResult{reg: reg, height: height}: + case <-a.ctx.Done(): + if reg != nil { + reg.Cancel() + } + } + }) +} + func (a *ConfActor) deliverConfirmation(event ConfirmationEvent) { a.promise.WhenSome(func(p actor.Promise[ConfirmationEvent]) { p.Complete(fn.Ok(event)) @@ -272,6 +588,53 @@ func (a *ConfActor) deliverConfirmation(event ConfirmationEvent) { }) } +// deliverConfReorged sends a reorg event to actor-mode subscribers. The +// correlation Txid is the registration's configured txid when set, since +// that is the identifier the caller asked us to watch; pkScript-only +// watches fall back to the txid carried on the most recent positive +// ConfirmationEvent. +func (a *ConfActor) deliverConfReorged(lastEvent *ConfirmationEvent) { + var event ConfReorgedEvent + switch { + case a.txid != nil: + event.Txid = *a.txid + + case lastEvent != nil: + event.Txid = lastEvent.Txid + } + + a.notifyReorged.WhenSome(func(ref actor.TellOnlyRef[ConfReorgedEvent]) { + log := a.logger(a.ctx) + if err := ref.Tell(a.ctx, event); err != nil { + log.WarnS(a.ctx, "Failed to deliver confirmation reorg", + err, + ) + } + }) +} + +// deliverConfDone sends a confirmation finality event to actor-mode +// subscribers. Txid follows the same precedence as deliverConfReorged. +func (a *ConfActor) deliverConfDone(lastEvent *ConfirmationEvent) { + var event ConfDoneEvent + switch { + case a.txid != nil: + event.Txid = *a.txid + + case lastEvent != nil: + event.Txid = lastEvent.Txid + } + + a.notifyDone.WhenSome(func(ref actor.TellOnlyRef[ConfDoneEvent]) { + log := a.logger(a.ctx) + if err := ref.Tell(a.ctx, event); err != nil { + log.WarnS(a.ctx, "Failed to deliver confirmation done", + err, + ) + } + }) +} + // failConfirmation completes the promise with an error (Future mode) or does // nothing (Actor mode - errors are not delivered in async mode). func (a *ConfActor) failConfirmation(err error) { diff --git a/chainsource/epoch_drain_test.go b/chainsource/epoch_drain_test.go new file mode 100644 index 000000000..9a439f84c --- /dev/null +++ b/chainsource/epoch_drain_test.go @@ -0,0 +1,76 @@ +package chainsource + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestDrainToLatestEpochCoalesces verifies that a backlog of queued block +// epochs collapses to the most recent one, so finality synthesis evaluates +// the highest observed height rather than re-checking once per stale epoch. +func TestDrainToLatestEpochCoalesces(t *testing.T) { + t.Parallel() + + ch := make(chan *BlockEpoch, 8) + cur := &BlockEpoch{Height: 100} + + // Queue several newer epochs behind the one already dequeued. + for h := int32(101); h <= 105; h++ { + ch <- &BlockEpoch{Height: h} + } + + got, closed := drainToLatestEpoch(ch, cur) + require.False(t, closed) + require.NotNil(t, got) + require.Equal(t, int32(105), got.Height) + + // The channel must be fully drained afterwards. + require.Len(t, ch, 0) +} + +// TestDrainToLatestEpochEmpty verifies that with nothing queued the helper +// returns the current epoch unchanged and reports the channel open. +func TestDrainToLatestEpochEmpty(t *testing.T) { + t.Parallel() + + ch := make(chan *BlockEpoch, 4) + cur := &BlockEpoch{Height: 42} + + got, closed := drainToLatestEpoch(ch, cur) + require.False(t, closed) + require.Same(t, cur, got) +} + +// TestDrainToLatestEpochSkipsNil verifies that nil epochs in the backlog are +// ignored: the most recent non-nil epoch wins. +func TestDrainToLatestEpochSkipsNil(t *testing.T) { + t.Parallel() + + ch := make(chan *BlockEpoch, 4) + cur := &BlockEpoch{Height: 10} + ch <- &BlockEpoch{Height: 11} + ch <- nil + + got, closed := drainToLatestEpoch(ch, cur) + require.False(t, closed) + require.NotNil(t, got) + require.Equal(t, int32(11), got.Height) +} + +// TestDrainToLatestEpochClosed verifies that a closed channel is reported so +// the caller can park its receive, while still returning the latest epoch +// observed before the close. +func TestDrainToLatestEpochClosed(t *testing.T) { + t.Parallel() + + ch := make(chan *BlockEpoch, 4) + cur := &BlockEpoch{Height: 7} + ch <- &BlockEpoch{Height: 8} + close(ch) + + got, closed := drainToLatestEpoch(ch, cur) + require.True(t, closed) + require.NotNil(t, got) + require.Equal(t, int32(8), got.Height) +} diff --git a/chainsource/finality.go b/chainsource/finality.go new file mode 100644 index 000000000..debed8eb5 --- /dev/null +++ b/chainsource/finality.go @@ -0,0 +1,124 @@ +package chainsource + +import ( + "context" + "time" + + "github.com/btcsuite/btclog/v2" +) + +const ( + // finalityArmInitialBackoff is the first delay between RegisterBlocks + // attempts when arming the block-epoch subscription that drives + // height-based finality synthesis. It doubles on each failure up to + // finalityArmMaxBackoff. + finalityArmInitialBackoff = 100 * time.Millisecond + + // finalityArmMaxBackoff caps the retry delay. Arming is retried + // indefinitely (until the watch's context is cancelled) rather than + // abandoned after a fixed count: for gRPC lndclient and lwwallet, + // height synthesis is the ONLY source of the terminal Done, so giving + // up would strand the round/exit in ProvisionallyConfirmed forever + // (a single-confirmation tx has no later event to trigger a re-arm). + // A brief backend outage at the arming moment therefore only delays + // finality, never permanently disables it. 30s stops a genuinely-down + // backend from spinning while bounding recovery latency once it heals. + finalityArmMaxBackoff = 30 * time.Second + + // finalityArmEscalateAfter is the consecutive-failure count past which + // the retry log escalates to an operator-visible "finality stalled" + // warning, so a persistently unarmable watch is detectable rather than + // silently stuck in provisional. + finalityArmEscalateAfter = 5 +) + +// finalityArmResult is handed from a conf/spend sub-actor's off-loop finality +// arming goroutine back to its select loop. reg is the block-epoch +// subscription (nil only when the watch context was cancelled). height is the +// best chain height observed at arm time (zero if it could not be read); the +// loop uses it to synthesize Done immediately when the arming +// confirmation/spend is already buried past FinalityDepth, rather than +// hanging until a fresh block epoch arrives (the subscription only delivers +// FUTURE epochs). +type finalityArmResult struct { + reg *BlockRegistration + height int32 +} + +// registerBlocksForFinality registers a block-epoch subscription used +// to synthesize a Done signal at FinalityDepth past an observed +// confirmation or spend. It retries RegisterBlocks indefinitely with a +// capped exponential backoff (until the passed context is cancelled) +// because finality synthesis is the only Done source for backends that +// do not write the upstream Done channel (notably lndclient over gRPC and +// lwwallet). Abandoning the arm after a fixed number of attempts would +// leak the per-watch sub-actor AND strand the round/exit in +// ProvisionallyConfirmed forever if the backend merely hiccups at the +// arming moment: a single-confirmation tx has no later event to trigger a +// re-arm, so the watch would only recover on a daemon restart. +// +// The retries run in a dedicated arming goroutine (not the sub-actor's +// select loop), so blocking here is safe: more confirmation/spend events +// on this specific watch are not expected during the retry window (we +// already consumed the one that triggered the arm), and ctx cancellation +// breaks out promptly. The goroutine is bounded by the watch's lifetime. +// +// The passed ctx MUST be the sub-actor's long-lived context, and it is +// handed to RegisterBlocks unwrapped: for in-process backends the +// block-epoch forwarder goroutine is tied to the ctx it receives, so +// bounding each attempt with a cancellable child ctx (and cancelling it +// once the call returns) would tear the subscription down the instant it +// was armed — starving finality synthesis of the very epochs it needs. +// +// Returns the registration on success, or a non-nil error only when the +// context is cancelled (the watch is shutting down). +func registerBlocksForFinality(ctx context.Context, backend ChainBackend, + log btclog.Logger) (*BlockRegistration, error) { + + backoff := finalityArmInitialBackoff + for attempt := 1; ; attempt++ { + reg, err := backend.RegisterBlocks(ctx) + if err == nil { + if attempt > 1 { + log.InfoS(ctx, "Finality block subscription "+ + "armed after retries", + "attempts", attempt, + ) + } + + return reg, nil + } + + // Height synthesis is the only finality source for gRPC + // lndclient / lwwallet, so a persistent arming failure stalls + // the round/exit in provisional. Escalate the log once retries + // pass finalityArmEscalateAfter so the stall is + // operator-visible rather than silent; the capped backoff + // throttles it. + if attempt >= finalityArmEscalateAfter { + log.WarnS(ctx, "Finality block subscription arming "+ + "persistently failing; round/exit finality is "+ + "stalled until the backend recovers", err, + "attempts", attempt, + "backoff", backoff, + ) + } else { + log.WarnS(ctx, "RegisterBlocks for finality synthesis "+ + "failed; retrying", err, + "attempt", attempt, + "backoff", backoff, + ) + } + + select { + case <-time.After(backoff): + case <-ctx.Done(): + return nil, ctx.Err() + } + + backoff *= 2 + if backoff > finalityArmMaxBackoff { + backoff = finalityArmMaxBackoff + } + } +} diff --git a/chainsource/finality_test.go b/chainsource/finality_test.go new file mode 100644 index 000000000..f3101089b --- /dev/null +++ b/chainsource/finality_test.go @@ -0,0 +1,107 @@ +package chainsource + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/btcsuite/btclog/v2" + "github.com/stretchr/testify/require" +) + +// flakyArmBackend fails RegisterBlocks the first failCount times, then +// succeeds. It models a backend that is briefly unavailable at the exact +// moment finality arming is attempted (the failure mode that previously +// stranded a watch once the bounded retry schedule was exhausted). +type flakyArmBackend struct { + *mockBackend + + mu sync.Mutex + failLeft int + attempts int +} + +// RegisterBlocks fails until failLeft is drained, then returns a live +// registration. +func (b *flakyArmBackend) RegisterBlocks(ctx context.Context) ( + *BlockRegistration, error) { + + b.mu.Lock() + defer b.mu.Unlock() + + b.attempts++ + if b.failLeft > 0 { + b.failLeft-- + + return nil, errors.New("backend temporarily unavailable") + } + + return &BlockRegistration{ + Epochs: make(chan *BlockEpoch, 1), + Cancel: func() {}, + }, nil +} + +// attemptCount returns how many RegisterBlocks calls have been made. +func (b *flakyArmBackend) attemptCount() int { + b.mu.Lock() + defer b.mu.Unlock() + + return b.attempts +} + +// TestRegisterBlocksForFinalityRetriesUntilArmed asserts that finality arming +// retries past the old fixed three-attempt cap and eventually arms once a +// transiently-unavailable backend recovers. This is the reorg-safety +// robustness property: for gRPC lndclient / lwwallet, height synthesis is the +// only Done source, so abandoning the arm would strand the round/exit in +// provisional until a daemon restart. failLeft is five (past both the old cap +// and finalityArmEscalateAfter) so the escalation-warning branch is exercised. +func TestRegisterBlocksForFinalityRetriesUntilArmed(t *testing.T) { + t.Parallel() + + backend := &flakyArmBackend{ + mockBackend: newMockBackend(), + failLeft: 5, + } + + reg, err := registerBlocksForFinality( + context.Background(), backend, btclog.Disabled, + ) + require.NoError(t, err, "arming must succeed once the backend recovers") + require.NotNil(t, reg, "a live block registration must be returned") + require.NotNil(t, reg.Epochs) + require.GreaterOrEqual( + t, backend.attemptCount(), 6, + "arming must retry past the old three-attempt cap", + ) +} + +// TestRegisterBlocksForFinalityStopsOnContextCancel asserts that the otherwise +// unbounded arming retry exits promptly when the watch's context is cancelled +// (daemon shutdown), returning the context error rather than looping forever. +func TestRegisterBlocksForFinalityStopsOnContextCancel(t *testing.T) { + t.Parallel() + + // A backend that never recovers, so arming would loop indefinitely + // were it not bounded by the context. + backend := &flakyArmBackend{ + mockBackend: newMockBackend(), + failLeft: 1 << 30, + } + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(150 * time.Millisecond) + cancel() + }() + + reg, err := registerBlocksForFinality(ctx, backend, btclog.Disabled) + require.Error( + t, err, "arming must return once the context is cancelled", + ) + require.ErrorIs(t, err, context.Canceled) + require.Nil(t, reg) +} diff --git a/chainsource/messages.go b/chainsource/messages.go index cd5eefba3..98e15ea91 100644 --- a/chainsource/messages.go +++ b/chainsource/messages.go @@ -261,6 +261,15 @@ type RegisterConfRequest struct { // events will be sent to this actor asynchronously. If None, a Future // is returned in the response for blocking await. NotifyActor fn.Option[actor.TellOnlyRef[ConfirmationEvent]] + + // NotifyReorged is an optional actor reference for negative + // confirmation events. It is only used in async actor mode. + NotifyReorged fn.Option[actor.TellOnlyRef[ConfReorgedEvent]] + + // NotifyDone is an optional actor reference notified when the + // confirmation watch is beyond the backend's reorg tracking horizon. + // It is only used in async actor mode. + NotifyDone fn.Option[actor.TellOnlyRef[ConfDoneEvent]] } // MessageType returns the message type identifier for logging and debugging. @@ -332,6 +341,48 @@ func (m ConfirmationEvent) MessageType() string { return "ConfirmationEvent" } +// ConfReorgedEvent is sent when a previously reported confirmation is +// reorged out of the canonical chain. After receiving this event a consumer +// should consider the prior confirmation no longer valid; if the transaction +// re-confirms on the new canonical chain a fresh ConfirmationEvent will +// follow on the same registration. +// +// No block hash, height, or reorg depth is carried because the lndclient +// gRPC transport does not preserve that information and we do not want to +// expose fields that are always zero in production. Consumers that need to +// invalidate cached block metadata should match on Txid and use the data +// from the most recent ConfirmationEvent they observed on this watch. +type ConfReorgedEvent struct { + actor.BaseMessage + + // Txid is the transaction ID whose confirmation was reorged. This + // matches the Txid carried on the originating ConfirmationEvent. + Txid chainhash.Hash +} + +// MessageType returns the message type identifier for logging and debugging. +func (m ConfReorgedEvent) MessageType() string { + return "ConfReorgedEvent" +} + +// ConfDoneEvent is sent when a confirmation watch has matured past the +// backend's reorg-safety depth and will receive no further events. +// Consumers may use this signal to drop any reorg-recovery bookkeeping they +// were holding for the registration. Not all backends synthesize this +// event; consumers must treat its absence as a normal operating condition +// rather than an error. +type ConfDoneEvent struct { + actor.BaseMessage + + // Txid is the transaction ID whose registration matured. + Txid chainhash.Hash +} + +// MessageType returns the message type identifier for logging and debugging. +func (m ConfDoneEvent) MessageType() string { + return "ConfDoneEvent" +} + // UnregisterConfRequest requests cancellation of a confirmation subscription. // The ChainSource actor uses the fields to construct the service key and // cancel the dedicated actor. @@ -417,6 +468,15 @@ type RegisterSpendRequest struct { // will be sent to this actor asynchronously. If None, a Future is // returned for blocking await. NotifyActor fn.Option[actor.TellOnlyRef[SpendEvent]] + + // NotifyReorged is an optional actor reference for spend reorg events. + // It is only used in async actor mode. + NotifyReorged fn.Option[actor.TellOnlyRef[SpendReorgedEvent]] + + // NotifyDone is an optional actor reference notified when the spend + // watch is beyond the backend's reorg tracking horizon. It is only used + // in async actor mode. + NotifyDone fn.Option[actor.TellOnlyRef[SpendDoneEvent]] } // MessageType returns the message type identifier for logging and debugging. @@ -481,6 +541,45 @@ func (m SpendEvent) MessageType() string { return "SpendEvent" } +// SpendReorgedEvent is sent when a previously reported spend is reorged out +// of the canonical chain. After receiving this event a consumer should +// consider the prior spend no longer valid; if the outpoint is re-spent on +// the new canonical chain a fresh SpendEvent will follow on the same +// registration. +// +// No spending txid, height, or block hash is carried because the lndclient +// gRPC transport does not preserve that information and we do not want to +// expose fields that are always zero in production. Consumers that need to +// invalidate cached spending metadata should match on Outpoint and use the +// data from the most recent SpendEvent they observed on this watch. +type SpendReorgedEvent struct { + actor.BaseMessage + + // Outpoint is the output whose spend was reorged. + Outpoint wire.OutPoint +} + +// MessageType returns the message type identifier for logging and debugging. +func (m SpendReorgedEvent) MessageType() string { + return "SpendReorgedEvent" +} + +// SpendDoneEvent is sent when a spend watch has matured past the backend's +// reorg-safety depth and will receive no further events. Not all backends +// synthesize this event; consumers must treat its absence as a normal +// operating condition rather than an error. +type SpendDoneEvent struct { + actor.BaseMessage + + // Outpoint is the output whose spend registration matured. + Outpoint wire.OutPoint +} + +// MessageType returns the message type identifier for logging and debugging. +func (m SpendDoneEvent) MessageType() string { + return "SpendDoneEvent" +} + // UnregisterSpendRequest requests cancellation of a spend subscription. // The ChainSource actor uses the fields to construct the service key and // cancel the dedicated actor. diff --git a/chainsource/reorg_test.go b/chainsource/reorg_test.go new file mode 100644 index 000000000..d045b8d49 --- /dev/null +++ b/chainsource/reorg_test.go @@ -0,0 +1,735 @@ +package chainsource + +import ( + "testing" + "time" + + "github.com/btcsuite/btcd/chainhash/v2" + "github.com/btcsuite/btcd/wire/v2" + "github.com/lightninglabs/darepo-client/baselib/actor" + "github.com/lightningnetwork/lnd/fn/v2" + "github.com/stretchr/testify/require" +) + +// awaitTimeout is the per-step wait used by the reorg lifecycle tests. It +// is generous enough to absorb scheduling jitter on overloaded CI machines +// but still short enough that a hung actor surfaces as a fast failure. +const awaitTimeout = 5 * time.Second + +// TestConfActorReorgAwareForwardsFullLifecycle drives the full +// Confirmed -> Reorged -> Confirmed -> Done sequence through a reorg-aware +// ConfActor and asserts each event is forwarded to the correct notify ref +// in order, and that the actor releases the backend registration after +// Done. +func TestConfActorReorgAwareForwardsFullLifecycle(t *testing.T) { + t.Parallel() + + backend := newMockBackend() + ctx := t.Context() + confActor := NewConfActor(ConfActorConfig{Backend: backend}) + defer confActor.Stop() + + txHash := chainhash.Hash{0x01} + confNotifier := actor.NewChannelTellOnlyRef[ConfirmationEvent]( + "conf-notify", 10, + ) + reorgNotifier := actor.NewChannelTellOnlyRef[ConfReorgedEvent]( + "conf-reorged", 10, + ) + doneNotifier := actor.NewChannelTellOnlyRef[ConfDoneEvent]( + "conf-done", 10, + ) + + var confRef actor.TellOnlyRef[ConfirmationEvent] = confNotifier + var reorgRef actor.TellOnlyRef[ConfReorgedEvent] = reorgNotifier + var doneRef actor.TellOnlyRef[ConfDoneEvent] = doneNotifier + + result := confActor.Receive(ctx, &RegisterConfRequest{ + CallerID: "test-conf-reorg-lifecycle", + Txid: &txHash, + PkScript: []byte{0x00, 0x14}, + TargetConfs: 1, + NotifyActor: fn.Some(confRef), + NotifyReorged: fn.Some(reorgRef), + NotifyDone: fn.Some(doneRef), + }) + require.True(t, result.IsOk()) + resp, err := result.Unpack() + require.NoError(t, err) + confResp, ok := resp.(*RegisterConfResponse) + require.True(t, ok) + + // Reorg-aware mode is actor-only, so the response must not carry a + // Future. + require.Nil(t, confResp.Future) + + // 1. First confirmation on the canonical chain. + blockHash1 := chainhash.Hash{0xaa} + backend.confChan <- &TxConfirmation{ + BlockHash: &blockHash1, + BlockHeight: 100, + Tx: wire.NewMsgTx(2), + } + + event1, ok := confNotifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for first ConfirmationEvent") + require.Equal(t, int32(100), event1.BlockHeight) + require.Equal(t, blockHash1, event1.BlockHash) + + // 2. Reorg evicts that confirmation. + backend.confReorgedChan <- uint64(0) + + reorgEvt, ok := reorgNotifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for ConfReorgedEvent") + require.Equal(t, txHash, reorgEvt.Txid) + + // 3. Transaction re-confirms in a different block on the new tip. + blockHash2 := chainhash.Hash{0xbb} + backend.confChan <- &TxConfirmation{ + BlockHash: &blockHash2, + BlockHeight: 101, + Tx: wire.NewMsgTx(2), + } + + event2, ok := confNotifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for re-ConfirmationEvent") + require.Equal(t, int32(101), event2.BlockHeight) + require.Equal(t, blockHash2, event2.BlockHash) + + // 4. Registration matures past reorg safety; backend fires Done. + backend.confDoneChan <- struct{}{} + + doneEvt, ok := doneNotifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for ConfDoneEvent") + require.Equal(t, txHash, doneEvt.Txid) + + // After Done the actor must release the registration. + require.Eventually(t, func() bool { + return backend.confCancelled.Load() >= 1 + }, awaitTimeout, 10*time.Millisecond, + "registration Cancel was never invoked after Done") +} + +// TestConfActorSynthesizesDoneFromFinalityDepth verifies that a +// reorg-aware ConfActor with a non-zero FinalityDepth fires +// ConfDoneEvent on its own once enough blocks have been observed past +// the first Confirmed event, even when the backend's Done channel +// never fires. This closes the lndclient gRPC gap, where lnd's +// internal "past reorg-safety depth" signal does not survive the +// transport. +func TestConfActorSynthesizesDoneFromFinalityDepth(t *testing.T) { + t.Parallel() + + backend := newMockBackend() + ctx := t.Context() + const finalityDepth = 6 + confActor := NewConfActor(ConfActorConfig{ + Backend: backend, + FinalityDepth: finalityDepth, + }) + defer confActor.Stop() + + txHash := chainhash.Hash{0x01} + confNotifier := actor.NewChannelTellOnlyRef[ConfirmationEvent]( + "conf-notify", 10, + ) + reorgNotifier := actor.NewChannelTellOnlyRef[ConfReorgedEvent]( + "conf-reorged", 10, + ) + doneNotifier := actor.NewChannelTellOnlyRef[ConfDoneEvent]( + "conf-done", 10, + ) + + var confRef actor.TellOnlyRef[ConfirmationEvent] = confNotifier + var reorgRef actor.TellOnlyRef[ConfReorgedEvent] = reorgNotifier + var doneRef actor.TellOnlyRef[ConfDoneEvent] = doneNotifier + + result := confActor.Receive(ctx, &RegisterConfRequest{ + CallerID: "test-conf-finality-depth", + Txid: &txHash, + PkScript: []byte{0x00, 0x14}, + TargetConfs: 1, + NotifyActor: fn.Some(confRef), + NotifyReorged: fn.Some(reorgRef), + NotifyDone: fn.Some(doneRef), + }) + require.True(t, result.IsOk()) + + // 1. First confirmation at height 100. This arms the height-based + // finality synthesizer. + blockHash1 := chainhash.Hash{0xaa} + backend.confChan <- &TxConfirmation{ + BlockHash: &blockHash1, + BlockHeight: 100, + Tx: wire.NewMsgTx(2), + } + _, ok := confNotifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for ConfirmationEvent") + + // 2. Push blocks up to height 104. Inclusive depth at that point + // is 5 (heights 100..104), one short of the finality threshold, + // so ConfDoneEvent MUST NOT have fired yet. + for height := int32(101); height <= + int32(100+finalityDepth-2); height++ { + + backend.epochChan <- &BlockEpoch{Height: height} + } + _, ok = doneNotifier.AwaitMessage(50 * time.Millisecond) + require.False(t, ok, "ConfDoneEvent fired before finality depth") + + // 3. One more block brings the inclusive depth to exactly + // FinalityDepth (heights 100..105). Done must fire now. + backend.epochChan <- &BlockEpoch{ + Height: 100 + int32(finalityDepth) - 1, + } + doneEvt, ok := doneNotifier.AwaitMessage(awaitTimeout) + require.True( + t, ok, + "ConfDoneEvent never fired despite reaching finality depth", + ) + require.Equal(t, txHash, doneEvt.Txid) + + // 4. After Done the actor exits and releases the registration. + require.Eventually(t, func() bool { + return backend.confCancelled.Load() >= 1 + }, awaitTimeout, 10*time.Millisecond, + "registration Cancel was never invoked after synthesized "+ + "Done") +} + +// TestConfActorDiscardsStaleReorgBySeq verifies that a reorg signal which +// lost a cross-channel race to a newer re-confirmation is discarded by +// sequence number. The re-confirmation (seq 3) is observed before the +// older reorg (seq 2); because Confirmed and Reorged arrive on separate +// channels the actor cannot order them by arrival, so it must order them +// by Seq and ignore the stale reorg. The proof is that height-based +// finality still fires against the re-confirmation height — if the stale +// reorg had been applied it would have reset confirmHeight to 0 and Done +// would never synthesize. +func TestConfActorDiscardsStaleReorgBySeq(t *testing.T) { + t.Parallel() + + backend := newMockBackend() + ctx := t.Context() + const finalityDepth = 6 + confActor := NewConfActor(ConfActorConfig{ + Backend: backend, + FinalityDepth: finalityDepth, + }) + defer confActor.Stop() + + txHash := chainhash.Hash{0x04} + confNotifier := actor.NewChannelTellOnlyRef[ConfirmationEvent]( + "conf-notify", 10, + ) + reorgNotifier := actor.NewChannelTellOnlyRef[ConfReorgedEvent]( + "conf-reorged", 10, + ) + doneNotifier := actor.NewChannelTellOnlyRef[ConfDoneEvent]( + "conf-done", 10, + ) + + var confRef actor.TellOnlyRef[ConfirmationEvent] = confNotifier + var reorgRef actor.TellOnlyRef[ConfReorgedEvent] = reorgNotifier + var doneRef actor.TellOnlyRef[ConfDoneEvent] = doneNotifier + + result := confActor.Receive(ctx, &RegisterConfRequest{ + CallerID: "test-conf-stale-reorg-seq", + Txid: &txHash, + PkScript: []byte{0x00, 0x14}, + TargetConfs: 1, + NotifyActor: fn.Some(confRef), + NotifyReorged: fn.Some(reorgRef), + NotifyDone: fn.Some(doneRef), + }) + require.True(t, result.IsOk()) + + // 1. First confirmation at height 100, seq 1. + blockHash1 := chainhash.Hash{0xaa} + backend.confChan <- &TxConfirmation{ + BlockHash: &blockHash1, + BlockHeight: 100, + Tx: wire.NewMsgTx(2), + Seq: 1, + } + _, ok := confNotifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for first ConfirmationEvent") + + // 2. The re-confirmation (seq 3) is delivered before the older reorg + // (seq 2) — the cross-channel race. It arms finality at height 101. + blockHash2 := chainhash.Hash{0xbb} + backend.confChan <- &TxConfirmation{ + BlockHash: &blockHash2, + BlockHeight: 101, + Tx: wire.NewMsgTx(2), + Seq: 3, + } + _, ok = confNotifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for re-ConfirmationEvent") + + // 3. The stale reorg (seq 2 <= 3) arrives late and must be discarded: + // no ConfReorgedEvent is delivered and confirmHeight is untouched. + backend.confReorgedChan <- uint64(2) + _, ok = reorgNotifier.AwaitMessage(50 * time.Millisecond) + require.False(t, ok, "stale reorg (seq 2) was not discarded") + + // 4. Drive blocks to the finality depth past height 101. Done must + // fire, proving confirmHeight survived the stale reorg. + for height := int32(102); height <= + int32(101+finalityDepth)-1; height++ { + + backend.epochChan <- &BlockEpoch{Height: height} + } + doneEvt, ok := doneNotifier.AwaitMessage(awaitTimeout) + require.True( + t, ok, "ConfDoneEvent never fired; stale reorg wrongly "+ + "reset confirmHeight", + ) + require.Equal(t, txHash, doneEvt.Txid) +} + +// TestConfActorDiscardsStaleConfirmBySeq verifies the opposite race: a +// re-confirmation that lost a cross-channel race to a newer reorg is +// discarded by sequence number. The reorg (seq 3) is observed before the +// stale confirmation (seq 2); the actor must ignore the confirmation and +// leave the watch unconfirmed, so height-based finality must NOT fire. +// Without sequence ordering the stale confirmation would set a non-zero +// confirmHeight and synthesize a false Done for a tx that is gone. +func TestConfActorDiscardsStaleConfirmBySeq(t *testing.T) { + t.Parallel() + + backend := newMockBackend() + ctx := t.Context() + const finalityDepth = 6 + confActor := NewConfActor(ConfActorConfig{ + Backend: backend, + FinalityDepth: finalityDepth, + }) + defer confActor.Stop() + + txHash := chainhash.Hash{0x05} + confNotifier := actor.NewChannelTellOnlyRef[ConfirmationEvent]( + "conf-notify", 10, + ) + reorgNotifier := actor.NewChannelTellOnlyRef[ConfReorgedEvent]( + "conf-reorged", 10, + ) + doneNotifier := actor.NewChannelTellOnlyRef[ConfDoneEvent]( + "conf-done", 10, + ) + + var confRef actor.TellOnlyRef[ConfirmationEvent] = confNotifier + var reorgRef actor.TellOnlyRef[ConfReorgedEvent] = reorgNotifier + var doneRef actor.TellOnlyRef[ConfDoneEvent] = doneNotifier + + result := confActor.Receive(ctx, &RegisterConfRequest{ + CallerID: "test-conf-stale-confirm-seq", + Txid: &txHash, + PkScript: []byte{0x00, 0x14}, + TargetConfs: 1, + NotifyActor: fn.Some(confRef), + NotifyReorged: fn.Some(reorgRef), + NotifyDone: fn.Some(doneRef), + }) + require.True(t, result.IsOk()) + + // 1. First confirmation at height 100, seq 1. + blockHash1 := chainhash.Hash{0xaa} + backend.confChan <- &TxConfirmation{ + BlockHash: &blockHash1, + BlockHeight: 100, + Tx: wire.NewMsgTx(2), + Seq: 1, + } + _, ok := confNotifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for first ConfirmationEvent") + + // 2. The newer reorg (seq 3) is observed first and resets the watch. + backend.confReorgedChan <- uint64(3) + _, ok = reorgNotifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for ConfReorgedEvent") + + // 3. The stale re-confirmation (seq 2 <= 3) arrives late and must be + // discarded: no ConfirmationEvent is delivered and the watch stays + // unconfirmed. + blockHash2 := chainhash.Hash{0xbb} + backend.confChan <- &TxConfirmation{ + BlockHash: &blockHash2, + BlockHeight: 101, + Tx: wire.NewMsgTx(2), + Seq: 2, + } + _, ok = confNotifier.AwaitMessage(50 * time.Millisecond) + require.False(t, ok, "stale confirmation (seq 2) was not discarded") + + // 4. Drive many blocks well past any finality window. Done must NOT + // fire because confirmHeight was never re-armed. + for height := int32(101); height <= int32(120); height++ { + backend.epochChan <- &BlockEpoch{Height: height} + } + _, ok = doneNotifier.AwaitMessage(100 * time.Millisecond) + require.False( + t, ok, "ConfDoneEvent fired for a reorged-out tx; stale "+ + "confirmation was wrongly applied", + ) +} + +// TestConfActorReorgAwareRejectsWithoutNotifyActor verifies that opting in +// to reorg-aware mode without an actor-mode confirmation ref is rejected at +// admission. Allowing it would silently drop every re-confirmation after +// the first, since a Future can only complete once. +func TestConfActorReorgAwareRejectsWithoutNotifyActor(t *testing.T) { + t.Parallel() + + backend := newMockBackend() + ctx := t.Context() + confActor := NewConfActor(ConfActorConfig{Backend: backend}) + defer confActor.Stop() + + txHash := chainhash.Hash{0x02} + reorgNotifier := actor.NewChannelTellOnlyRef[ConfReorgedEvent]( + "conf-reorged", 1, + ) + var reorgRef actor.TellOnlyRef[ConfReorgedEvent] = reorgNotifier + + result := confActor.Receive(ctx, &RegisterConfRequest{ + CallerID: "test-conf-reorg-no-notify", + Txid: &txHash, + PkScript: []byte{0x00, 0x14}, + TargetConfs: 1, + NotifyReorged: fn.Some(reorgRef), + }) + require.True(t, result.IsErr()) + _, err := result.Unpack() + require.ErrorContains( + t, err, + "reorg/done notifications require actor-mode NotifyActor", + ) +} + +// TestConfActorLegacyExitsAfterFirstConfirm ensures legacy Actor-mode +// subscribers (no NotifyReorged / NotifyDone) keep their historical +// single-shot contract even when the backend would later emit Reorged or +// Done. The actor must cancel the registration after the first +// confirmation. +func TestConfActorLegacyExitsAfterFirstConfirm(t *testing.T) { + t.Parallel() + + backend := newMockBackend() + ctx := t.Context() + confActor := NewConfActor(ConfActorConfig{Backend: backend}) + defer confActor.Stop() + + txHash := chainhash.Hash{0x03} + notifier := actor.NewChannelTellOnlyRef[ConfirmationEvent]( + "conf-notify", 10, + ) + var confRef actor.TellOnlyRef[ConfirmationEvent] = notifier + + result := confActor.Receive(ctx, &RegisterConfRequest{ + CallerID: "test-conf-legacy-single-shot", + Txid: &txHash, + PkScript: []byte{0x00, 0x14}, + TargetConfs: 1, + NotifyActor: fn.Some(confRef), + }) + require.True(t, result.IsOk()) + + // First confirmation goes through. + blockHash := chainhash.Hash{0x10} + backend.confChan <- &TxConfirmation{ + BlockHash: &blockHash, + BlockHeight: 200, + Tx: wire.NewMsgTx(2), + } + + event, ok := notifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for first ConfirmationEvent") + require.Equal(t, int32(200), event.BlockHeight) + + // Actor must have exited after the first confirmation, releasing the + // registration. + require.Eventually(t, func() bool { + return backend.confCancelled.Load() >= 1 + }, awaitTimeout, 10*time.Millisecond, + "legacy ConfActor did not cancel registration after first "+ + "confirmation") + + // No further events should reach the notifier. + _, ok = notifier.AwaitMessage(50 * time.Millisecond) + require.False( + t, ok, "legacy ConfActor delivered an unexpected second event", + ) +} + +// TestSpendActorReorgAwareForwardsFullLifecycle drives the spend lifecycle +// (Spend -> Reorged -> Spend -> Done) through a reorg-aware SpendActor and +// asserts every event is forwarded to the correct notify ref in order. +func TestSpendActorReorgAwareForwardsFullLifecycle(t *testing.T) { + t.Parallel() + + backend := newMockBackend() + ctx := t.Context() + spendActor := NewSpendActor(SpendActorConfig{Backend: backend}) + defer spendActor.Stop() + + outpoint := wire.OutPoint{Hash: chainhash.Hash{0x11}, Index: 0} + + spendNotifier := actor.NewChannelTellOnlyRef[SpendEvent]( + "spend-notify", 10, + ) + reorgNotifier := actor.NewChannelTellOnlyRef[SpendReorgedEvent]( + "spend-reorged", 10, + ) + doneNotifier := actor.NewChannelTellOnlyRef[SpendDoneEvent]( + "spend-done", 10, + ) + + var spendRef actor.TellOnlyRef[SpendEvent] = spendNotifier + var reorgRef actor.TellOnlyRef[SpendReorgedEvent] = reorgNotifier + var doneRef actor.TellOnlyRef[SpendDoneEvent] = doneNotifier + + result := spendActor.Receive(ctx, &RegisterSpendRequest{ + CallerID: "test-spend-reorg-lifecycle", + Outpoint: &outpoint, + PkScript: []byte{0x00, 0x14}, + NotifyActor: fn.Some(spendRef), + NotifyReorged: fn.Some(reorgRef), + NotifyDone: fn.Some(doneRef), + }) + require.True(t, result.IsOk()) + resp, err := result.Unpack() + require.NoError(t, err) + spendResp, ok := resp.(*RegisterSpendResponse) + require.True(t, ok) + require.Nil(t, spendResp.Future) + + // 1. First spend confirms. + spendingTx1 := wire.NewMsgTx(2) + hash1 := spendingTx1.TxHash() + backend.spendChan <- &SpendDetail{ + SpentOutPoint: &outpoint, + SpenderTxHash: &hash1, + SpendingTx: spendingTx1, + SpendingHeight: 150, + } + + spend1, ok := spendNotifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for first SpendEvent") + require.Equal(t, outpoint, spend1.Outpoint) + require.Equal(t, int32(150), spend1.SpendingHeight) + require.Equal(t, hash1, spend1.SpendingTxid) + + // 2. Reorg evicts that spend. + backend.spendReorgedChan <- uint64(0) + + reorgEvt, ok := reorgNotifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for SpendReorgedEvent") + require.Equal(t, outpoint, reorgEvt.Outpoint) + + // 3. A different spender wins the new chain. + spendingTx2 := wire.NewMsgTx(2) + spendingTx2.AddTxIn(&wire.TxIn{Sequence: 1}) + hash2 := spendingTx2.TxHash() + require.NotEqual(t, hash1, hash2) + backend.spendChan <- &SpendDetail{ + SpentOutPoint: &outpoint, + SpenderTxHash: &hash2, + SpendingTx: spendingTx2, + SpendingHeight: 151, + } + + spend2, ok := spendNotifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for re-SpendEvent") + require.Equal(t, outpoint, spend2.Outpoint) + require.Equal(t, int32(151), spend2.SpendingHeight) + require.Equal(t, hash2, spend2.SpendingTxid) + + // 4. Registration matures past reorg safety. + backend.spendDoneChan <- struct{}{} + + doneEvt, ok := doneNotifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for SpendDoneEvent") + require.Equal(t, outpoint, doneEvt.Outpoint) + + require.Eventually(t, func() bool { + return backend.spendCancelled.Load() >= 1 + }, awaitTimeout, 10*time.Millisecond, + "registration Cancel was never invoked after Done") +} + +// TestSpendActorSynthesizesDoneFromFinalityDepth mirrors the conf-side +// height-based finality test for the spend watch. A reorg-aware +// SpendActor with non-zero FinalityDepth fires SpendDoneEvent on its +// own once enough blocks have been observed past the first Spend, +// closing the same lndclient gRPC gap that ConfActor closes for +// confirmations. +func TestSpendActorSynthesizesDoneFromFinalityDepth(t *testing.T) { + t.Parallel() + + backend := newMockBackend() + ctx := t.Context() + const finalityDepth = 6 + spendActor := NewSpendActor(SpendActorConfig{ + Backend: backend, + FinalityDepth: finalityDepth, + }) + defer spendActor.Stop() + + outpoint := wire.OutPoint{Hash: chainhash.Hash{0x11}, Index: 0} + + spendNotifier := actor.NewChannelTellOnlyRef[SpendEvent]( + "spend-notify", 10, + ) + reorgNotifier := actor.NewChannelTellOnlyRef[SpendReorgedEvent]( + "spend-reorged", 10, + ) + doneNotifier := actor.NewChannelTellOnlyRef[SpendDoneEvent]( + "spend-done", 10, + ) + + var spendRef actor.TellOnlyRef[SpendEvent] = spendNotifier + var reorgRef actor.TellOnlyRef[SpendReorgedEvent] = reorgNotifier + var doneRef actor.TellOnlyRef[SpendDoneEvent] = doneNotifier + + result := spendActor.Receive(ctx, &RegisterSpendRequest{ + CallerID: "test-spend-finality-depth", + Outpoint: &outpoint, + PkScript: []byte{0x00, 0x14}, + NotifyActor: fn.Some(spendRef), + NotifyReorged: fn.Some(reorgRef), + NotifyDone: fn.Some(doneRef), + }) + require.True(t, result.IsOk()) + + // 1. First spend confirms at height 150. This arms the + // height-based finality synthesizer. + spendingTx := wire.NewMsgTx(2) + hash := spendingTx.TxHash() + backend.spendChan <- &SpendDetail{ + SpentOutPoint: &outpoint, + SpenderTxHash: &hash, + SpendingTx: spendingTx, + SpendingHeight: 150, + } + _, ok := spendNotifier.AwaitMessage(awaitTimeout) + require.True(t, ok, "timeout waiting for SpendEvent") + + // 2. Push blocks up to height 154. Inclusive depth at that point + // is 5 (heights 150..154), one short of the finality threshold. + // SpendDoneEvent MUST NOT have fired yet. + for height := int32(151); height <= + int32(150+finalityDepth-2); height++ { + + backend.epochChan <- &BlockEpoch{Height: height} + } + _, ok = doneNotifier.AwaitMessage(50 * time.Millisecond) + require.False(t, ok, "SpendDoneEvent fired before finality depth") + + // 3. One more block brings the inclusive depth to exactly + // FinalityDepth (heights 150..155). Done must fire now. + backend.epochChan <- &BlockEpoch{ + Height: 150 + int32(finalityDepth) - 1, + } + doneEvt, ok := doneNotifier.AwaitMessage(awaitTimeout) + require.True( + t, ok, + "SpendDoneEvent never fired despite reaching finality depth", + ) + require.Equal(t, outpoint, doneEvt.Outpoint) + + require.Eventually(t, func() bool { + return backend.spendCancelled.Load() >= 1 + }, awaitTimeout, 10*time.Millisecond, + "registration Cancel was never invoked after synthesized "+ + "Done") +} + +// TestSpendActorReorgAwareRejectsWithoutNotifyActor mirrors the conf-side +// admission check: opting in to reorg-aware spend forwarding without a +// NotifyActor must be rejected. +func TestSpendActorReorgAwareRejectsWithoutNotifyActor(t *testing.T) { + t.Parallel() + + backend := newMockBackend() + ctx := t.Context() + spendActor := NewSpendActor(SpendActorConfig{Backend: backend}) + defer spendActor.Stop() + + outpoint := wire.OutPoint{Hash: chainhash.Hash{0x21}, Index: 0} + reorgNotifier := actor.NewChannelTellOnlyRef[SpendReorgedEvent]( + "spend-reorged", 1, + ) + var reorgRef actor.TellOnlyRef[SpendReorgedEvent] = reorgNotifier + + result := spendActor.Receive(ctx, &RegisterSpendRequest{ + CallerID: "test-spend-reorg-no-notify", + Outpoint: &outpoint, + PkScript: []byte{0x00, 0x14}, + NotifyReorged: fn.Some(reorgRef), + }) + require.True(t, result.IsErr()) + _, err := result.Unpack() + require.ErrorContains( + t, err, + "reorg/done notifications require actor-mode NotifyActor", + ) +} + +// TestSpendActorLegacyExitsAfterFirstSpend exercises the legacy Actor-mode +// path: a watch that did not opt into the reorg lifecycle +// (NotifyReorged/NotifyDone both unset) is single-shot, exiting and +// releasing its backend registration after the first spend. This mirrors +// ConfActor's legacy contract and the documented invariant in CLAUDE.md; +// without the reorgAware gate such a watch would run forever and, with +// FinalityDepth > 0, arm a block subscription it never requested. +func TestSpendActorLegacyExitsAfterFirstSpend(t *testing.T) { + t.Parallel() + + backend := newMockBackend() + ctx := t.Context() + spendActor := NewSpendActor(SpendActorConfig{Backend: backend}) + defer spendActor.Stop() + + outpoint := wire.OutPoint{Hash: chainhash.Hash{0x31}, Index: 0} + notifier := actor.NewChannelTellOnlyRef[SpendEvent]( + "spend-notify", 10, + ) + var spendRef actor.TellOnlyRef[SpendEvent] = notifier + + result := spendActor.Receive(ctx, &RegisterSpendRequest{ + CallerID: "test-spend-legacy-single-shot", + Outpoint: &outpoint, + PkScript: []byte{0x00, 0x14}, + NotifyActor: fn.Some(spendRef), + }) + require.True(t, result.IsOk()) + + // First spend goes through. + spendingTx := wire.NewMsgTx(2) + hash := spendingTx.TxHash() + backend.spendChan <- &SpendDetail{ + SpentOutPoint: &outpoint, + SpenderTxHash: &hash, + SpendingTx: spendingTx, + SpendingHeight: 300, + } + + first, ok := notifier.AwaitMessage(awaitTimeout) + require.True(t, ok) + require.Equal(t, int32(300), first.SpendingHeight) + + // The actor must have exited after the first spend, releasing the + // registration. + require.Eventually(t, func() bool { + return backend.spendCancelled.Load() >= 1 + }, awaitTimeout, 10*time.Millisecond, + "legacy SpendActor did not cancel registration after first "+ + "spend") + + // No further events should reach the notifier. + _, ok = notifier.AwaitMessage(50 * time.Millisecond) + require.False( + t, ok, "legacy SpendActor delivered an unexpected second event", + ) +} diff --git a/chainsource/spend_actor.go b/chainsource/spend_actor.go index 287d1b3e3..4dbaa267a 100644 --- a/chainsource/spend_actor.go +++ b/chainsource/spend_actor.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "sync" + "time" "github.com/btcsuite/btcd/chainhash/v2" "github.com/btcsuite/btcd/wire/v2" @@ -23,6 +24,15 @@ type SpendActorConfig struct { // falls back to extracting a logger from context via LoggerFromContext, // or uses btclog.Disabled if no logger is found. Log fn.Option[btclog.Logger] + + // FinalityDepth is the number of confirmations past the first + // observed Spend event that the actor uses to synthesize a Done + // signal when the backend cannot deliver one. See the matching + // field on ConfActorConfig for the rationale; the same constraint + // applies on the spend watch — lnd's chainntnfs.SpendEvent.Done + // does not survive the lndclient gRPC transport, so consumers + // that gate eviction on Done would otherwise leak per-spend state. + FinalityDepth uint32 } // WithLogger returns a new config with the given logger set. @@ -61,9 +71,26 @@ type SpendActor struct { // mode. notifyActor fn.Option[actor.TellOnlyRef[SpendEvent]] + // notifyReorged receives spend reorg events in Actor mode. + notifyReorged fn.Option[actor.TellOnlyRef[SpendReorgedEvent]] + + // notifyDone receives spend finality events in Actor mode. + notifyDone fn.Option[actor.TellOnlyRef[SpendDoneEvent]] + // registration is the backend registration for this watch. registration *SpendRegistration + // blockReg is the block-epoch subscription used by height-based + // finality synthesis. Allocated lazily after the first Spend + // event when FinalityDepth > 0, torn down when the actor exits. + blockReg *BlockRegistration + + // spendHeight records the block height the most recent Spend + // event arrived at. Zero means there is no active spend to count + // from (either we have not yet seen one, or the last one was + // reorged out). + spendHeight int32 + // ctx is the actor's internal context for cancellation, created from // context.Background() to ensure it outlives any request context. //nolint:containedctx @@ -132,12 +159,21 @@ func (a *SpendActor) handleRegisterSpend(actorCtx context.Context, "provided"), ) } + if req.NotifyActor.IsNone() && + (req.NotifyReorged.IsSome() || req.NotifyDone.IsSome()) { + return fn.Err[SpendResp]( + fmt.Errorf("spend reorg/done notifications require " + + "actor-mode NotifyActor"), + ) + } // Configure the actor with request parameters. a.outpoint = req.Outpoint a.pkScript = req.PkScript a.heightHint = req.HeightHint a.notifyActor = req.NotifyActor + a.notifyReorged = req.NotifyReorged + a.notifyDone = req.NotifyDone // Create promise for Future mode. var promise fn.Option[actor.Promise[SpendEvent]] @@ -152,10 +188,17 @@ func (a *SpendActor) handleRegisterSpend(actorCtx context.Context, // Register with the backend to receive spend notifications. We do this // before starting the goroutine so we can return an error to the - // caller if registration fails. + // caller if registration fails. The bounded timeout mirrors + // ConfActor.handleRegisterConf: a backend (LND) that is slow under + // heavy block processing load must not pin the parent Receive call + // indefinitely, since that would back-pressure the chainsource + // factory actor onto every other in-flight registration. + regCtx, regCancel := context.WithTimeout(a.ctx, 10*time.Second) + defer regCancel() + //nolint:contextcheck // actor root context owns registration lifetime registration, err := a.cfg.Backend.RegisterSpend( - a.ctx, a.outpoint, a.pkScript, a.heightHint, + regCtx, a.outpoint, a.pkScript, a.heightHint, ) if err != nil { return fn.Err[SpendResp]( @@ -191,10 +234,47 @@ func (a *SpendActor) monitorSpend() { if a.registration != nil { a.registration.Cancel() } + if a.blockReg != nil { + a.blockReg.Cancel() + } }() + log := a.logger(a.ctx) + // Monitor for spends indefinitely until cancelled or shutdown. // This allows us to catch re-org events where a spend is replaced. + var lastEvent *SpendEvent + + // reorgAware reports whether the caller opted into the multi-shot + // reorg lifecycle (at least one of NotifyReorged/NotifyDone). When + // false the watch is single-shot for backwards compatibility: the + // actor exits after the first spend, mirroring ConfActor. Without + // this gate a plain actor-mode spend watch would run forever and, with + // FinalityDepth > 0, arm a block subscription it never asked for. + reorgAware := a.notifyReorged.IsSome() || a.notifyDone.IsSome() + + // lastSeq is the highest backend forwarder sequence applied so far. + // Spend and Reorged signals arrive on separate channels and a select + // cannot order two ready channels, so we order them by the shared + // sequence instead: an event whose Seq does not exceed lastSeq lost a + // cross-channel race to a newer signal and is discarded. This makes + // the actor's view correct regardless of delivery interleaving. Seq 0 + // means the backend does not stamp sequences (it never reorgs); those + // events are always applied. + var lastSeq uint64 + + // blockEpochs is rebound when height-based finality synthesis + // arms a block subscription. Until then a nil channel keeps the + // select arm parked. + var blockEpochs <-chan *BlockEpoch + + // blockRegCh hands a finality block subscription from the off-loop + // arming goroutine back to this loop; arming guards against launching + // more than one armer at a time. See armFinalityAsync for why arming + // runs off the select loop. + blockRegCh := make(chan finalityArmResult) + var arming bool + for { select { case spend, ok := <-a.registration.Spend: @@ -206,6 +286,16 @@ func (a *SpendActor) monitorSpend() { return } + // Discard a spend that lost a cross-channel race to a + // newer reorg: an event whose sequence does not exceed + // the highest applied is stale. + if spend.Seq != 0 && spend.Seq <= lastSeq { + continue + } + if spend.Seq > lastSeq { + lastSeq = spend.Seq + } + event, err := buildSpendEvent(spend, a) if err != nil { a.failSpend(err) @@ -215,13 +305,151 @@ func (a *SpendActor) monitorSpend() { // Deliver the event. a.deliverSpend(event) - - // In Future mode, exit after first event. In Actor - // mode, continue monitoring for re-org events. - if a.promise.IsSome() { + lastEvent = &event + a.spendHeight = event.SpendingHeight + + // Exit after the first event in Future mode, and in + // actor mode that did not opt into the reorg lifecycle + // (single-shot backwards-compatible contract). Only a + // reorg-aware actor watch keeps monitoring. + if !reorgAware || a.promise.IsSome() { return } + // Arm height-based finality synthesis on the first + // spend if requested, off the select loop so the + // bounded RegisterBlocks retries cannot stall delivery + // of Reorged/Done/ctx.Done on this watch. A nil + // blockEpochs channel keeps the synthesis arm parked + // until the registration is handed back on blockRegCh. + if a.cfg.FinalityDepth > 0 && a.blockReg == nil && + !arming { + + arming = true + a.armFinalityAsync(blockRegCh, log) + } + + case armed := <-blockRegCh: + // Finality arming completed. Clear the flag; a nil reg + // only happens when the watch context was cancelled + // (arming otherwise retries until it succeeds). + arming = false + if armed.reg == nil { + continue + } + a.blockReg = armed.reg + blockEpochs = armed.reg.Epochs + + // The block-epoch subscription only delivers FUTURE + // epochs, but the spend that armed it may already be + // buried past FinalityDepth (it confirmed several + // blocks ago, or we re-armed after a restart). Use the + // tip observed at arm time to synthesize Done at once + // rather than hang until a fresh block is mined. The + // spendHeight==0 / FinalityDepth==0 guards mirror the + // epoch handler below. + if a.spendHeight == 0 || a.cfg.FinalityDepth == 0 { + continue + } + if armed.height-a.spendHeight+1 < + int32(a.cfg.FinalityDepth) { + + continue + } + + log.InfoS(a.ctx, "Synthesizing spend done on arm from "+ + "height-based safety depth", + "spend_height", a.spendHeight, + "current_height", armed.height, + "finality_depth", int(a.cfg.FinalityDepth), + ) + a.deliverSpendDone(lastEvent) + + return + + case seq, ok := <-a.registration.Reorged: + if !ok { + a.registration.Reorged = nil + continue + } + + // Discard a stale reorg that lost a cross-channel race + // to a newer spend. + if seq != 0 && seq <= lastSeq { + continue + } + if seq > lastSeq { + lastSeq = seq + } + + a.deliverSpendReorged(lastEvent) + + // The previous spend is no longer on the canonical + // chain. Clear the cached event so a later Done cannot + // report the reorged-out outpoint, and reset the depth + // counter so the next re-spend starts a fresh window. + lastEvent = nil + a.spendHeight = 0 + + case _, ok := <-a.registration.Done: + if !ok { + a.registration.Done = nil + continue + } + + a.deliverSpendDone(lastEvent) + + return + + case epoch, ok := <-blockEpochs: + if !ok || epoch == nil { + blockEpochs = nil + continue + } + + // Coalesce any epochs already queued behind this one + // and evaluate finality against the most recent height + // only. With rapid-fire blocks the channel can hold + // several epochs at once; processing them one per loop + // iteration would re-check the same monotonic Done + // condition repeatedly and risk synthesizing against a + // stale height. If the channel closed during the drain, + // park it so we stop selecting on it. + var closed bool + epoch, closed = drainToLatestEpoch(blockEpochs, epoch) + if closed { + blockEpochs = nil + } + + // The spendHeight==0 guard is load-bearing: a reorg + // resets spendHeight to 0 (the Reorged arm above), so + // a fresh epoch arriving before the re-spend would + // otherwise compute depth against a zero base and + // could synthesize Done prematurely. While + // spendHeight==0 there is no active spend to count + // from, so the depth comparison is meaningless. + // FinalityDepth==0 disables synthesis entirely. + if a.spendHeight == 0 || + a.cfg.FinalityDepth == 0 { + + continue + } + + depth := epoch.Height - a.spendHeight + 1 + if depth < int32(a.cfg.FinalityDepth) { + continue + } + + log.InfoS(a.ctx, "Synthesizing spend done from "+ + "height-based safety depth", + "spend_height", a.spendHeight, + "current_height", epoch.Height, + "finality_depth", int(a.cfg.FinalityDepth), + ) + a.deliverSpendDone(lastEvent) + + return + case <-a.ctx.Done(): // Actor was cancelled. a.failSpend(a.ctx.Err()) @@ -231,6 +459,52 @@ func (a *SpendActor) monitorSpend() { } } +// armFinalityAsync registers a block-epoch subscription for height-based +// finality synthesis off the actor's select loop. registerBlocksForFinality +// retries with a bounded backoff that can run for tens of seconds; doing it +// inline would block delivery of Reorged/Done/ctx.Done on this watch for the +// whole window. The registration (or nil on failure) is handed back on regCh, +// or cancelled if the actor exits before the loop reads it. The goroutine is +// tracked by the actor's wait group so Stop drains it. +func (a *SpendActor) armFinalityAsync(regCh chan<- finalityArmResult, + log btclog.Logger) { + + a.wg.Go(func() { + reg, err := registerBlocksForFinality(a.ctx, a.cfg.Backend, log) + if err != nil { + log.WarnS(a.ctx, "Giving up on height-based finality "+ + "synthesis; spend sub-actor will rely on "+ + "backend Done", err) + reg = nil + } + + // Capture the tip at arm time so the loop can finalize + // immediately when the arming spend is already buried past + // FinalityDepth (the block-epoch sub only delivers future + // epochs). A read failure is non-fatal: height stays zero and + // the loop falls back to waiting for the next epoch. + var height int32 + if reg != nil { + h, _, hErr := a.cfg.Backend.BestBlock(a.ctx) + if hErr != nil { + log.WarnS(a.ctx, "Failed to read best height "+ + "for on-arm finality check; will wait "+ + "for next epoch", hErr) + } else { + height = h + } + } + + select { + case regCh <- finalityArmResult{reg: reg, height: height}: + case <-a.ctx.Done(): + if reg != nil { + reg.Cancel() + } + } + }) +} + // deliverSpend delivers a spend event to the subscriber. In Future mode, it // completes the promise. In Actor mode, it sends to the registered actor. func (a *SpendActor) deliverSpend(event SpendEvent) { @@ -249,6 +523,55 @@ func (a *SpendActor) deliverSpend(event SpendEvent) { }) } +// deliverSpendReorged delivers a spend reorg event to actor-mode +// subscribers. The correlation Outpoint is the registration's configured +// outpoint when set, since that is the identifier the caller asked us to +// watch; pkScript-only watches fall back to the outpoint carried on the +// most recent positive SpendEvent. +func (a *SpendActor) deliverSpendReorged(lastEvent *SpendEvent) { + var event SpendReorgedEvent + switch { + case a.outpoint != nil: + event.Outpoint = *a.outpoint + + case lastEvent != nil: + event.Outpoint = lastEvent.Outpoint + } + + a.notifyReorged.WhenSome( + func(ref actor.TellOnlyRef[SpendReorgedEvent]) { + log := a.logger(a.ctx) + if err := ref.Tell(a.ctx, event); err != nil { + log.WarnS( + a.ctx, + "Failed to deliver spend reorg", + err, + ) + } + }, + ) +} + +// deliverSpendDone delivers a spend finality event to actor-mode subscribers. +// Outpoint follows the same precedence as deliverSpendReorged. +func (a *SpendActor) deliverSpendDone(lastEvent *SpendEvent) { + var event SpendDoneEvent + switch { + case a.outpoint != nil: + event.Outpoint = *a.outpoint + + case lastEvent != nil: + event.Outpoint = lastEvent.Outpoint + } + + a.notifyDone.WhenSome(func(ref actor.TellOnlyRef[SpendDoneEvent]) { + log := a.logger(a.ctx) + if err := ref.Tell(a.ctx, event); err != nil { + log.WarnS(a.ctx, "Failed to deliver spend done", err) + } + }) +} + // failSpend completes the promise with an error (Future mode) or does nothing // (Actor mode - errors are not delivered in async mode). func (a *SpendActor) failSpend(err error) { diff --git a/chainsource/transform.go b/chainsource/transform.go index 16f9f164a..9e392ae62 100644 --- a/chainsource/transform.go +++ b/chainsource/transform.go @@ -33,6 +33,25 @@ func MapConfirmationEvent[Out actor.Message]( return actor.NewMapInputRef(targetRef, mapFn) } +// MapConfReorgedEvent creates a transformed TellOnlyRef that accepts +// ConfReorgedEvent and transforms it to the caller's desired output message +// type. +func MapConfReorgedEvent[Out actor.Message]( + targetRef actor.TellOnlyRef[Out], mapFn func(ConfReorgedEvent) Out, +) actor.TellOnlyRef[ConfReorgedEvent] { + + return actor.NewMapInputRef(targetRef, mapFn) +} + +// MapConfDoneEvent creates a transformed TellOnlyRef that accepts +// ConfDoneEvent and transforms it to the caller's desired output message type. +func MapConfDoneEvent[Out actor.Message]( + targetRef actor.TellOnlyRef[Out], mapFn func(ConfDoneEvent) Out, +) actor.TellOnlyRef[ConfDoneEvent] { + + return actor.NewMapInputRef(targetRef, mapFn) +} + // MapSpendEvent creates a transformed TellOnlyRef that accepts SpendEvent and // transforms it to the caller's desired output message type. This is a // convenience wrapper for the common pattern of adapting chainsource spend @@ -64,6 +83,25 @@ func MapSpendEvent[Out actor.Message]( return actor.NewMapInputRef(targetRef, mapFn) } +// MapSpendReorgedEvent creates a transformed TellOnlyRef that accepts +// SpendReorgedEvent and transforms it to the caller's desired output message +// type. +func MapSpendReorgedEvent[Out actor.Message]( + targetRef actor.TellOnlyRef[Out], mapFn func(SpendReorgedEvent) Out, +) actor.TellOnlyRef[SpendReorgedEvent] { + + return actor.NewMapInputRef(targetRef, mapFn) +} + +// MapSpendDoneEvent creates a transformed TellOnlyRef that accepts +// SpendDoneEvent and transforms it to the caller's desired output message type. +func MapSpendDoneEvent[Out actor.Message]( + targetRef actor.TellOnlyRef[Out], mapFn func(SpendDoneEvent) Out, +) actor.TellOnlyRef[SpendDoneEvent] { + + return actor.NewMapInputRef(targetRef, mapFn) +} + // MapBlockEpoch creates a transformed TellOnlyRef that accepts BlockEpoch // and transforms it to the caller's desired output message type. This is a // convenience wrapper for the common pattern of adapting chainsource block diff --git a/darepod/server.go b/darepod/server.go index 58b0082c9..c671f4c0a 100644 --- a/darepod/server.go +++ b/darepod/server.go @@ -2082,6 +2082,13 @@ func (s *Server) registerChainSourceActor( chainsource.ChainSourceConfig{ Backend: s.chainBackend, System: s.actorSystem, + // Enable height-based Done synthesis at the default + // safety depth. Without this, conf/spend sub-actors + // driven through lndclient (whose Done channel is + // allocated-but-never-written) would never receive a + // finality signal, and reorg-aware consumers like + // txconfirm would leak watch state forever. + FinalityDepth: chainsource.DefaultFinalityDepth, }, ) diff --git a/go.mod b/go.mod index 098ad76dd..11cb33b9e 100644 --- a/go.mod +++ b/go.mod @@ -236,3 +236,7 @@ replace google.golang.org/protobuf => github.com/lightninglabs/protobuf-go-hex-d // Use local baselib for development and CI. replace github.com/lightninglabs/darepo-client/baselib => ./baselib + +replace github.com/lightningnetwork/lnd => github.com/ellemouton/lnd v0.8.0-beta-rc3.0.20260701213016-8b4da077a431 + +replace github.com/lightninglabs/lndclient => github.com/ellemouton/lndclient v1.0.1-0.20260701213152-a3baa8dfda6c diff --git a/go.sum b/go.sum index be7e4b95b..503bc3c1f 100644 --- a/go.sum +++ b/go.sum @@ -757,6 +757,10 @@ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ellemouton/lnd v0.8.0-beta-rc3.0.20260701213016-8b4da077a431 h1:KOLxRQYXb++mDgkqXhudfh5M2rZ/GG1USFsyb44LBSU= +github.com/ellemouton/lnd v0.8.0-beta-rc3.0.20260701213016-8b4da077a431/go.mod h1:LWfQaNNXlZghUJqIIh+JsgfxiVtC+ecZkrIe5bm9o4U= +github.com/ellemouton/lndclient v1.0.1-0.20260701213152-a3baa8dfda6c h1:v1aM7KEH0l4cR8FzYK9WwNQsFyj10Qm904aTbffGJZc= +github.com/ellemouton/lndclient v1.0.1-0.20260701213152-a3baa8dfda6c/go.mod h1:QlUlKGfYeQ+6lR8BpWDe4qBM91OGdiOl09esQMTEq8k= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -1046,8 +1050,6 @@ github.com/lightninglabs/gozmq v0.0.0-20191113021534-d20a764486bf h1:HZKvJUHlcXI github.com/lightninglabs/gozmq v0.0.0-20191113021534-d20a764486bf/go.mod h1:vxmQPeIQxPf6Jf9rM8R+B4rKBqLA2AjttNxkFBL2Plk= github.com/lightninglabs/lightning-node-connect/hashmailrpc v1.0.4-0.20250610182311-2f1d46ef18b7 h1:373o5lNr1udAdhcf5+zq/0dYpRtkvYLl8Lk6wG7I0DY= github.com/lightninglabs/lightning-node-connect/hashmailrpc v1.0.4-0.20250610182311-2f1d46ef18b7/go.mod h1:bDnEKRN1u13NFBuy/C+bFLhxA5bfd3clT25y76QY0AM= -github.com/lightninglabs/lndclient v1.0.1-0.20260701212139-09c54915cfba h1:PbtOHmA9P41mjWt7HDZDDzYshv8fmyhIBPkdOf8wKws= -github.com/lightninglabs/lndclient v1.0.1-0.20260701212139-09c54915cfba/go.mod h1:DtZg/wz7YFXVCwOwYOJyWKQBnUp1NFg4RQdax3MSYbc= github.com/lightninglabs/loop v0.33.0-beta h1:cfs+JrnmKw+ANa09XFS/2Jjm9wxHq3CfAzh9B1ym3e0= github.com/lightninglabs/loop v0.33.0-beta/go.mod h1:uTcjRmEC4ni4lWrOdnIvI5pyY8XJ2wXSXSE12vwFgzk= github.com/lightninglabs/migrate/v4 v4.18.2-9023d66a-fork-pr-2 h1:eFjp1dIB2BhhQp/THKrjLdlYuPugO9UU4kDqu91OX/Q= @@ -1064,8 +1066,6 @@ github.com/lightninglabs/taproot-assets/taprpc v1.1.1-0.20260706193822-2adfadc58 github.com/lightninglabs/taproot-assets/taprpc v1.1.1-0.20260706193822-2adfadc58e3c/go.mod h1:nOjUfSstCfHYY0QNnxlhN+N915Vcl5ziyHiOD8hVoJI= github.com/lightningnetwork/lightning-onion v1.4.0 h1:qWE1icOH4AKXRcq1KCzt6P/TesqptgBTP++V7wowTc0= github.com/lightningnetwork/lightning-onion v1.4.0/go.mod h1:YDPkvVTVQ6FBBE6Yj93tDd7zA3iTSrryi9xq46i7bKE= -github.com/lightningnetwork/lnd v0.21.0-beta.rc2.0.20260630214209-40c64f9db30d h1:mDcB9mwuwJXRbDmxps6mBgGs318Iz5ZbOrt0ieEx0os= -github.com/lightningnetwork/lnd v0.21.0-beta.rc2.0.20260630214209-40c64f9db30d/go.mod h1:LWfQaNNXlZghUJqIIh+JsgfxiVtC+ecZkrIe5bm9o4U= github.com/lightningnetwork/lnd/actor v0.0.6 h1:Ge8N2wivARG+27qJBwTlB0vwsypStZYZy8vk4Zl38sU= github.com/lightningnetwork/lnd/actor v0.0.6/go.mod h1:YAsoniSbY/cAM9HTVNfZLvt7RI6swDxy6wzPspTcMZg= github.com/lightningnetwork/lnd/cert v1.2.2 h1:71YK6hogeJtxSxw2teq3eGeuy4rHGKcFf0d0Uy4qBjI= diff --git a/harness/harness.go b/harness/harness.go index a5c9a3cd5..12c5f4258 100644 --- a/harness/harness.go +++ b/harness/harness.go @@ -9,6 +9,7 @@ import ( "bytes" "context" crand "crypto/rand" + "encoding/hex" "encoding/json" "flag" "fmt" @@ -26,8 +27,13 @@ import ( "testing" "time" + btcaddr "github.com/btcsuite/btcd/address/v2" "github.com/btcsuite/btcd/btcutil/v2" + "github.com/btcsuite/btcd/chaincfg/v2" + "github.com/btcsuite/btcd/chainhash/v2" "github.com/btcsuite/btcd/rpcclient" + "github.com/btcsuite/btcd/txscript/v2" + "github.com/btcsuite/btcd/wire/v2" "github.com/lightninglabs/darepo-client/chain" "github.com/lightninglabs/lndclient" "github.com/lightninglabs/taproot-assets/taprpc" @@ -1773,6 +1779,169 @@ func (h *Harness) BlockHeader(hash string) BlockHeader { return hdr } +// GetRawTransaction fetches the raw transaction body for txid from +// bitcoind and deserializes it into a wire.MsgTx. The transaction must +// exist either in the current chain or in bitcoind's mempool; +// `getrawtransaction` returns it via the wallet's lookup path in either +// case for a faucet-sent tx, so callers do not need to mine first. +func (h *Harness) GetRawTransaction(txid string) *wire.MsgTx { + h.T.Helper() + + res, err := h.bitcoinRPCCall("getrawtransaction", txid) + require.NoError(h.T, err, "getrawtransaction rpc failed") + + var hexStr string + err = json.Unmarshal(res, &hexStr) + require.NoError(h.T, err, "getrawtransaction unmarshal failed") + + rawBytes, err := hex.DecodeString(hexStr) + require.NoError(h.T, err, "decode raw tx hex failed") + + tx := wire.NewMsgTx(2) + err = tx.Deserialize(bytes.NewReader(rawBytes)) + require.NoError(h.T, err, "deserialize raw tx failed") + + return tx +} + +// SignedV3Tx builds a TRUC (v3) transaction that sends `amount` to +// `destPkScript`, uses one of bitcoind's wallet UTXOs as input, and +// returns it fully signed but NOT broadcast. The change output goes +// back to a fresh address from bitcoind's wallet. +// +// This helper exists for reorg systests that need to feed a v3 tx into +// txconfirm: bitcoind's `sendtoaddress` faucet path produces v2 txs, +// and txconfirm's CPFP broadcaster enforces v3/TRUC at the version +// gate. Building the v3 tx ourselves and letting txconfirm broadcast +// it is the cleanest way to exercise the full tracked-tx lifecycle on +// a freshly-mined chain entry. +// +// Fee rate is fixed at 5 sat/vB — well above any regtest min-relay-fee +// floor and small enough that the leftover change is reusable. +func (h *Harness) SignedV3Tx(destPkScript []byte, + amount btcutil.Amount) *wire.MsgTx { + + h.T.Helper() + + h.bitcoindEnsureWallet() + + // Find a confirmed wallet UTXO with enough value to cover the + // destination + change + a generous fee. + utxoTxid, utxoVout, utxoValueBTC := h.bitcoindFirstSpendableUTXO() + utxoValue := btcutil.Amount(utxoValueBTC * btcutil.SatoshiPerBitcoin) + + feeSat := btcutil.Amount(2_000) // ~5 sat/vB on a ~400 vB tx. + require.Greater( + h.T, utxoValue, amount+feeSat, + "selected UTXO not large enough for destination + fee", + ) + changeSat := utxoValue - amount - feeSat + + // Fresh change address. + changeAddrRes, err := h.bitcoinRPCCall("getnewaddress") + require.NoError(h.T, err, "getnewaddress for change failed") + var changeAddrStr string + require.NoError( + h.T, json.Unmarshal(changeAddrRes, &changeAddrStr), + "getnewaddress unmarshal failed", + ) + changeAddr, err := btcaddr.DecodeAddress( + changeAddrStr, &chaincfg.RegressionNetParams, + ) + require.NoError(h.T, err, "decode change address failed") + changePkScript, err := txscript.PayToAddrScript(changeAddr) + require.NoError(h.T, err, "derive change pkScript failed") + + // Build the unsigned v3 tx. + tx := wire.NewMsgTx(3) + prevHash, err := chainhash.NewHashFromStr(utxoTxid) + require.NoError(h.T, err, "parse selected UTXO txid") + tx.AddTxIn(&wire.TxIn{ + PreviousOutPoint: wire.OutPoint{ + Hash: *prevHash, Index: utxoVout, + }, + Sequence: wire.MaxTxInSequenceNum - 1, + }) + tx.AddTxOut(&wire.TxOut{ + Value: int64(amount), + PkScript: destPkScript, + }) + tx.AddTxOut(&wire.TxOut{ + Value: int64(changeSat), + PkScript: changePkScript, + }) + + // Hex-encode and ask bitcoind to sign. + var unsignedBuf bytes.Buffer + require.NoError( + h.T, tx.Serialize(&unsignedBuf), + "serialize unsigned v3 tx", + ) + unsignedHex := hex.EncodeToString(unsignedBuf.Bytes()) + + signRes, err := h.bitcoinRPCCall( + "signrawtransactionwithwallet", unsignedHex, + ) + require.NoError(h.T, err, "signrawtransactionwithwallet failed") + + var signResult struct { + Hex string `json:"hex"` + Complete bool `json:"complete"` + } + require.NoError( + h.T, json.Unmarshal(signRes, &signResult), + "signrawtransactionwithwallet unmarshal failed", + ) + require.True( + h.T, signResult.Complete, "tx signing incomplete: %s", + signResult.Hex, + ) + + signedBytes, err := hex.DecodeString(signResult.Hex) + require.NoError(h.T, err, "decode signed v3 tx hex failed") + + signed := wire.NewMsgTx(3) + require.NoError( + h.T, + signed.Deserialize( + bytes.NewReader(signedBytes), + ), + "deserialize signed v3 tx failed", + ) + require.Equal( + h.T, int32(3), signed.Version, + "signing must preserve v3 version", + ) + + return signed +} + +// bitcoindFirstSpendableUTXO picks the first confirmed wallet UTXO via +// `listunspent` and returns its txid, vout, and amount in BTC. +func (h *Harness) bitcoindFirstSpendableUTXO() (string, uint32, float64) { + h.T.Helper() + + // Restrict to confirmed and spendable; bitcoind defaults are + // already minconf=1, maxconf=9999999. + res, err := h.bitcoinRPCCall("listunspent") + require.NoError(h.T, err, "listunspent rpc failed") + + var utxos []struct { + Txid string `json:"txid"` + Vout uint32 `json:"vout"` + Amount float64 `json:"amount"` + } + require.NoError( + h.T, json.Unmarshal(res, &utxos), + "listunspent unmarshal failed", + ) + require.NotEmpty(h.T, utxos, "no spendable wallet UTXOs") + + first := utxos[0] + + return first.Txid, first.Vout, first.Amount +} + // Faucet funds a test address by sending the specified amount from bitcoind's // default wallet, creating unconfirmed UTXOs for tests to spend. This mimics // external funding without requiring manual transaction construction. diff --git a/lwwallet/chain_backend.go b/lwwallet/chain_backend.go index 573bec48d..442700ac4 100644 --- a/lwwallet/chain_backend.go +++ b/lwwallet/chain_backend.go @@ -16,6 +16,7 @@ import ( "github.com/btcsuite/btcd/wire/v2" "github.com/btcsuite/btclog/v2" "github.com/lightninglabs/darepo-client/chainsource" + fn "github.com/lightningnetwork/lnd/fn/v2" "golang.org/x/sync/singleflight" ) @@ -42,6 +43,29 @@ const ( // raise Esplora load over the default poll cadence. const recheckHeartbeatInterval = 60 * time.Second +// regState is the lifecycle state of a conf/spend registration. +// +// stateWatching: no positive event has been delivered yet (or the +// last positive event was reorged out). The next checkSingle... that +// finds the tx confirmed / outpoint spent will emit Confirmed / +// Spend and transition to statePositive. +// +// statePositive: a Confirmed / Spend event has been delivered and the +// associated block-hash is cached on the registration. The next +// checkSingle... that does NOT find the tx confirmed / outpoint +// spent in the same block leaves the registration alone (the +// chainsource actor handles Done synthesis at FinalityDepth via +// block epochs). A reorg event that names the cached block-hash in +// its Disconnected set fires Reorged, resets cached state, and +// transitions back to stateWatching so the next re-check can fire +// Confirmed / Spend again on the new chain. +type regState uint8 + +const ( + stateWatching regState = iota + statePositive +) + // confRegistration tracks a pending confirmation registration within the // polling loop. type confRegistration struct { @@ -63,8 +87,37 @@ type confRegistration struct { // confChan is the channel to send the confirmation on. confChan chan *chainsource.TxConfirmation + // reorgChan is the channel that fires when a previously delivered + // confirmation is reorged out of the canonical chain. Buffered to + // 1; never written by the backend before stateWatching has been + // re-entered. The value is the chainsource ordering sequence; this + // backend does not stamp sequences (it emits conf/reorg from a + // single ordered handler goroutine), so it always sends 0, which + // the chainsource actor treats as always-apply. + reorgChan chan uint64 + + // doneChan is allocated for API symmetry with the chainsource + // contract. The Esplora-backed backend does not write to it: the + // chainsource ConfActor synthesizes Done at FinalityDepth from + // block epochs once the backend stops re-firing events. + doneChan chan struct{} + // cancelCh signals that this registration has been cancelled. cancelCh chan struct{} + + // regMu guards the fields below that mutate over the + // registration's lifetime. It is held only briefly during state + // transitions; channel sends are performed without it. + regMu sync.Mutex + + // state is the current lifecycle state. + state regState + + // lastBlockHash is the BlockHash of the last delivered + // confirmation when state == statePositive. Used to detect when + // a reorg's Disconnected hash list invalidates this + // registration's last event. + lastBlockHash chainhash.Hash } // spendRegistration tracks a pending spend registration within the @@ -82,8 +135,34 @@ type spendRegistration struct { // spendChan is the channel to send the spend detail on. spendChan chan *chainsource.SpendDetail + // reorgChan is the channel that fires when a previously delivered + // spend is reorged out of the canonical chain. Buffered to 1. + reorgChan chan uint64 + + // doneChan is allocated for API symmetry with the chainsource + // contract. The Esplora-backed backend does not write to it: the + // chainsource SpendActor synthesizes Done at FinalityDepth. + doneChan chan struct{} + // cancelCh signals that this registration has been cancelled. cancelCh chan struct{} + + // regMu guards the fields below. + regMu sync.Mutex + + // state is the current lifecycle state. + state regState + + // lastSpenderHash is the SpenderTxHash of the last delivered + // spend when state == statePositive. + lastSpenderHash chainhash.Hash + + // lastSpendingBlockHash is the block hash that the last + // delivered spend confirmed in, parsed from the same + // outspend response that confirmed the spend. Reorg + // comparison against ReorgEvent.Disconnected uses this so + // the spend-watch path does not need an extra HTTP round-trip. + lastSpendingBlockHash chainhash.Hash } // blockRegistration tracks a block epoch subscription. @@ -205,7 +284,8 @@ func (b *ChainBackend) Start() error { } } - height, hash, _, sub, err := b.tipPoller.BestBlockAndSubscribe() + height, hash, _, chainSub, err := + b.tipPoller.BestBlockAndSubscribeChain() if err != nil { return fmt.Errorf("subscribe to tip poller: %w", err) } @@ -216,7 +296,7 @@ func (b *ChainBackend) Start() error { b.mu.Unlock() b.wg.Add(1) - go b.handleTipEvents(sub) + go b.handleChainEvents(chainSub) b.log.InfoS(context.Background(), "Chain backend started", slog.Int("tip_height", int(height)), @@ -388,11 +468,20 @@ func (b *ChainBackend) SubmitPackage(ctx context.Context, parents []*wire.MsgTx, } // RegisterConf registers for confirmation notifications of a transaction. +// The registration is reorg-aware: the returned ConfRegistration's +// Reorged channel fires when a previously delivered confirmation is +// reorged out of the canonical chain, and the Confirmed channel may +// fire again when the tx re-confirms on the new chain. The Done +// channel is allocated but never written by this backend; the +// chainsource ConfActor synthesizes Done at its configured +// FinalityDepth using block epochs. func (b *ChainBackend) RegisterConf(ctx context.Context, txid *chainhash.Hash, pkScript []byte, numConfs uint32, heightHint uint32, includeBlock bool) (*chainsource.ConfRegistration, error) { confChan := make(chan *chainsource.TxConfirmation, 1) + reorgChan := make(chan uint64, 1) + doneChan := make(chan struct{}, 1) cancelCh := make(chan struct{}) reg := &confRegistration{ @@ -402,6 +491,8 @@ func (b *ChainBackend) RegisterConf(ctx context.Context, txid *chainhash.Hash, heightHint: heightHint, includeBlock: includeBlock, confChan: confChan, + reorgChan: reorgChan, + doneChan: doneChan, cancelCh: cancelCh, } @@ -424,11 +515,7 @@ func (b *ChainBackend) RegisterConf(ctx context.Context, txid *chainhash.Hash, ) cancelFn := func() { - close(cancelCh) - - b.mu.Lock() - delete(b.confRegs, id) - b.mu.Unlock() + b.cancelConfReg(id, reg) } // Run an immediate single-shot check scoped to JUST this @@ -450,16 +537,41 @@ func (b *ChainBackend) RegisterConf(ctx context.Context, txid *chainhash.Hash, return &chainsource.ConfRegistration{ Confirmed: confChan, + Reorged: reorgChan, + Done: doneChan, Cancel: cancelFn, }, nil } +// cancelConfReg tears down a confirmation registration. It is safe +// to invoke from any state; the close(cancelCh) is guarded by a +// once-style check to make double-Cancel a no-op. +func (b *ChainBackend) cancelConfReg(id uint64, reg *confRegistration) { + reg.regMu.Lock() + select { + case <-reg.cancelCh: + // Already cancelled. + reg.regMu.Unlock() + + return + + default: + } + close(reg.cancelCh) + reg.regMu.Unlock() + + b.mu.Lock() + delete(b.confRegs, id) + b.mu.Unlock() +} + // runConfOneShot performs the per-registration confirmation check // triggered at RegisterConf time. It snapshots the current best // height under the chain backend's lock, asks Esplora for this one // registration's status, and delivers the result if confirmed. The -// goroutine exits on any of: cancellation, stopCh closure, a -// successful delivery, or a non-confirmed status. +// registration is NOT deleted after delivery: a reorg may later +// reset it to stateWatching and the next re-check must fire +// Confirmed again on the new chain. func (b *ChainBackend) runConfOneShot(id uint64, reg *confRegistration) { defer b.wg.Done() @@ -477,11 +589,49 @@ func (b *ChainBackend) runConfOneShot(id uint64, reg *confRegistration) { currentHeight := b.bestHeight b.mu.Unlock() + b.deliverConfIfNew(id, reg, currentHeight) +} + +// deliverConfIfNew runs a confirmation re-check and fires the +// Confirmed channel iff the registration is in stateWatching and a +// confirmation is now available. It is the single delivery path +// shared by the one-shot at registration time and the broad +// re-check driven by tip / reorg events. The registration's regMu +// is held only across the state read and the state transition; the +// channel send happens outside the lock so a slow consumer never +// blocks the broad re-check goroutine. +func (b *ChainBackend) deliverConfIfNew(id uint64, reg *confRegistration, + currentHeight int32) { + + reg.regMu.Lock() + if reg.state == statePositive { + reg.regMu.Unlock() + + return + } + reg.regMu.Unlock() + conf := b.checkSingleConf(reg, currentHeight) if conf == nil { return } + // Re-check state under the lock: a concurrent reorg handler + // could have flipped us back to stateWatching after the check + // started, but it could not have flipped us forward to + // statePositive (that's exclusively this function's job). + reg.regMu.Lock() + if reg.state == statePositive { + reg.regMu.Unlock() + + return + } + reg.state = statePositive + if conf.BlockHash != nil { + reg.lastBlockHash = *conf.BlockHash + } + reg.regMu.Unlock() + select { case reg.confChan <- conf: case <-reg.cancelCh: @@ -493,22 +643,26 @@ func (b *ChainBackend) runConfOneShot(id uint64, reg *confRegistration) { b.log.DebugS( context.Background(), - "Confirmation registration fulfilled (one-shot)", + "Confirmation registration fulfilled", slog.Uint64("reg_id", id), slog.Int("block_height", int(conf.BlockHeight)), ) - - b.mu.Lock() - delete(b.confRegs, id) - b.mu.Unlock() } // RegisterSpend registers for spend notifications of a transaction output. +// The registration is reorg-aware: the returned SpendRegistration's +// Reorged channel fires when a previously delivered spend is reorged +// out of the canonical chain, and the Spend channel may fire again +// when the outpoint is re-spent on the new chain. The Done channel +// is allocated but never written by this backend; the chainsource +// SpendActor synthesizes Done at its configured FinalityDepth. func (b *ChainBackend) RegisterSpend(ctx context.Context, outpoint *wire.OutPoint, pkScript []byte, heightHint uint32) ( *chainsource.SpendRegistration, error) { spendChan := make(chan *chainsource.SpendDetail, 1) + reorgChan := make(chan uint64, 1) + doneChan := make(chan struct{}, 1) cancelCh := make(chan struct{}) reg := &spendRegistration{ @@ -516,6 +670,8 @@ func (b *ChainBackend) RegisterSpend(ctx context.Context, pkScript: pkScript, heightHint: heightHint, spendChan: spendChan, + reorgChan: reorgChan, + doneChan: doneChan, cancelCh: cancelCh, } @@ -537,11 +693,7 @@ func (b *ChainBackend) RegisterSpend(ctx context.Context, ) cancelFn := func() { - close(cancelCh) - - b.mu.Lock() - delete(b.spendRegs, id) - b.mu.Unlock() + b.cancelSpendReg(id, reg) } // Per-registration one-shot to handle outpoints that are @@ -552,16 +704,37 @@ func (b *ChainBackend) RegisterSpend(ctx context.Context, go b.runSpendOneShot(id, reg) return &chainsource.SpendRegistration{ - Spend: spendChan, - Cancel: cancelFn, + Spend: spendChan, + Reorged: reorgChan, + Done: doneChan, + Cancel: cancelFn, }, nil } +// cancelSpendReg tears down a spend registration. Idempotent: a +// double-Cancel is treated as a no-op. +func (b *ChainBackend) cancelSpendReg(id uint64, reg *spendRegistration) { + reg.regMu.Lock() + select { + case <-reg.cancelCh: + reg.regMu.Unlock() + + return + + default: + } + close(reg.cancelCh) + reg.regMu.Unlock() + + b.mu.Lock() + delete(b.spendRegs, id) + b.mu.Unlock() +} + // runSpendOneShot performs the per-registration spend check -// triggered at RegisterSpend time. It exits on cancellation, stopCh -// closure, a successful delivery, or any non-spent / unconfirmed -// status; the broad checkSpends called from processTipEvent re-runs -// it on every tip advance. +// triggered at RegisterSpend time. The registration is NOT deleted +// after delivery: a reorg may later reset it to stateWatching and +// the next re-check must fire Spend again on the new chain. func (b *ChainBackend) runSpendOneShot(id uint64, reg *spendRegistration) { defer b.wg.Done() @@ -579,11 +752,46 @@ func (b *ChainBackend) runSpendOneShot(id uint64, reg *spendRegistration) { return } - detail := b.checkSingleSpend(reg) + b.deliverSpendIfNew(id, reg) +} + +// deliverSpendIfNew runs a spend re-check and fires the Spend +// channel iff the registration is in stateWatching and a spend is +// now available. Mirror of deliverConfIfNew; see that function's +// comment for the regMu / channel-send ordering rationale. +func (b *ChainBackend) deliverSpendIfNew(id uint64, reg *spendRegistration) { + reg.regMu.Lock() + if reg.state == statePositive { + reg.regMu.Unlock() + + return + } + reg.regMu.Unlock() + + // checkSingleSpend returns the spending block hash alongside + // the detail (parsed from the same /outspend response that + // confirmed the spend). A zero hash here means the response + // did not parse and reorgSpendReg will fall back to a + // conservative re-check rather than try to match against + // ReorgEvent.Disconnected. + detail, spendingBlockHash := b.checkSingleSpend(reg) if detail == nil { return } + reg.regMu.Lock() + if reg.state == statePositive { + reg.regMu.Unlock() + + return + } + reg.state = statePositive + if detail.SpenderTxHash != nil { + reg.lastSpenderHash = *detail.SpenderTxHash + } + reg.lastSpendingBlockHash = spendingBlockHash + reg.regMu.Unlock() + select { case reg.spendChan <- detail: case <-reg.cancelCh: @@ -594,15 +802,11 @@ func (b *ChainBackend) runSpendOneShot(id uint64, reg *spendRegistration) { } b.log.DebugS(context.Background(), - "Spend registration fulfilled (one-shot)", + "Spend registration fulfilled", slog.Uint64("reg_id", id), slog.String("outpoint", reg.outpoint.String()), slog.String("spender_txid", detail.SpenderTxHash.String())) - - b.mu.Lock() - delete(b.spendRegs, id) - b.mu.Unlock() } // RegisterBlocks registers for new block notifications. @@ -637,17 +841,258 @@ func (b *ChainBackend) RegisterBlocks(_ context.Context) ( }, nil } -// handleTipEvents drains TipBlock events from the shared poller and -// translates them into chain backend work: emit a BlockEpoch to each -// block-registration subscriber, advance the cached tip, and re-check -// pending confirmation/spend registrations. -// -// On stopCh the loop exits and cancels its subscription so the -// poller does not waste effort fanning to a dead consumer. The -// subscription's Quit channel covers the inverse direction: if the -// poller is shut down externally, we exit promptly without waiting -// for stopCh. -func (b *ChainBackend) handleTipEvents(sub *TipSubscription) { +// processReorgEvent reconciles all active registrations with one +// ReorgEvent. It is invoked from the unified handleChainEvents +// goroutine in producer order: a ReorgEvent is fully processed +// (registration state reset, Reorged channels fired) before any +// subsequent TipBlock dispatches block epochs or runs broad +// re-check on the replacement chain. Registrations whose last +// positive event names a disconnected hash are reset and +// re-checked; all others are left alone (the broad tip-driven +// re-check still runs separately). +func (b *ChainBackend) processReorgEvent(event *ReorgEvent) { + if event == nil { + return + } + + b.log.InfoS(context.Background(), "Processing reorg", + slog.Int("fork_height", int(event.ForkHeight)), + slog.Int("disconnected", len(event.Disconnected)), + slog.Int("connected", len(event.Connected)), + ) + + disconnectedSet := make( + map[chainhash.Hash]struct{}, len(event.Disconnected), + ) + for _, hash := range event.Disconnected { + disconnectedSet[hash] = struct{}{} + } + + // Snapshot the registration sets under the chain backend + // lock so we don't hold it across the per-registration + // channel sends below. + b.mu.Lock() + confRegs := make(map[uint64]*confRegistration, len(b.confRegs)) + for id, reg := range b.confRegs { + confRegs[id] = reg + } + spendRegs := make(map[uint64]*spendRegistration, len(b.spendRegs)) + for id, reg := range b.spendRegs { + spendRegs[id] = reg + } + currentHeight := b.bestHeight + b.mu.Unlock() + + for id, reg := range confRegs { + b.reorgConfReg(id, reg, disconnectedSet, currentHeight) + } + for id, reg := range spendRegs { + b.reorgSpendReg(id, reg, disconnectedSet) + } +} + +// reorgConfReg processes one confirmation registration against a +// reorg event. If the registration is in statePositive and its +// last-known block hash is in disconnectedSet, fire Reorged, reset +// to stateWatching, and run a fresh check that may immediately +// fire Confirmed against the new chain. +func (b *ChainBackend) reorgConfReg(id uint64, reg *confRegistration, + disconnectedSet map[chainhash.Hash]struct{}, currentHeight int32) { + + select { + case <-reg.cancelCh: + return + + default: + } + + reg.regMu.Lock() + if reg.state != statePositive { + reg.regMu.Unlock() + + return + } + cachedHash := reg.lastBlockHash + reg.regMu.Unlock() + + // Fast path: the registration's cached block hash appears in + // the reorg event's disconnected set. This covers every + // reorg of a block we previously broadcast. + _, fastHit := disconnectedSet[cachedHash] + + if !fastHit { + // Fallback: the registration may have delivered against + // a block older than the poller's seeded hash history + // (e.g. on a fresh daemon where RegisterConf landed an + // immediate historical positive). The reorg's + // disconnected set is bounded by recentHashes, so a + // reorg deep enough to invalidate that historical block + // would not appear here. Re-query canonical status and + // compare against the cached block hash. + // + // confirmedBlockHash returns a tri-state so we never + // fire a spurious reorg: Some(h) means the tx is still + // confirmed in block h (independent of the numConfs + // threshold), None means it is definitively unconfirmed, + // and a non-nil error means canonical status could not + // be determined right now. + gotHash, err := b.confirmedBlockHash(reg) + switch { + // Transient backend failure: we cannot tell whether this + // is a reorg. Leave the registration in statePositive and + // bail; a later tip or reorg event re-evaluates once the + // backend recovers. Firing Reorged here would strand the + // consumer on a false alarm. + case err != nil: + b.log.DebugS( + context.Background(), + "Skipping conf reorg eval; canonical "+ + "status undeterminable", + slog.Uint64("reg_id", id), + btclog.Fmt("err", "%v", err), + ) + + return + + // Still confirmed in the same block: definitively not a + // reorg of this registration. + case gotHash.IsSome() && + gotHash.UnsafeFromSome() == cachedHash: + return + } + + // Otherwise the tx is confirmed elsewhere or definitively + // unconfirmed: fall through and fire Reorged. + } + + reg.regMu.Lock() + if reg.state != statePositive { + reg.regMu.Unlock() + + return + } + reg.state = stateWatching + reg.lastBlockHash = chainhash.Hash{} + reg.regMu.Unlock() + + // Fire Reorged non-blocking: the channel is buffered to 1 + // and the consumer (chainsource conf actor) is single- + // threaded so a missed coalesced reorg is correct — the + // consumer will re-query state anyway. + select { + case reg.reorgChan <- uint64(0): + case <-reg.cancelCh: + return + + default: + b.log.DebugS( + context.Background(), + "Conf reorged signal coalesced", + slog.Uint64("reg_id", id), + ) + } + + b.log.InfoS( + context.Background(), + "Conf registration reorged; re-checking", + slog.Uint64("reg_id", id), + ) + + // Re-check now so a re-confirmation on the new chain fires + // Confirmed in the same reorg-handler turn. + b.deliverConfIfNew(id, reg, currentHeight) +} + +// reorgSpendReg is the spend-side equivalent of reorgConfReg. +func (b *ChainBackend) reorgSpendReg(id uint64, reg *spendRegistration, + disconnectedSet map[chainhash.Hash]struct{}) { + + select { + case <-reg.cancelCh: + return + + default: + } + + reg.regMu.Lock() + if reg.state != statePositive { + reg.regMu.Unlock() + + return + } + cachedBlockHash := reg.lastSpendingBlockHash + cachedSpenderHash := reg.lastSpenderHash + reg.regMu.Unlock() + + // Fast path: the cached spending-block hash appears in the + // reorg event's disconnected set. Empty cachedBlockHash + // (e.g. delivered before the outspend response was parseable) + // falls through to the fallback re-query rather than being + // treated as an automatic hit. + fastHit := false + if cachedBlockHash != (chainhash.Hash{}) { + _, fastHit = disconnectedSet[cachedBlockHash] + } + + if !fastHit { + // Fallback: re-query canonical chain. Covers (a) regs + // delivered against a block older than the poller's + // seeded hash history, and (b) regs whose + // cachedBlockHash is zero because the outspend response + // was unparseable at delivery time. If the outpoint is + // still spent by the same spender in the same block, + // no reorg for this registration. + current, currentBlock := b.checkSingleSpend(reg) + if current != nil && + current.SpenderTxHash != nil && + *current.SpenderTxHash == cachedSpenderHash && + cachedBlockHash != (chainhash.Hash{}) && + currentBlock == cachedBlockHash { + return + } + } + + reg.regMu.Lock() + if reg.state != statePositive { + reg.regMu.Unlock() + + return + } + reg.state = stateWatching + reg.lastSpenderHash = chainhash.Hash{} + reg.lastSpendingBlockHash = chainhash.Hash{} + reg.regMu.Unlock() + + select { + case reg.reorgChan <- uint64(0): + case <-reg.cancelCh: + return + + default: + b.log.DebugS( + context.Background(), + "Spend reorged signal coalesced", + slog.Uint64("reg_id", id), + ) + } + + b.log.InfoS( + context.Background(), + "Spend registration reorged; re-checking", + slog.Uint64("reg_id", id), + ) + + b.deliverSpendIfNew(id, reg) +} + +// handleChainEvents drains the unified chain stream and dispatches +// each event in producer order. Using a single subscription + single +// goroutine guarantees that a ReorgEvent is fully processed +// (registration state reset, Reorged channels fired) before any +// post-reorg TipBlock drives block-epoch dispatch or broad re-check +// — otherwise a block-epoch driven Confirmed could land on a stale +// registration before reorgConfReg has a chance to reset it. +func (b *ChainBackend) handleChainEvents(sub *ChainSubscription) { defer b.wg.Done() defer sub.Cancel() @@ -661,18 +1106,15 @@ func (b *ChainBackend) handleTipEvents(sub *TipSubscription) { return } - b.processTipEvent(event) + switch { + case event.Reorg != nil: + b.processReorgEvent(event.Reorg) + + case event.Tip != nil: + b.processTipEvent(event.Tip) + } case <-heartbeat.C: - // Re-run the broad checks even when the tip - // hasn't moved. Esplora's status indexer lags - // the block-found event by 1-3 seconds, so a - // processTipEvent that ran before the indexer - // caught up would otherwise not retry until - // the next block lands. Coalesced via the same - // singleflight keys used by processTipEvent so - // a tip event arriving on the same tick does - // not produce two parallel scans. b.runRecheckHeartbeat() case <-sub.Quit(): @@ -774,8 +1216,13 @@ func (b *ChainBackend) processTipEvent(event *TipBlock) { }) } -// checkConfirmations iterates over all pending confirmation registrations -// and checks their status via the Esplora API. +// checkConfirmations iterates over all pending confirmation +// registrations and re-checks their status via the Esplora API. +// Registrations already in statePositive are skipped: the reorg +// handler is responsible for transitioning them back to +// stateWatching, and an unconditional re-check would just burn +// Esplora calls for no behavior change (the chainsource actor's +// FinalityDepth synthesis handles Done from block epochs). func (b *ChainBackend) checkConfirmations() { b.mu.Lock() regs := make(map[uint64]*confRegistration, len(b.confRegs)) @@ -794,28 +1241,7 @@ func (b *ChainBackend) checkConfirmations() { default: } - conf := b.checkSingleConf(reg, currentHeight) - if conf == nil { - continue - } - - // Send the confirmation. - select { - case reg.confChan <- conf: - case <-reg.cancelCh: - continue - } - - b.log.DebugS(context.Background(), - "Confirmation registration fulfilled", - slog.Uint64("reg_id", id), - slog.Int("block_height", - int(conf.BlockHeight))) - - // Remove the fulfilled registration. - b.mu.Lock() - delete(b.confRegs, id) - b.mu.Unlock() + b.deliverConfIfNew(id, reg, currentHeight) } } @@ -948,8 +1374,73 @@ func (b *ChainBackend) checkConfByScript(reg *confRegistration, return nil } -// checkSpends iterates over all pending spend registrations and checks -// their status via the Esplora API. +// confirmedBlockHash reports the block hash a registration's transaction is +// currently confirmed in, independent of the registration's numConfs +// threshold. It exists for the reorg-detection fallback, which must +// distinguish three states that checkSingleConf collapses into a single nil: +// +// - Some(hash), nil: the tx is confirmed in block hash (any depth); +// - None, nil: the tx is definitively unconfirmed (genuine reorg); +// - None, err: canonical status could not be determined right now +// (transient backend error) — callers MUST NOT treat this as a reorg. +// +// Unlike checkSingleConf it never gates on numConfs, because a reorg that +// merely reduces a tx's confirmation depth without moving it out of its block +// is not a reorg of that registration. +func (b *ChainBackend) confirmedBlockHash(reg *confRegistration) ( + fn.Option[chainhash.Hash], error) { + + none := fn.None[chainhash.Hash]() + + // Explicit-txid registrations resolve via a direct status lookup. + if reg.txid != nil { + status, err := b.esplora.GetTxStatus( + context.Background(), *reg.txid, + ) + if err != nil { + return none, err + } + if !status.Confirmed { + return none, nil + } + + hash, err := chainhash.NewHashFromStr(status.BlockHash) + if err != nil { + return none, err + } + + return fn.Some(*hash), nil + } + + // Script registrations resolve via the first confirmed UTXO paying + // the watched script. + utxos, err := b.esplora.GetScriptUtxos( + context.Background(), reg.pkScript, + ) + if err != nil { + return none, err + } + + for _, utxo := range utxos { + if !utxo.Status.Confirmed { + continue + } + + hash, err := chainhash.NewHashFromStr(utxo.Status.BlockHash) + if err != nil { + return none, err + } + + return fn.Some(*hash), nil + } + + return none, nil +} + +// checkSpends iterates over all pending spend registrations and +// re-checks their status via the Esplora API. Registrations in +// statePositive are skipped — see checkConfirmations for the +// rationale. func (b *ChainBackend) checkSpends() { b.mu.Lock() regs := make(map[uint64]*spendRegistration, len(b.spendRegs)) @@ -971,68 +1462,59 @@ func (b *ChainBackend) checkSpends() { continue } - detail := b.checkSingleSpend(reg) - if detail == nil { - continue - } - - // Send the spend detail. - select { - case reg.spendChan <- detail: - case <-reg.cancelCh: - continue - } - - b.log.DebugS(context.Background(), - "Spend registration fulfilled", - slog.Uint64("reg_id", id), - slog.String("outpoint", - reg.outpoint.String()), - slog.String( - "spender_txid", detail.SpenderTxHash.String(), - )) - - // Remove the fulfilled registration. - b.mu.Lock() - delete(b.spendRegs, id) - b.mu.Unlock() + b.deliverSpendIfNew(id, reg) } } // checkSingleSpend resolves the spend status of a single spend // registration via Esplora. Returns nil when the outpoint is not yet // confirmed-spent, when any HTTP / parse error occurs, or when the -// registration has no outpoint. The caller is responsible for +// registration has no outpoint. The second return value is the block +// hash containing the spending tx (parsed from outspend.Status); the +// zero hash indicates the spending block hash was unparseable but the +// spend itself is still valid. The caller is responsible for // delivery, logging, and removing the fulfilled registration; this // helper only resolves the on-chain question. -func (b *ChainBackend) checkSingleSpend( - reg *spendRegistration) *chainsource.SpendDetail { +func (b *ChainBackend) checkSingleSpend(reg *spendRegistration) ( + *chainsource.SpendDetail, chainhash.Hash) { if reg.outpoint == nil { - return nil + return nil, chainhash.Hash{} } outspend, err := b.esplora.GetOutspend( context.Background(), reg.outpoint.Hash, reg.outpoint.Index, ) if err != nil { - return nil + return nil, chainhash.Hash{} } if !outspend.Spent || !outspend.Status.Confirmed { - return nil + return nil, chainhash.Hash{} } spenderHash, err := chainhash.NewHashFromStr(outspend.Txid) if err != nil { - return nil + return nil, chainhash.Hash{} } spendingTx, err := b.esplora.GetRawTx( context.Background(), *spenderHash, ) if err != nil { - return nil + return nil, chainhash.Hash{} + } + + // outspend.Status.BlockHash is hex from the same /outspend + // response that confirmed the spend; an unparseable value is + // not fatal — the spend itself is valid and the reorg path can + // fall back to a conservative re-check (see reorgSpendReg). + var spendingBlockHash chainhash.Hash + if outspend.Status.BlockHash != "" { + h, err := chainhash.NewHashFromStr(outspend.Status.BlockHash) + if err == nil { + spendingBlockHash = *h + } } return &chainsource.SpendDetail{ @@ -1041,7 +1523,7 @@ func (b *ChainBackend) checkSingleSpend( SpendingTx: spendingTx, SpenderInputIndex: outspend.Vin, SpendingHeight: int32(outspend.Status.BlockHeight), - } + }, spendingBlockHash } // Compile-time check that ChainBackend implements diff --git a/lwwallet/chain_backend_reorg_test.go b/lwwallet/chain_backend_reorg_test.go new file mode 100644 index 000000000..593138b38 --- /dev/null +++ b/lwwallet/chain_backend_reorg_test.go @@ -0,0 +1,1136 @@ +package lwwallet + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "runtime" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/btcsuite/btcd/chainhash/v2" + "github.com/btcsuite/btcd/wire/v2" + "github.com/btcsuite/btclog/v2" + "github.com/lightninglabs/darepo-client/chainsource" + "github.com/stretchr/testify/require" +) + +// reorgTestTimeout is the per-step wait used by the lwwallet reorg +// tests. Generous enough to absorb scheduling jitter on overloaded +// CI machines but short enough that a hung backend surfaces as a +// fast failure. +const reorgTestTimeout = 3 * time.Second + +// fakeChain is a mutable chain fixture for the reorg tests. Each +// height maps to a block whose hash, raw header, and contents the +// embedded HTTP handler will serve. Hashes are computable so the +// EsploraClient's content-hash verification passes. +// +// fakeChain is intentionally orthogonal to stubChain (used by the +// existing tip-poller tests): it supports same-height hash +// replacement (reorg), per-tx status overrides, per-outpoint +// outspend overrides, and serves /block//header so the tip +// poller's continuity check can resolve PrevBlock. +type fakeChain struct { + t *testing.T + mu sync.Mutex + + // tip is the current tip height. + tip int32 + + // blocks holds the per-height block model. Replacing the entry + // at height h simulates a same-height reorg; appending new + // heights simulates chain advance. + blocks map[int32]*fakeBlock + + // txStatus is the response to /tx//status. + txStatus map[chainhash.Hash]esploraTxStatus + + // rawTx is the response to /tx//raw. + rawTx map[chainhash.Hash][]byte + + // outspends is the response to /tx//outspend/. + outspends map[wire.OutPoint]esploraOutspend + + // failRawHeader holds block hashes for which the + // /block//header endpoint should return 500. Used to + // simulate a transient Esplora flake during continuity + // checking. + failRawHeader map[chainhash.Hash]struct{} +} + +// fakeBlock describes one height's block in the fakeChain. The hash +// is derived from a synthetic 80-byte header so the EsploraClient's +// content-hash verification accepts the raw-header response. +type fakeBlock struct { + height int32 + hash chainhash.Hash + prevHash chainhash.Hash + header *wire.BlockHeader + timestamp int64 +} + +// newFakeChain seeds a fakeChain with a single block at the given +// tip height. tag is mixed into the block hash so independent +// fakeChains in parallel tests produce distinct hashes. +func newFakeChain(t *testing.T, tip int32, tag string) *fakeChain { + t.Helper() + + c := &fakeChain{ + t: t, + tip: tip, + blocks: make(map[int32]*fakeBlock), + txStatus: make(map[chainhash.Hash]esploraTxStatus), + rawTx: make(map[chainhash.Hash][]byte), + outspends: make(map[wire.OutPoint]esploraOutspend), + failRawHeader: make(map[chainhash.Hash]struct{}), + } + + c.blocks[tip] = c.mintBlock(tip, chainhash.Hash{}, tag) + + return c +} + +// mintBlock builds a fakeBlock at height with the given prev hash +// and a tag that varies the resulting block hash. We synthesize a +// minimal 80-byte header with a unique nonce so BlockHash() varies +// reliably across invocations. +func (c *fakeChain) mintBlock(height int32, prev chainhash.Hash, + tag string) *fakeBlock { + + c.t.Helper() + + hdr := &wire.BlockHeader{ + Version: 1, + PrevBlock: prev, + MerkleRoot: chainhash.HashH([]byte(tag + "-merkle")), + Timestamp: time.Unix(int64(height)*600, 0), + Bits: 0x207fffff, + Nonce: uint32(height) ^ + uint32(chainhash.HashH([]byte(tag)).String()[0])<<16, + } + + // Make the nonce truly unique per (height, tag) so two + // adjacent tags do not collide on PrevBlock=zero replays. + salt := chainhash.HashH([]byte(fmt.Sprintf("%d-%s", height, tag))) + hdr.Nonce = uint32(salt[0])<<24 | uint32(salt[1])<<16 | + uint32(salt[2])<<8 | uint32(salt[3]) + + return &fakeBlock{ + height: height, + hash: hdr.BlockHash(), + prevHash: prev, + header: hdr, + timestamp: hdr.Timestamp.Unix(), + } +} + +// replaceTip swaps in a brand-new block at the current tip height +// (same height, different hash) and returns the new block. Used to +// drive a same-height reorg. +func (c *fakeChain) replaceTip(tag string) *fakeBlock { + c.mu.Lock() + defer c.mu.Unlock() + + prev, ok := c.blocks[c.tip-1] + var prevHash chainhash.Hash + if ok { + prevHash = prev.hash + } + + blk := c.mintBlock(c.tip, prevHash, tag) + c.blocks[c.tip] = blk + + return blk +} + +// extend appends a new block on top of the current tip and returns +// it. tag varies the resulting hash for parallel tests. +func (c *fakeChain) extend(tag string) *fakeBlock { + c.mu.Lock() + defer c.mu.Unlock() + + tipBlk := c.blocks[c.tip] + blk := c.mintBlock(c.tip+1, tipBlk.hash, tag) + c.tip++ + c.blocks[c.tip] = blk + + return blk +} + +// rewriteFrom rebuilds the chain from height start upward, giving +// each new block a different hash than the previous occupant at +// the same height. Used to simulate a deeper reorg where multiple +// heights diverge at once. +func (c *fakeChain) rewriteFrom(start int32, tagPrefix string) { + c.mu.Lock() + defer c.mu.Unlock() + + var prev chainhash.Hash + if prior, ok := c.blocks[start-1]; ok { + prev = prior.hash + } + for h := start; h <= c.tip; h++ { + blk := c.mintBlock(h, prev, + fmt.Sprintf("%s-%d", tagPrefix, h)) + c.blocks[h] = blk + prev = blk.hash + } +} + +// setTxStatus pins the response for /tx//status. +func (c *fakeChain) setTxStatus(txid chainhash.Hash, status esploraTxStatus) { + c.mu.Lock() + defer c.mu.Unlock() + + c.txStatus[txid] = status +} + +// setRawTx pins the response for /tx//raw. +func (c *fakeChain) setRawTx(txid chainhash.Hash, raw []byte) { + c.mu.Lock() + defer c.mu.Unlock() + + c.rawTx[txid] = raw +} + +// setOutspend pins the response for /tx//outspend/. +func (c *fakeChain) setOutspend(op wire.OutPoint, outspend esploraOutspend) { + c.mu.Lock() + defer c.mu.Unlock() + + c.outspends[op] = outspend +} + +// handler returns an http.HandlerFunc that serves the routes the +// chain backend / tip poller exercise. Anything else returns 404. +func (c *fakeChain) handler() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + c.mu.Lock() + defer c.mu.Unlock() + + path := r.URL.Path + switch { + case path == "/blocks/tip/height": + _, _ = fmt.Fprint(w, c.tip) + + case path == "/blocks/tip/hash": + blk, ok := c.blocks[c.tip] + if !ok { + http.Error(w, "no tip", http.StatusNotFound) + + return + } + + _, _ = fmt.Fprint(w, blk.hash.String()) + + case strings.HasPrefix(path, "/block-height/"): + heightStr := strings.TrimPrefix( + path, "/block-height/", + ) + height, err := strconv.ParseInt(heightStr, 10, 32) + if err != nil { + http.Error( + w, "bad height", http.StatusBadRequest, + ) + + return + } + blk, ok := c.blocks[int32(height)] + if !ok { + http.Error(w, "not found", + http.StatusNotFound) + + return + } + + _, _ = fmt.Fprint(w, blk.hash.String()) + + case strings.HasPrefix(path, "/block/"): + c.serveBlockReq(w, r, path) + + case strings.HasPrefix(path, "/tx/"): + c.serveTxReq(w, r, path) + + default: + http.Error(w, "not found", http.StatusNotFound) + } + } +} + +// serveBlockReq handles /block/ and /block//header. +// Caller holds c.mu. +func (c *fakeChain) serveBlockReq(w http.ResponseWriter, _ *http.Request, + path string) { + + rest := strings.TrimPrefix(path, "/block/") + hashStr := rest + suffix := "" + if idx := strings.Index(rest, "/"); idx >= 0 { + hashStr = rest[:idx] + suffix = rest[idx:] + } + + hash, err := chainhash.NewHashFromStr(hashStr) + if err != nil { + http.Error(w, "bad hash", http.StatusBadRequest) + + return + } + + var found *fakeBlock + for _, b := range c.blocks { + if b.hash == *hash { + found = b + + break + } + } + if found == nil { + http.Error(w, "not found", http.StatusNotFound) + + return + } + + switch suffix { + case "": + // JSON header. + _, _ = fmt.Fprintf( + w, `{"id":%q,"height":%d,"timestamp":%d}`, + found.hash.String(), found.height, found.timestamp, + ) + + case "/header": + // Raw 80-byte header, hex-encoded. + if _, fail := c.failRawHeader[found.hash]; fail { + http.Error( + w, "raw header unavailable", + http.StatusInternalServerError, + ) + + return + } + var buf bytes.Buffer + require.NoError(c.t, found.header.Serialize(&buf)) + _, _ = fmt.Fprint(w, hex.EncodeToString(buf.Bytes())) + + default: + http.Error(w, "not implemented", + http.StatusNotImplemented) + } +} + +// serveTxReq handles /tx//status, /tx//raw, and +// /tx//outspend/. Caller holds c.mu. +func (c *fakeChain) serveTxReq(w http.ResponseWriter, _ *http.Request, + path string) { + + rest := strings.TrimPrefix(path, "/tx/") + parts := strings.SplitN(rest, "/", 3) + if len(parts) < 2 { + http.Error(w, "not found", http.StatusNotFound) + + return + } + + txid, err := chainhash.NewHashFromStr(parts[0]) + if err != nil { + http.Error(w, "bad txid", http.StatusBadRequest) + + return + } + + switch parts[1] { + case "status": + status, ok := c.txStatus[*txid] + if !ok { + // Default to unconfirmed when no override exists. + status = esploraTxStatus{Confirmed: false} + } + + err := json.NewEncoder(w).Encode(status) + require.NoError(c.t, err) + + case "raw": + raw, ok := c.rawTx[*txid] + if !ok { + http.Error(w, "not found", + http.StatusNotFound) + + return + } + _, err := w.Write(raw) + require.NoError(c.t, err) + + case "outspend": + if len(parts) < 3 { + http.Error(w, "not found", + http.StatusNotFound) + + return + } + vout, err := strconv.ParseUint(parts[2], 10, 32) + if err != nil { + http.Error(w, "bad vout", + http.StatusBadRequest) + + return + } + op := wire.OutPoint{ + Hash: *txid, Index: uint32(vout), + } + outspend, ok := c.outspends[op] + if !ok { + outspend = esploraOutspend{Spent: false} + } + err = json.NewEncoder(w).Encode(outspend) + require.NoError(c.t, err) + + default: + http.Error(w, "not found", http.StatusNotFound) + } +} + +// fakeChainServer wraps a fakeChain with an httptest.Server for +// drop-in use by the backend tests. The server is auto-closed via +// t.Cleanup. +func fakeChainServer(t *testing.T, chain *fakeChain) *httptest.Server { + t.Helper() + + srv := httptest.NewServer(chain.handler()) + t.Cleanup(srv.Close) + + return srv +} + +// awaitConf reads one TxConfirmation with a deadline. +func awaitConf(t *testing.T, ch <-chan *chainsourceConf) *chainsourceConf { + t.Helper() + + select { + case c, ok := <-ch: + require.True(t, ok, "conf channel closed unexpectedly") + + return c + + case <-time.After(reorgTestTimeout): + t.Fatal("timeout waiting for confirmation") + + return nil + } +} + +// awaitSpend reads one SpendDetail with a deadline. +func awaitSpend(t *testing.T, ch <-chan *chainsourceSpend) *chainsourceSpend { + t.Helper() + + select { + case s, ok := <-ch: + require.True(t, ok, "spend channel closed unexpectedly") + + return s + + case <-time.After(reorgTestTimeout): + t.Fatal("timeout waiting for spend") + + return nil + } +} + +// awaitSeqSignal is awaitSignal for the sequence-carrying Reorged +// channel (chainsource retyped it from struct{} to the ordering +// sequence; this backend always sends 0). +func awaitSeqSignal(t *testing.T, ch <-chan uint64, label string) { + t.Helper() + + select { + case _, ok := <-ch: + require.True(t, ok, + "%s channel closed unexpectedly", label) + + case <-time.After(reorgTestTimeout): + t.Fatalf("timeout waiting for %s", label) + } +} + +// requireQuiet asserts that ch has no event for a short window. +// Used to ensure registrations are NOT double-firing. +func requireQuiet(t *testing.T, ch <-chan struct{}, label string, + dur time.Duration) { + + t.Helper() + + select { + case <-ch: + t.Fatalf("unexpected %s signal", label) + + case <-time.After(dur): + } +} + +// chainsourceConf / chainsourceSpend are type aliases to keep the +// test helper signatures short. +type ( + chainsourceConf = chainsource.TxConfirmation + chainsourceSpend = chainsource.SpendDetail +) + +// TestChainBackendSameHeightHashDrift verifies that a same-height +// hash replacement is detected by the tip poller and routed to the +// chain backend as a reorg, even when the chain height does not +// advance. This is the core "same-height reorgs invisible" gap the +// PR closes. +func TestChainBackendSameHeightHashDrift(t *testing.T) { + t.Parallel() + + chain := newFakeChain(t, 100, "drift-init") + srv := fakeChainServer(t, chain) + + esplora := NewEsploraClient(srv.URL, btclog.Disabled) + backend := NewChainBackend( + esplora, 20*time.Millisecond, btclog.Disabled, + ) + require.NoError(t, backend.Start()) + t.Cleanup(func() { + require.NoError(t, backend.Stop()) + }) + + // Subscribe directly to the poller's reorg stream to verify + // detection. The backend also subscribes; both subscribers + // must observe the reorg. + reorgSub, err := backend.tipPoller.SubscribeReorgs() + require.NoError(t, err) + defer reorgSub.Cancel() + + // Replace the tip block at the SAME height with a new hash. + chain.replaceTip("drift-new") + + select { + case ev := <-reorgSub.Updates(): + require.NotNil(t, ev) + require.Equal(t, int32(99), ev.ForkHeight) + require.Len(t, ev.Disconnected, 1) + require.Len(t, ev.Connected, 1) + require.Equal(t, int32(100), ev.Connected[0].Height) + + case <-time.After(reorgTestTimeout): + t.Fatal("timed out waiting for same-height reorg") + } +} + +// TestChainBackendConfReorgRoundTrip drives a registration through +// Confirmed -> Reorged -> Confirmed by replacing the block that +// confirmed the tx with a new block at the same height that still +// confirms the tx (different BlockHash). +func TestChainBackendConfReorgRoundTrip(t *testing.T) { + t.Parallel() + + chain := newFakeChain(t, 100, "conf-init") + srv := fakeChainServer(t, chain) + + // Use minimalRawTx so the EsploraClient's content-hash + // verification accepts the raw-tx response. + rawTx := minimalRawTx() + txid := minimalRawTxID(t) + chain.setRawTx(txid, rawTx) + + confBlock := chain.blocks[100] + chain.setTxStatus(txid, esploraTxStatus{ + Confirmed: true, + BlockHeight: 100, + BlockHash: confBlock.hash.String(), + }) + + esplora := NewEsploraClient(srv.URL, btclog.Disabled) + backend := NewChainBackend( + esplora, 20*time.Millisecond, btclog.Disabled, + ) + require.NoError(t, backend.Start()) + t.Cleanup(func() { + require.NoError(t, backend.Stop()) + }) + + reg, err := backend.RegisterConf( + t.Context(), &txid, nil, 1, 99, false, + ) + require.NoError(t, err) + defer reg.Cancel() + + conf1 := awaitConf(t, reg.Confirmed) + require.Equal(t, uint32(100), conf1.BlockHeight) + require.Equal(t, confBlock.hash, *conf1.BlockHash) + + // Same-height reorg: replace the confirming block with a + // different one at height 100, and pin the tx status to + // the new block hash so the post-reorg re-check finds the + // tx confirmed in the new block. + newBlock := chain.replaceTip("conf-replaced") + chain.setTxStatus(txid, esploraTxStatus{ + Confirmed: true, + BlockHeight: 100, + BlockHash: newBlock.hash.String(), + }) + + awaitSeqSignal(t, reg.Reorged, "conf Reorged") + + conf2 := awaitConf(t, reg.Confirmed) + require.Equal(t, uint32(100), conf2.BlockHeight) + require.Equal(t, newBlock.hash, *conf2.BlockHash) + require.NotEqual( + t, conf1.BlockHash.String(), conf2.BlockHash.String(), + "reorg should surface a different block hash", + ) +} + +// TestChainBackendConfReorgEvictedDoesNotReConfirm verifies that +// when the post-reorg chain no longer contains the tx, Reorged +// fires but Confirmed does not re-fire. +func TestChainBackendConfReorgEvictedDoesNotReConfirm(t *testing.T) { + t.Parallel() + + chain := newFakeChain(t, 100, "evict-init") + srv := fakeChainServer(t, chain) + + rawTx := minimalRawTx() + txid := minimalRawTxID(t) + chain.setRawTx(txid, rawTx) + + confBlock := chain.blocks[100] + chain.setTxStatus(txid, esploraTxStatus{ + Confirmed: true, + BlockHeight: 100, + BlockHash: confBlock.hash.String(), + }) + + esplora := NewEsploraClient(srv.URL, btclog.Disabled) + backend := NewChainBackend( + esplora, 20*time.Millisecond, btclog.Disabled, + ) + require.NoError(t, backend.Start()) + t.Cleanup(func() { + require.NoError(t, backend.Stop()) + }) + + reg, err := backend.RegisterConf( + t.Context(), &txid, nil, 1, 99, false, + ) + require.NoError(t, err) + defer reg.Cancel() + + awaitConf(t, reg.Confirmed) + + // Reorg evicts the confirming block AND the tx is no longer + // found on the new chain. + chain.replaceTip("evict-new") + chain.setTxStatus(txid, esploraTxStatus{Confirmed: false}) + + awaitSeqSignal(t, reg.Reorged, "conf Reorged") + + // No re-confirmation: assert silence. + requireQuiet( + t, structOnlyChan(reg.Confirmed), + "unexpected re-Confirmed", 500*time.Millisecond, + ) +} + +// structOnlyChan adapts a TxConfirmation channel to a struct{} +// channel for use with requireQuiet. The forwarder goroutine is +// short-lived; once the test ends both channels go out of scope. +func structOnlyChan(in <-chan *chainsourceConf) <-chan struct{} { + out := make(chan struct{}, 4) + + go func() { + for c := range in { + if c == nil { + continue + } + + select { + case out <- struct{}{}: + default: + } + } + }() + + return out +} + +// TestChainBackendSpendReorgRoundTrip drives a spend registration +// through Spend -> Reorged -> Spend by reorging the block that +// confirmed the spending tx and pinning a different (still spent) +// outspend on the new chain. +func TestChainBackendSpendReorgRoundTrip(t *testing.T) { + t.Parallel() + + chain := newFakeChain(t, 100, "spend-init") + srv := fakeChainServer(t, chain) + + rawTx := minimalRawTx() + spenderTxid := minimalRawTxID(t) + chain.setRawTx(spenderTxid, rawTx) + + fundingTxid := chainhash.HashH([]byte("funding")) + outpoint := wire.OutPoint{Hash: fundingTxid, Index: 0} + + chain.setOutspend(outpoint, esploraOutspend{ + Spent: true, Txid: spenderTxid.String(), Vin: 0, + Status: esploraStatus{ + Confirmed: true, BlockHeight: 100, + }, + }) + + esplora := NewEsploraClient(srv.URL, btclog.Disabled) + backend := NewChainBackend( + esplora, 20*time.Millisecond, btclog.Disabled, + ) + require.NoError(t, backend.Start()) + t.Cleanup(func() { + require.NoError(t, backend.Stop()) + }) + + reg, err := backend.RegisterSpend( + t.Context(), &outpoint, nil, 90, + ) + require.NoError(t, err) + defer reg.Cancel() + + spend1 := awaitSpend(t, reg.Spend) + require.Equal(t, spenderTxid, *spend1.SpenderTxHash) + require.Equal(t, int32(100), spend1.SpendingHeight) + + // Replace the confirming block AND swap the spender on the + // new chain. The spender tx id is the same (we're reusing + // minimalRawTx) but the block hash differs, which is what + // the reorg handler keys off. + chain.replaceTip("spend-replaced") + chain.setOutspend(outpoint, esploraOutspend{ + Spent: true, Txid: spenderTxid.String(), Vin: 0, + Status: esploraStatus{ + Confirmed: true, BlockHeight: 100, + }, + }) + + awaitSeqSignal(t, reg.Reorged, "spend Reorged") + + spend2 := awaitSpend(t, reg.Spend) + require.Equal(t, spenderTxid, *spend2.SpenderTxHash) + require.Equal(t, int32(100), spend2.SpendingHeight) +} + +// TestTipPollerSeedsHashHistoryOnStart pins the property that the +// poller seeds its recent-hash ring back through historySize-1 +// heights at Start. Without this, a fresh poller would only ever +// cache the initial tip, leaving any reorg-event consumer with an +// incomplete disconnected set for reorgs deeper than 1 block but +// within the configured history. The chain.Interface consumer +// (EsploraChainService) depends on a complete disconnected set to +// emit a BlockDisconnected for every height btcwallet must roll +// back. +func TestTipPollerSeedsHashHistoryOnStart(t *testing.T) { + t.Parallel() + + // Build a chain at tip 100 plus three lower heights. The poller + // starts at tip 100; with seed-history, recentHashes will be + // pre-populated with {97, 98, 99, 100}. + chain := newFakeChain(t, 100, "seed-100") + // fakeChain only populates the tip block; the test needs lower + // heights in the response. Extend backwards by minting blocks + // at 99, 98, 97 with the correct PrevBlock chain so the + // poller's seed walk resolves each height. + chain.mu.Lock() + var prev chainhash.Hash + for h := int32(97); h <= 100; h++ { + blk := chain.mintBlock(h, prev, fmt.Sprintf("seed-%d", h)) + chain.blocks[h] = blk + prev = blk.hash + } + chain.mu.Unlock() + + srv := fakeChainServer(t, chain) + esplora := NewEsploraClient(srv.URL, btclog.Disabled) + + // Cap historySize at 4 so the seed walk fills exactly the + // heights we care about. + tipPoller := NewTipPollerWithConfig( + esplora, 20*time.Millisecond, 4, btclog.Disabled, + ) + require.NoError(t, tipPoller.Start()) + t.Cleanup(tipPoller.Stop) + + reorgSub, err := tipPoller.SubscribeReorgs() + require.NoError(t, err) + defer reorgSub.Cancel() + + // Reorg the deepest 3 heights (98, 99, 100). With seed-history + // the poller's recentHashes contains 97, 98, 99, 100 so the + // walk-back terminates at fork point 97 and disconnects 3 + // hashes. Without the seed, recentHashes would only contain + // 100; walk-back would stop at 99 (no cached hash) and the + // disconnected list would carry only 1 hash. + chain.rewriteFrom(98, "seed-rewritten") + + select { + case ev := <-reorgSub.Updates(): + require.NotNil(t, ev) + require.Equal( + t, int32(97), ev.ForkHeight, "fork point should be "+ + "the deepest unchanged height; "+ + "seed-history failed if it lands above 97", + ) + require.Len( + t, ev.Disconnected, 3, "all three reorged heights "+ + "must appear in Disconnected; seed-history "+ + "is what makes this property hold for "+ + "reorgs of depth > 1 against a freshly "+ + "started poller", + ) + + case <-time.After(reorgTestTimeout): + t.Fatal("timed out waiting for reorg event") + } +} + +// TestChainBackendDeeperReorgDetected verifies that a multi-block +// reorg (where height advances AND PrevBlock continuity is broken) +// produces a ReorgEvent with the correct fork point. +// +// The test first advances the chain past the eventual fork point so +// the poller's recent-hash ring buffer contains the to-be-rewritten +// heights. Without that, the walk-back would terminate early on the +// first !haveCached probe and report a shallower fork than the test +// intends to exercise. +func TestChainBackendDeeperReorgDetected(t *testing.T) { + t.Parallel() + + chain := newFakeChain(t, 100, "deep-init") + + srv := fakeChainServer(t, chain) + esplora := NewEsploraClient(srv.URL, btclog.Disabled) + tp := NewTipPoller(esplora, 20*time.Millisecond, btclog.Disabled) + require.NoError(t, tp.Start()) + t.Cleanup(tp.Stop) + + // Subscribe BEFORE the rewrite so we receive the reorg + // event live. + reorgSub, err := tp.SubscribeReorgs() + require.NoError(t, err) + defer reorgSub.Cancel() + + tipSub, err := tp.Subscribe() + require.NoError(t, err) + defer tipSub.Cancel() + + // Advance the chain through the poller so it caches + // heights 101 and 102 in its ring buffer. + chain.extend("init-101") + chain.extend("init-102") + + for expected := int32(101); expected <= 102; expected++ { + select { + case ev := <-tipSub.Updates(): + require.Equal(t, expected, ev.Height) + + case <-time.After(reorgTestTimeout): + t.Fatalf("timed out waiting for height %d to be cached", + expected) + } + } + + // Now rewrite from height 101 upward (heights 101 and 102 + // get new hashes) and extend by one to push the new tip + // past the old. + chain.rewriteFrom(101, "fork") + chain.extend("fork-103") + + // Wait for ReorgEvent. ForkHeight should be 100 because + // the rewrite started at 101 (so 100 is the last height + // that agrees between old and new chains). + select { + case ev := <-reorgSub.Updates(): + require.NotNil(t, ev) + require.Equal(t, int32(100), ev.ForkHeight) + require.GreaterOrEqual(t, len(ev.Connected), 2) + require.GreaterOrEqual(t, len(ev.Disconnected), 2) + + case <-time.After(reorgTestTimeout): + t.Fatal("timed out waiting for deeper reorg") + } +} + +// TestChainBackendCancelCleanup verifies that Cancel on a +// conf/spend registration removes it from internal maps and does +// not leak goroutines. +func TestChainBackendCancelCleanup(t *testing.T) { + t.Parallel() + + chain := newFakeChain(t, 100, "cancel-init") + srv := fakeChainServer(t, chain) + + esplora := NewEsploraClient(srv.URL, btclog.Disabled) + backend := NewChainBackend( + esplora, time.Hour, btclog.Disabled, + ) + require.NoError(t, backend.Start()) + t.Cleanup(func() { + require.NoError(t, backend.Stop()) + }) + + txid := minimalRawTxID(t) + outpoint := wire.OutPoint{ + Hash: chainhash.HashH([]byte("funding")), Index: 0, + } + + // Sample the baseline goroutine count AFTER the backend is + // started so the poller / tip handler / reorg handler + // goroutines are already included. + time.Sleep(50 * time.Millisecond) + baseline := runtime.NumGoroutine() + + const iterations = 20 + for i := 0; i < iterations; i++ { + confReg, err := backend.RegisterConf( + t.Context(), &txid, nil, 1, 99, false, + ) + require.NoError(t, err) + + spendReg, err := backend.RegisterSpend( + t.Context(), &outpoint, nil, 90, + ) + require.NoError(t, err) + + confReg.Cancel() + spendReg.Cancel() + + // Double Cancel must be a safe no-op. + confReg.Cancel() + spendReg.Cancel() + } + + // Give the spawned one-shot goroutines time to exit. + require.Eventually(t, func() bool { + + // Allow a small tolerance for scheduling jitter; the + // upper bound here is generous (20) versus the + // per-iteration spawn count (2) to absorb any + // in-flight Esplora request goroutines that have not + // yet returned. The key invariant is that the count + // does not GROW unbounded. + return runtime.NumGoroutine() <= baseline+5 + }, 2*time.Second, 50*time.Millisecond, + "goroutines leaked after Cancel: baseline=%d current=%d", + baseline, runtime.NumGoroutine()) + + // After Cancel, internal maps should be empty. + backend.mu.Lock() + confLen := len(backend.confRegs) + spendLen := len(backend.spendRegs) + backend.mu.Unlock() + require.Equal(t, 0, confLen, + "confRegs not cleaned up after Cancel") + require.Equal(t, 0, spendLen, + "spendRegs not cleaned up after Cancel") +} + +// TestChainBackendConfStaysAliveAfterFirstFire verifies that the +// confirmation registration is NOT deleted after the first +// Confirmed delivery, since the chainsource ConfActor needs the +// registration to remain alive to receive future Reorged events +// and finally a synthesized Done. +func TestChainBackendConfStaysAliveAfterFirstFire(t *testing.T) { + t.Parallel() + + chain := newFakeChain(t, 100, "alive-init") + srv := fakeChainServer(t, chain) + + rawTx := minimalRawTx() + txid := minimalRawTxID(t) + chain.setRawTx(txid, rawTx) + chain.setTxStatus(txid, esploraTxStatus{ + Confirmed: true, + BlockHeight: 100, + BlockHash: chain.blocks[100].hash.String(), + }) + + esplora := NewEsploraClient(srv.URL, btclog.Disabled) + backend := NewChainBackend( + esplora, 50*time.Millisecond, btclog.Disabled, + ) + require.NoError(t, backend.Start()) + t.Cleanup(func() { + require.NoError(t, backend.Stop()) + }) + + reg, err := backend.RegisterConf( + t.Context(), &txid, nil, 1, 99, false, + ) + require.NoError(t, err) + defer reg.Cancel() + + awaitConf(t, reg.Confirmed) + + // Give the polling loop time to fire several heartbeats. + // The registration must still be present. + time.Sleep(200 * time.Millisecond) + + backend.mu.Lock() + confLen := len(backend.confRegs) + backend.mu.Unlock() + + require.Equal( + t, 1, confLen, "reg deleted after first fire; reorg-aware "+ + "contract requires it to stay alive", + ) +} + +// TestChainBackendConfReorgBelowSeededHistory pins the property that a +// reorg deeper than the poller's seeded hash history still surfaces +// Reorged for a registration that delivered against the +// now-out-of-window block. The poller's disconnected set is bounded +// by the cached history, so without a canonical re-query fallback the +// reorg signal would be silently dropped. +func TestChainBackendConfReorgBelowSeededHistory(t *testing.T) { + t.Parallel() + + // Seed a chain at height 100 and extend up to 105 so the + // walk-back has live block hashes to compare against. Use a + // tiny history size so the registration's conf block falls + // outside the seeded window from the poller's perspective. + chain := newFakeChain(t, 100, "below-100") + chain.extend("below-101") + chain.extend("below-102") + chain.extend("below-103") + chain.extend("below-104") + chain.extend("below-105") + + confBlock := chain.blocks[100] + srv := fakeChainServer(t, chain) + + rawTx := minimalRawTx() + txid := minimalRawTxID(t) + chain.setRawTx(txid, rawTx) + chain.setTxStatus(txid, esploraTxStatus{ + Confirmed: true, + BlockHeight: 100, + BlockHash: confBlock.hash.String(), + }) + + esplora := NewEsploraClient(srv.URL, btclog.Disabled) + // historySize=3 means the poller only retains the last three + // canonical heights it has observed. The conf block at height + // 100 will be five heights below the seeded tip (105) and is + // guaranteed never to enter recentHashes. + tipPoller := NewTipPollerWithConfig( + esplora, 20*time.Millisecond, 3, btclog.Disabled, + ) + require.NoError(t, tipPoller.Start()) + t.Cleanup(tipPoller.Stop) + backend, err := NewChainBackendWithPoller( + esplora, tipPoller, btclog.Disabled, + ) + require.NoError(t, err) + require.NoError(t, backend.Start()) + t.Cleanup(func() { + require.NoError(t, backend.Stop()) + }) + + reg, err := backend.RegisterConf( + t.Context(), &txid, nil, 1, 99, false, + ) + require.NoError(t, err) + defer reg.Cancel() + + conf1 := awaitConf(t, reg.Confirmed) + require.Equal(t, confBlock.hash, *conf1.BlockHash) + + // Deep reorg: rewrite every height from 100 upward with + // different hashes. The new block at height 100 has a new + // hash, but the tx is still pinned there so the canonical + // re-query in reorgConfReg can fire a fresh Confirmed. + chain.rewriteFrom(100, "below-new") + newConfBlock := chain.blocks[100] + chain.setTxStatus(txid, esploraTxStatus{ + Confirmed: true, + BlockHeight: 100, + BlockHash: newConfBlock.hash.String(), + }) + + awaitSeqSignal(t, reg.Reorged, "conf Reorged on below-history reorg") + + conf2 := awaitConf(t, reg.Confirmed) + require.Equal(t, newConfBlock.hash, *conf2.BlockHash) + require.NotEqual( + t, conf1.BlockHash.String(), conf2.BlockHash.String(), + "re-confirmation must surface the new block hash", + ) +} + +// TestChainBackendAbortsOnRawHeaderFailure pins the property that a +// failed raw-header fetch during continuity check aborts the poll +// cycle rather than optimistically advancing. The optimistic path +// would permanently hide a reorg that crossed the old-tip boundary +// if the raw-header fetch flaked at exactly the wrong moment. +func TestChainBackendAbortsOnRawHeaderFailure(t *testing.T) { + t.Parallel() + + chain := newFakeChain(t, 100, "abort-init") + srv := fakeChainServer(t, chain) + + esplora := NewEsploraClient(srv.URL, btclog.Disabled) + tipPoller := NewTipPoller( + esplora, 20*time.Millisecond, btclog.Disabled, + ) + require.NoError(t, tipPoller.Start()) + t.Cleanup(tipPoller.Stop) + + tipSub, err := tipPoller.Subscribe() + require.NoError(t, err) + defer tipSub.Cancel() + reorgSub, err := tipPoller.SubscribeReorgs() + require.NoError(t, err) + defer reorgSub.Cancel() + + // Add a height-101 block whose PrevBlock does NOT chain off + // the seeded tip at height 100. Then poison the raw-header + // endpoint for that block so the continuity check cannot + // resolve. The poller must abort the cycle: not broadcast + // the new block, not update its cached tip, not fire a reorg. + chain.mu.Lock() + stranger := chain.mintBlock( + 101, chainhash.Hash{0xde, 0xad}, "abort-stranger", + ) + chain.blocks[101] = stranger + chain.tip = 101 + chain.failRawHeader[stranger.hash] = struct{}{} + chain.mu.Unlock() + + // Sleep long enough for a few poll cycles to fire; verify + // neither tip advance nor reorg ever fires. + select { + case ev := <-tipSub.Updates(): + t.Fatalf("poller broadcast a tip event despite raw-header "+ + "failure: height=%d hash=%s", ev.Height, ev.Hash) + + case ev := <-reorgSub.Updates(): + t.Fatalf("poller broadcast a reorg event despite raw-header "+ + "failure: fork=%d", ev.ForkHeight) + + case <-time.After(200 * time.Millisecond): + // Expected: silent. + } + + height, hash, _ := tipPoller.BestBlock() + require.Equal( + t, int32(100), height, + "cached tip height advanced despite aborted cycle", + ) + require.Equal( + t, chain.blocks[100].hash, hash, + "cached tip hash advanced despite aborted cycle", + ) +} diff --git a/lwwallet/chain_backend_test.go b/lwwallet/chain_backend_test.go index bcea6be99..6ee39c5ad 100644 --- a/lwwallet/chain_backend_test.go +++ b/lwwallet/chain_backend_test.go @@ -2,6 +2,7 @@ package lwwallet import ( "bytes" + "encoding/hex" "encoding/json" "fmt" "net/http" @@ -80,15 +81,20 @@ func TestChainBackendBlockNotification(t *testing.T) { mu sync.Mutex tipHeight int32 = 100 blockHashes = make(map[int32]chainhash.Hash) + blockHdrs = make(map[int32]*wire.BlockHeader) ) - // Pre-generate block hashes. + // Pre-generate real chained block headers so the tip poller's + // PrevBlock continuity check during advance can resolve the raw + // header endpoint with a header that actually links back to the + // previous height. + var prevHash chainhash.Hash for h := int32(100); h <= 103; h++ { - blockHashes[h] = chainhash.HashH( - []byte( - fmt.Sprintf("block-%d", h), - ), - ) + hdr := mintStubHeader(h, prevHash) + blockHdrs[h] = hdr + hash := hdr.BlockHash() + blockHashes[h] = hash + prevHash = hash } srv := mockEsploraServer( @@ -109,7 +115,7 @@ func TestChainBackendBlockNotification(t *testing.T) { default: handleBlockReqs( - t, w, r, &mu, blockHashes, + t, w, r, &mu, blockHashes, blockHdrs, ) } }, @@ -147,10 +153,13 @@ func TestChainBackendBlockNotification(t *testing.T) { } } -// handleBlockReqs handles /block-height/:height and /block/:hash -// requests in the mock Esplora server. +// handleBlockReqs handles /block-height/:height, /block/:hash, and +// /block/:hash/header requests in the mock Esplora server. The +// blockHdrs map supplies real wire.BlockHeaders so the raw-header +// continuity check the tip poller runs on advance can succeed. func handleBlockReqs(t *testing.T, w http.ResponseWriter, r *http.Request, - mu *sync.Mutex, blockHashes map[int32]chainhash.Hash) { + mu *sync.Mutex, blockHashes map[int32]chainhash.Hash, + blockHdrs map[int32]*wire.BlockHeader) { t.Helper() @@ -176,20 +185,40 @@ func handleBlockReqs(t *testing.T, w http.ResponseWriter, r *http.Request, return } - // Handle /block/:hash (block header). - for _, h := range blockHashes { + // Handle /block/:hash and /block/:hash/header. + mu.Lock() + defer mu.Unlock() + for height, h := range blockHashes { hashStr := h.String() path := "/block/" + hashStr - if r.URL.Path == path { + headerPath := path + "/header" + + switch r.URL.Path { + case path: resp := esploraBlock{ ID: hashStr, - Height: 100, + Height: height, Timestamp: 1700000000, } - err := json.NewEncoder(w).Encode(resp) require.NoError(t, err) + return + + case headerPath: + hdr := blockHdrs[height] + if hdr == nil { + http.Error(w, "no header", + http.StatusNotFound) + + return + } + var buf bytes.Buffer + require.NoError(t, hdr.Serialize(&buf)) + _, _ = fmt.Fprint( + w, hex.EncodeToString(buf.Bytes()), + ) + return } } diff --git a/lwwallet/esplora_chain.go b/lwwallet/esplora_chain.go index bd37534ee..2d1c944fc 100644 --- a/lwwallet/esplora_chain.go +++ b/lwwallet/esplora_chain.go @@ -110,12 +110,15 @@ func NewEsploraChainService(esplora *EsploraClient, tipPoller *TipPoller, // Start seeds the initial chain tip from the configured TipPoller // (which the caller must have started already) and spawns the -// goroutine that translates each TipBlock event into btcwallet -// chain notifications. +// goroutine that translates each chain event into btcwallet chain +// notifications. The service subscribes to the unified chain stream +// so reorg and tip updates arrive on a single producer-ordered +// channel; this is what lets chain.BlockDisconnected reliably precede +// the replacement chain.BlockConnected events for the new tip. func (s *EsploraChainService) Start(ctx context.Context) error { //nolint:contextcheck // tip subscription lifecycle is owned by Stop - tipHeight, tipHash, tipTime, sub, err := - s.tipPoller.BestBlockAndSubscribe() + tipHeight, tipHash, tipTime, chainSub, err := + s.tipPoller.BestBlockAndSubscribeChain() if err != nil { return fmt.Errorf("subscribe to tip poller: %w", err) } @@ -134,7 +137,7 @@ func (s *EsploraChainService) Start(ctx context.Context) error { s.notifications <- chain.ClientConnected{} s.wg.Add(1) - go s.handleTipEvents(ctx, sub) + go s.handleChainEvents(ctx, chainSub) s.log.InfoS(ctx, "Esplora chain service started", slog.Int("tip_height", int(tipHeight)), @@ -685,26 +688,35 @@ func (s *EsploraChainService) MapRPCErr(err error) error { return err } -// handleTipEvents drains TipBlock events from the shared poller and -// translates each event into the FilteredBlockConnected + -// BlockConnected notification pair that btcwallet's wallet syncer -// expects. The loop exits when the chain service is stopped, when -// the poller signals shutdown via Quit, or when the subscription's -// Updates channel is closed by Cancel. -func (s *EsploraChainService) handleTipEvents(ctx context.Context, - sub *TipSubscription) { +// handleChainEvents drains the unified chain stream and translates +// each event into btcwallet chain notifications. Using a single +// subscription delivers ReorgEvent and TipBlock updates in producer +// order through one channel, which is what guarantees +// BlockDisconnected lands on btcwallet's notification queue before +// the BlockConnected events for the replacement chain (btcwallet's +// disconnectBlock would otherwise refuse the rollback if the cached +// hash at that height had already been overwritten by a stale +// BlockConnected). +func (s *EsploraChainService) handleChainEvents(ctx context.Context, + sub *ChainSubscription) { defer s.wg.Done() defer sub.Cancel() for { select { - case event, ok := <-sub.Updates(): + case ev, ok := <-sub.Updates(): if !ok { return } - s.processTipEvent(ctx, event) + switch { + case ev.Reorg != nil: + s.processReorgEvent(ctx, ev.Reorg) + + case ev.Tip != nil: + s.processTipEvent(ctx, ev.Tip) + } case <-sub.Quit(): return @@ -729,6 +741,63 @@ func (s *EsploraChainService) handleTipEvents(ctx context.Context, // per-event cap branch without producing 256+ heights of traffic. const defaultMaxGapFillPerTipEvent int32 = 256 +// processReorgEvent translates a ReorgEvent into chain.BlockDisconnected +// notifications. The replacement Connected blocks arrive separately on +// the tip stream and are processed by processTipEvent in the usual way; +// emitting Disconnected here is what lets btcwallet roll back the +// reorged-out heights via its disconnectBlock path before it re-sees +// the canonical chain. Newest height is disconnected first so btcwallet +// walks back from its own cached tip. +// +// After emitting the disconnects, s.bestBlock is rolled back to +// event.ForkHeight so the subsequent replacement-chain TipBlock events +// (which arrive at heights forkHeight+1..newTipHeight) pass +// processTipEvent's "event.Height <= lastDelivered" duplicate guard. +// Without this rollback, the replacement BlockConnected events would +// silently be dropped because s.bestBlock still points at the old tip. +// The Hash is intentionally cleared because the reorg event does not +// carry the fork-point hash; the next BlockConnected fully repopulates +// s.bestBlock and only the Height is load-bearing for the duplicate +// guard. +func (s *EsploraChainService) processReorgEvent(ctx context.Context, + event *ReorgEvent) { + + if event == nil { + return + } + + for i := len(event.Disconnected) - 1; i >= 0; i-- { + hash := event.Disconnected[i] + height := event.ForkHeight + int32(i) + 1 + + meta := wtxmgr.BlockMeta{ + Block: wtxmgr.Block{ + Hash: hash, + Height: height, + }, + } + + select { + case s.notifications <- chain.BlockDisconnected(meta): + case <-s.quit: + return + } + } + + s.mu.Lock() + if s.bestBlock.Height > event.ForkHeight { + s.bestBlock = waddrmgr.BlockStamp{ + Height: event.ForkHeight, + } + } + s.mu.Unlock() + + s.log.InfoS(ctx, "Chain service emitted BlockDisconnected", + slog.Int("fork_height", int(event.ForkHeight)), + slog.Int("disconnected", len(event.Disconnected)), + ) +} + // processTipEvent applies one TipBlock to btcwallet's notification // channel. The chain service owns its own delivery cursor // (s.bestBlock); this function first walks any gap between diff --git a/lwwallet/esplora_chain_reorg_test.go b/lwwallet/esplora_chain_reorg_test.go new file mode 100644 index 000000000..a4b22c1c1 --- /dev/null +++ b/lwwallet/esplora_chain_reorg_test.go @@ -0,0 +1,175 @@ +package lwwallet + +import ( + "testing" + "time" + + "github.com/btcsuite/btcd/chainhash/v2" + "github.com/btcsuite/btclog/v2" + "github.com/btcsuite/btcwallet/chain" + "github.com/btcsuite/btcwallet/wtxmgr" + "github.com/stretchr/testify/require" +) + +// awaitNotification pulls the next notification from the chain +// service or fails the test on timeout. +func awaitNotification(t *testing.T, s *EsploraChainService) interface{} { + t.Helper() + + select { + case n := <-s.Notifications(): + return n + + case <-time.After(reorgTestTimeout): + t.Fatalf("timed out waiting for chain notification") + + return nil + } +} + +// drainUntilConnected drains notifications until a BlockConnected +// event at the given height is observed (or the test times out). +// Used to skip over the startup ClientConnected and initial connected +// events so the reorg-specific assertions can run on a known cursor. +func drainUntilConnected(t *testing.T, s *EsploraChainService, height int32) { + t.Helper() + + deadline := time.After(reorgTestTimeout) + + for { + select { + case n := <-s.Notifications(): + conn, ok := n.(chain.BlockConnected) + if !ok { + continue + } + if conn.Block.Height == height { + return + } + + case <-deadline: + t.Fatalf("timed out waiting for BlockConnected at %d", + height) + } + } +} + +// TestEsploraChainServiceReorgEmitsBlockDisconnected pins the property +// that a chain reorg surfaces chain.BlockDisconnected notifications +// before btcwallet sees the BlockConnected events that announce the +// new canonical chain, AND that the disconnect events arrive in +// newest-height-first order (the order btcwallet's disconnectBlock +// expects when walking back from its cached tip). Both ordering +// properties are load-bearing: without disconnect-before-connect, +// btcwallet would refuse the rollback because the cached hash at +// each height would already be overwritten; without +// newest-first, btcwallet's per-step rollback could trip on a +// height it hasn't seen disconnected yet. +func TestEsploraChainServiceReorgEmitsBlockDisconnected(t *testing.T) { + t.Parallel() + + chainModel := newFakeChain(t, 100, "svc-reorg-100") + chainModel.extend("svc-reorg-101") + chainModel.extend("svc-reorg-102") + + srv := fakeChainServer(t, chainModel) + + esplora := NewEsploraClient(srv.URL, btclog.Disabled) + tipPoller := NewTipPoller( + esplora, 20*time.Millisecond, btclog.Disabled, + ) + require.NoError(t, tipPoller.Start()) + t.Cleanup(tipPoller.Stop) + + service := NewEsploraChainService( + esplora, tipPoller, btclog.Disabled, + ) + require.NoError(t, service.Start(t.Context())) + t.Cleanup(func() { + service.Stop() + service.WaitForShutdown() + }) + + // First notification is always ClientConnected. + first := awaitNotification(t, service) + _, ok := first.(chain.ClientConnected) + require.True(t, ok, "expected ClientConnected first, got %T", first) + + // Drain forward to a known tip + a couple of new blocks so the + // service has a known set of recently-emitted connected blocks + // before the reorg fires. + chainModel.extend("svc-reorg-103") + drainUntilConnected(t, service, 103) + + // Stash hashes of the blocks the reorg is about to invalidate. + oldHash102 := chainModel.blocks[102].hash + oldHash103 := chainModel.blocks[103].hash + + // Rewrite heights 102..103 with a new chain. The tip stays at + // 103 but with a different hash, so the poller detects a + // same-height drift at 103 and walks back to fork point 101. + chainModel.rewriteFrom(102, "svc-reorg-new") + newHash102 := chainModel.blocks[102].hash + newHash103 := chainModel.blocks[103].hash + + // Drain notifications and pin the strict ordering. Each + // BlockConnected for a new-chain hash must arrive AFTER both + // BlockDisconnected events for the old chain — the unified + // chain stream guarantees this by serializing reorg + tip + // events through a single producer-ordered channel. + disconnectsSeen := 0 + gotDisconnected := []chainhash.Hash{} + gotConnectedNew := map[chainhash.Hash]struct{}{} + deadline := time.After(reorgTestTimeout) + for len(gotConnectedNew) < 2 { + select { + case n := <-service.Notifications(): + switch ev := n.(type) { + case chain.BlockDisconnected: + gotDisconnected = append( + gotDisconnected, ev.Block.Hash, + ) + disconnectsSeen++ + + case chain.BlockConnected: + meta := wtxmgr.BlockMeta(ev) + if meta.Hash != newHash102 && + meta.Hash != newHash103 { + + continue + } + + require.Equal( + t, 2, disconnectsSeen, "BlockConnect"+ + "ed for replacement chain "+ + "arrived before all "+ + "BlockDisconnected events: "+ + "saw %d disconnects so far", + disconnectsSeen, + ) + gotConnectedNew[meta.Hash] = struct{}{} + } + + case <-deadline: + t.Fatalf("timed out: disconnects=%v connected_new=%v", + gotDisconnected, gotConnectedNew) + } + } + + require.Equal( + t, oldHash103, gotDisconnected[0], "first "+ + "BlockDisconnected should be the newest old-chain "+ + "height (103) for btcwallet's per-step walk-back", + ) + require.Equal( + t, oldHash102, gotDisconnected[1], + "second BlockDisconnected should be height 102", + ) + + _, sawConn102 := gotConnectedNew[newHash102] + _, sawConn103 := gotConnectedNew[newHash103] + require.True(t, sawConn102, + "BlockConnected for new 102 not emitted") + require.True(t, sawConn103, + "BlockConnected for new 103 not emitted") +} diff --git a/lwwallet/esplora_chain_test.go b/lwwallet/esplora_chain_test.go index ee43b022f..151363cc0 100644 --- a/lwwallet/esplora_chain_test.go +++ b/lwwallet/esplora_chain_test.go @@ -2,6 +2,7 @@ package lwwallet import ( "bytes" + "encoding/hex" "encoding/json" "fmt" "net/http" @@ -299,6 +300,16 @@ func (c *rawBlockStubChain) serveBlockRoute(t *testing.T, w http.ResponseWriter, require.NoError(t, block.Serialize(&buf)) _, _ = w.Write(buf.Bytes()) + case "/header": + // Hex-encoded 80-byte block header. The TipPoller fetches + // this per new tip to verify PrevBlock continuity against + // the cached tip hash; without it the poller cannot + // distinguish a clean tip advance from a reorg crossing the + // boundary and aborts the cycle. + var buf bytes.Buffer + require.NoError(t, block.Header.Serialize(&buf)) + _, _ = fmt.Fprint(w, hex.EncodeToString(buf.Bytes())) + default: http.Error(w, "not implemented", http.StatusNotImplemented) diff --git a/lwwallet/tip_poller.go b/lwwallet/tip_poller.go index dbce6c97d..f2fd3557f 100644 --- a/lwwallet/tip_poller.go +++ b/lwwallet/tip_poller.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log/slog" + "sort" "sync" "time" @@ -11,6 +12,13 @@ import ( "github.com/btcsuite/btclog/v2" ) +// DefaultHashHistorySize is the default upper bound on entries retained +// in the TipPoller's bounded height -> hash history map. It is sized to +// at least twice the conventional Bitcoin reorg-safety depth (6) so a +// reorg at finality depth still has its disconnected hashes available +// for walk-back, with headroom. Configurable via NewTipPollerWithConfig. +const DefaultHashHistorySize = 100 + // TipBlock describes a newly detected block emitted by TipPoller. // Each subscriber receives one TipBlock per advance: when the tip // moves from oldHeight to newHeight the poller fans out (newHeight - @@ -38,6 +46,62 @@ type TipBlock struct { // for ergonomic call-site naming. type TipSubscription = Subscription[*TipBlock] +// ReorgEvent describes a chain reorganization observed by the +// TipPoller. The poller emits one ReorgEvent every time it detects +// that one or more previously broadcast blocks are no longer in the +// canonical chain. Disconnected hashes are listed in canonical +// (low-height first) order; Connected blocks are the new tip's path +// from ForkHeight+1 onward, also in canonical order. Either slice may +// be empty: a same-height hash-replacement reorg has Disconnected of +// length 1 and Connected of length 1. +type ReorgEvent struct { + // ForkHeight is the highest height at which the old chain and the + // new chain agree on the block hash. The first disconnected / + // connected block is at ForkHeight+1. + ForkHeight int32 + + // Disconnected lists block hashes that were previously broadcast + // as part of the canonical chain and are no longer on it, in + // ascending height order (ForkHeight+1 first). + Disconnected []chainhash.Hash + + // Connected lists blocks now on the canonical chain starting at + // ForkHeight+1, in ascending height order. The poller will also + // fan these out individually as TipBlock events on the standard + // Subscribe channel after the ReorgEvent is delivered. + Connected []*TipBlock +} + +// ReorgSubscription is the typed handle returned by +// TipPoller.SubscribeReorgs. +type ReorgSubscription = Subscription[*ReorgEvent] + +// ChainEvent is the unified update delivered on the ordered chain +// stream returned by SubscribeChain. Exactly one of Reorg and Tip is +// non-nil per event: +// +// - Reorg-only events announce a reorg's disconnected range and +// precede the replacement tip events on the same stream. +// - Tip-only events announce a new block (either a forward advance +// or a post-reorg connected block). +// +// Cross-event ordering on this stream is producer-ordered: the +// embedded EventServer delivers updates in SendUpdate order to a +// single subscriber goroutine, so a downstream consumer that +// dispatches on event type from one channel observes the same order +// the TipPoller emitted. That is the load-bearing property +// downstream needs to emit BlockDisconnected before BlockConnected +// for the replacement chain (btcwallet's disconnectBlock requires +// it; chainsource finality synthesis requires it too). +type ChainEvent struct { + Reorg *ReorgEvent + Tip *TipBlock +} + +// ChainSubscription is the typed handle returned by +// TipPoller.SubscribeChain. It carries the unified ChainEvent stream. +type ChainSubscription = Subscription[*ChainEvent] + // TipPoller is the single source of truth for the lwwallet chain // tip. Exactly one polling goroutine periodically asks the Esplora // backend for the current best height. When new blocks are detected @@ -58,18 +122,46 @@ type TipPoller struct { pollInterval time.Duration log btclog.Logger + // historySize caps the bounded height -> hash history map that + // lets the poller resolve old-chain hashes during reorg walk-back. + historySize int + // events is the typed event server that fans TipBlock updates // out to all active subscribers. Its Start/Stop are driven by // TipPoller.Start/Stop. events *EventServer[*TipBlock] + // reorgs is the typed event server that fans ReorgEvent updates + // out to subscribers that opt in via SubscribeReorgs. Reorgs are + // rare enough that a separate server is cheaper than re-fitting + // every TipBlock consumer with reorg-aware logic. + reorgs *EventServer[*ReorgEvent] + + // chain is the unified event server that fans both tip and reorg + // updates out on a single ordered stream. Consumers that need + // strict reorg-before-replacement-tip ordering (chain.Interface + // adapter, chainsource backend) subscribe here rather than to + // the separate events / reorgs servers, which would race across + // two independent translator goroutines. + chain *EventServer[*ChainEvent] + // mu guards the cached tip so BestBlock readers see a - // consistent height/hash/timestamp triple. + // consistent height/hash/timestamp triple. It also guards + // recentHashes, which is sized small enough that linear scans + // under the lock are cheap. mu sync.Mutex tipHeight int32 tipHash chainhash.Hash tipTime time.Time + // recentHashes maps a recent canonical-chain height to the hash + // the poller broadcast for it. It is bounded to historySize + // entries: on every insert we prune any entry whose height is + // more than historySize below the current tip. The buffer lets + // reorg detection walk back to the fork point using the cached + // hashes rather than re-fetching the entire pre-fork chain. + recentHashes map[int32]chainhash.Hash + // started gates re-entrant Start calls; the underlying // EventServer is also idempotent on Start. started bool @@ -82,15 +174,35 @@ type TipPoller struct { // NewTipPoller constructs a TipPoller bound to the given Esplora // client. The poll interval controls how often the goroutine asks // Esplora for the latest tip height; subscribers do not influence -// the cadence. +// the cadence. The bounded hash-history map is sized to +// DefaultHashHistorySize. func NewTipPoller(esplora *EsploraClient, pollInterval time.Duration, log btclog.Logger) *TipPoller { + return NewTipPollerWithConfig( + esplora, pollInterval, DefaultHashHistorySize, log, + ) +} + +// NewTipPollerWithConfig is the explicit-history-size constructor. +// historySize <= 0 falls back to DefaultHashHistorySize so callers +// cannot accidentally disable reorg walk-back by passing a zero value. +func NewTipPollerWithConfig(esplora *EsploraClient, pollInterval time.Duration, + historySize int, log btclog.Logger) *TipPoller { + + if historySize <= 0 { + historySize = DefaultHashHistorySize + } + return &TipPoller{ esplora: esplora, pollInterval: pollInterval, + historySize: historySize, log: log, events: NewEventServer[*TipBlock](log), + reorgs: NewEventServer[*ReorgEvent](log), + chain: NewEventServer[*ChainEvent](log), + recentHashes: make(map[int32]chainhash.Hash), quit: make(chan struct{}), } } @@ -170,12 +282,60 @@ func (t *TipPoller) Start() error { return fmt.Errorf("start event server: %w", err) } + if err := t.reorgs.Start(); err != nil { + // Roll back the tip-event server so a partial Start does + // not leave a half-running poller behind. + _ = t.events.Stop() + resetStarted() + + return fmt.Errorf("start reorg event server: %w", err) + } + + if err := t.chain.Start(); err != nil { + _ = t.events.Stop() + _ = t.reorgs.Stop() + resetStarted() + + return fmt.Errorf("start chain event server: %w", err) + } + + // Seed recentHashes by walking back historySize-1 heights so a + // reorg whose disconnected range extends below the seeded tip + // but within the configured history can still resolve every + // disconnected hash from the cache. Without this, a fresh + // poller would only ever cache the initial tip, leaving a + // downstream chain.Interface consumer unable to enumerate every + // hash btcwallet must roll back on a multi-block reorg below + // the seeded tip. The walk-back is best-effort: a single + // per-height fetch failure ends the seed loop early; the next + // poll tick still drives the cache forward as the chain grows. t.mu.Lock() t.tipHeight = height t.tipHash = hash t.tipTime = tipTime + t.recordHashLocked(height, hash) t.mu.Unlock() + for h := height - 1; h > height-int32(t.historySize) && h >= 0; h-- { + liveHash, err := t.esplora.GetBlockHashByHeight( + context.Background(), h, + ) + if err != nil { + t.log.WarnS( + context.Background(), + "Tip poller history seed fetch failed", + err, + slog.Int("height", int(h)), + ) + + break + } + + t.mu.Lock() + t.recordHashLocked(h, liveHash) + t.mu.Unlock() + } + t.wg.Add(1) go t.pollLoop() @@ -197,7 +357,7 @@ func (t *TipPoller) Stop() { t.wg.Wait() - // Stop the event server after the poll loop has exited so + // Stop the event servers after the poll loop has exited so // that no SendUpdate is in flight when the server tears down // its subscriber handler. if err := t.events.Stop(); err != nil { @@ -207,6 +367,22 @@ func (t *TipPoller) Stop() { err, ) } + + if err := t.reorgs.Stop(); err != nil { + t.log.WarnS( + context.Background(), + "Tip poller reorg event server stop returned error", + err, + ) + } + + if err := t.chain.Stop(); err != nil { + t.log.WarnS( + context.Background(), + "Tip poller chain event server stop returned error", + err, + ) + } } // BestBlock returns a snapshot of the currently cached tip. Callers @@ -233,6 +409,18 @@ func (t *TipPoller) Subscribe() (*TipSubscription, error) { return t.events.Subscribe() } +// SubscribeReorgs returns a typed subscription that receives a +// ReorgEvent every time the poller detects that one or more +// previously broadcast blocks have been replaced on the canonical +// chain. Callers that need both tip events and reorg events should +// subscribe to both streams; reorg events are delivered BEFORE the +// connected blocks are fanned out on the TipBlock stream so that +// consumers can mark their registrations dirty before the +// re-confirmation arrives. +func (t *TipPoller) SubscribeReorgs() (*ReorgSubscription, error) { + return t.reorgs.Subscribe() +} + // BestBlockAndSubscribe atomically reads the current cached tip and // registers a new subscription. The poll goroutine holds t.mu // during the {update tip + SendUpdate} pair, and this function @@ -257,6 +445,85 @@ func (t *TipPoller) BestBlockAndSubscribe() (int32, chainhash.Hash, time.Time, return t.tipHeight, t.tipHash, t.tipTime, sub, nil } +// BestBlockAndSubscribeAll atomically reads the cached tip and +// registers both a TipBlock subscription and a ReorgEvent +// subscription. Reorg-aware consumers (e.g. ChainBackend) use this +// to avoid the small window where a reorg could land between +// independently registering on the two streams. +func (t *TipPoller) BestBlockAndSubscribeAll() (int32, chainhash.Hash, + time.Time, *TipSubscription, *ReorgSubscription, error) { + + t.mu.Lock() + defer t.mu.Unlock() + + sub, err := t.events.Subscribe() + if err != nil { + return 0, chainhash.Hash{}, time.Time{}, nil, nil, + fmt.Errorf("subscribe to tip events: %w", err) + } + + reorgSub, err := t.reorgs.Subscribe() + if err != nil { + sub.Cancel() + + return 0, chainhash.Hash{}, time.Time{}, nil, nil, + fmt.Errorf("subscribe to reorg events: %w", err) + } + + return t.tipHeight, t.tipHash, t.tipTime, sub, reorgSub, nil +} + +// SubscribeChain returns a typed subscription on the unified chain +// event stream. Updates arrive in producer order through a single +// channel; consumers that need strict reorg-before-replacement-tip +// ordering must subscribe here rather than to the separate tip / +// reorg streams (which race across two independent translator +// goroutines). +func (t *TipPoller) SubscribeChain() (*ChainSubscription, error) { + return t.chain.Subscribe() +} + +// BestBlockAndSubscribeChain atomically reads the cached tip and +// registers a ChainSubscription. Same atomicity guarantee as +// BestBlockAndSubscribe, applied to the unified stream. +func (t *TipPoller) BestBlockAndSubscribeChain() (int32, chainhash.Hash, + time.Time, *ChainSubscription, error) { + + t.mu.Lock() + defer t.mu.Unlock() + + sub, err := t.chain.Subscribe() + if err != nil { + return 0, chainhash.Hash{}, time.Time{}, nil, + fmt.Errorf("subscribe to chain events: %w", err) + } + + return t.tipHeight, t.tipHash, t.tipTime, sub, nil +} + +// recordHashLocked inserts a (height, hash) pair into the recent- +// hash history map and prunes any entry whose height has fallen out +// of the historySize window relative to the new entry. Caller must +// hold t.mu. +func (t *TipPoller) recordHashLocked(height int32, hash chainhash.Hash) { + t.recentHashes[height] = hash + + cutoff := height - int32(t.historySize) + for h := range t.recentHashes { + if h <= cutoff { + delete(t.recentHashes, h) + } + } +} + +// hashAtHeightLocked returns the cached hash for a height plus +// whether it was present. Caller must hold t.mu. +func (t *TipPoller) hashAtHeightLocked(height int32) (chainhash.Hash, bool) { + h, ok := t.recentHashes[height] + + return h, ok +} + // pollLoop is the single tip-polling goroutine. It ticks at // pollInterval, asks Esplora for the latest tip, and walks the // gap from the cached tip to the new tip emitting one TipBlock per @@ -278,12 +545,26 @@ func (t *TipPoller) pollLoop() { } } -// poll performs one tip-detection cycle. On detected progress it -// fetches the hash and header for each new height, broadcasts a -// TipBlock to every subscriber, and advances the cached tip -// monotonically. A failure to fetch any single block aborts the -// remainder of the cycle so subscribers never see an out-of-order -// event; the next tick re-attempts from the same starting point. +// poll performs one tip-detection cycle. The cycle is reorg-aware: +// +// 1. Query (tip height, tip hash). +// 2. If the tip is at the same height as the cached tip but the hash +// differs, walk back through cached hashes to find the fork point +// and emit a ReorgEvent + a re-broadcast TipBlock at the same +// height. This is the "same-height reorg" case that earlier +// versions of the poller could not detect. +// 3. If the new height is strictly greater than the cached height, +// fetch each intervening hash; if the first new block's PrevBlock +// header field does not point at the cached tip hash, walk back +// to find the fork point, emit a ReorgEvent for the disconnected +// range, and then fan TipBlock events for the new chain in +// ascending order. If the new chain extends the old tip cleanly +// (the common case), no ReorgEvent fires and TipBlock events are +// dispatched as before. +// +// A failure to fetch any single block aborts the remainder of the +// cycle so subscribers never see an out-of-order event; the next +// tick re-attempts from the same starting point. func (t *TipPoller) poll() { newHeight, err := t.esplora.GetTipHeight(context.Background()) if err != nil { @@ -298,23 +579,123 @@ func (t *TipPoller) poll() { t.mu.Lock() oldHeight := t.tipHeight + oldHash := t.tipHash t.mu.Unlock() - // Known limitation: a same-height reorg (block at height N - // replaced by a different block at height N) is invisible to - // this loop until the chain advances to N+1, because we gate - // progress on height alone rather than (height, hash). This - // matches the behavior of the per-component pollers that - // preceded the unified TipPoller and has historically been - // acceptable for lwwallet's confirmation-target use case - // (downstream callers re-check status against Esplora on every - // tip event, so a stale hash at height N converges within one - // extra tip advance). Documented here so it is not filed as a - // regression by a future reader. - if newHeight <= oldHeight { + switch { + case newHeight < oldHeight: + // The remote reports fewer blocks than we have cached. + // Two distinct cases hide behind a lower height: + // + // 1. A transient indexer hiccup where the remote + // momentarily lags our cached tip but is still on + // the same chain (our tip remains canonical). + // 2. A genuine reorg onto a SHORTER but higher-work + // chain: blocks above newHeight were orphaned and + // the surviving chain has fewer total blocks. Bitcoin + // follows most-work, not most-blocks, so a shorter + // chain can legitimately win. + // + // Disambiguate by comparing the live hash at newHeight to + // our cached hash for that height. Agreement means the + // remote is merely behind (case 1, no-op); divergence + // means the chain reorged out from under us (case 2) and + // we must walk back to the fork point. Without this an + // orphaning reorg to a shorter chain would be silently + // ignored until the new chain grew past our stale tip. + newTipHash, err := t.esplora.GetBlockHashByHeight( + context.Background(), newHeight, + ) + if err != nil { + t.log.WarnS( + context.Background(), + "Tip poller shorter-chain hash check failed", + err, + slog.Int("height", int(newHeight)), + ) + + return + } + + t.mu.Lock() + cachedHash, haveCached := t.hashAtHeightLocked(newHeight) + t.mu.Unlock() + + switch { + // newHeight predates our retained history floor, so we + // cannot prove a reorg by hash comparison. A reorg this + // deep exceeds historySize and is an operator-level + // problem rather than a productionally recoverable one + // (mirrors handleReorg's own deep-reorg guard); log and + // re-check next tick rather than walk a pruned history. + case !haveCached: + t.log.WarnS( + context.Background(), + "Tip poller remote tip below retained "+ + "history floor; ignoring", + fmt.Errorf("history floor exceeded"), + slog.Int("new_height", int(newHeight)), + slog.Int("old_height", int(oldHeight)), + ) + + return + + // Same hash at newHeight: the remote is merely lagging + // our cached tip on the same chain. No-op. + case newTipHash == cachedHash: + return + } + + // Divergent hash at newHeight: a reorg onto a shorter, + // higher-work chain. handleReorg builds Disconnected up + // to our old cached tip (covering the orphaned blocks + // above newHeight) and Connected up to newHeight. + t.handleReorg(newHeight, newTipHash) + + return + + case newHeight == oldHeight: + // Same-height: query the hash at this height (which + // the BlockHashByHeight endpoint resolves directly). + // If it differs from the cached hash at the same + // height we have a same-height reorg; if it matches + // the chain has not moved. + newTipHash, err := t.esplora.GetBlockHashByHeight( + context.Background(), newHeight, + ) + if err != nil { + t.log.WarnS( + context.Background(), + "Tip poller same-height hash check failed", + err, + slog.Int("height", int(newHeight)), + ) + + return + } + + if newTipHash == oldHash { + return + } + + t.handleReorg(newHeight, newTipHash) + return + + default: + // newHeight > oldHeight: forward advance, possibly with + // a deeper reorg if the first new height's predecessor + // is not the cached tip hash. + t.advance(oldHeight, newHeight) } +} +// advance walks the chain forward from oldHeight+1 to newHeight, +// detecting reorgs along the way. The very first new height's raw +// header is consulted to verify its PrevBlock matches the cached +// tip hash; mismatch triggers a walk-back to the fork point and a +// reorg emission before any new TipBlock events fire. +func (t *TipPoller) advance(oldHeight, newHeight int32) { t.log.DebugS(context.Background(), "Tip poller advancing", slog.Int("old_height", int(oldHeight)), slog.Int("new_height", int(newHeight)), @@ -349,44 +730,397 @@ func (t *TipPoller) poll() { return } + // On the FIRST new height, verify continuity against + // the cached tip. We use the raw 80-byte header (which + // carries PrevBlock) because the JSON header endpoint + // does not include the previous-block hash. If + // PrevBlock does not match the cached tip hash a reorg + // crossed the boundary; walk back to the fork point + // and emit a ReorgEvent before continuing. + if height == oldHeight+1 { + rawHdr, err := t.esplora.GetRawBlockHeader( + context.Background(), hash, + ) + if err != nil { + // Without the raw header we cannot verify + // that the new block connects to the cached + // tip. Abort the cycle without broadcasting + // or updating the cached tip; the next tick + // will retry. Optimistically advancing here + // would permanently hide a reorg that + // crossed the oldHeight boundary if the + // raw-header fetch flaked exactly when the + // reorg happened. + t.log.WarnS( + context.Background(), + "Tip poller raw header fetch "+ + "failed; aborting cycle", + err, + slog.String("hash", hash.String()), + ) + + return + } + + t.mu.Lock() + cachedOldHash, ok := t.hashAtHeightLocked( + oldHeight, + ) + t.mu.Unlock() + + if ok && rawHdr.PrevBlock != cachedOldHash { + // Deeper reorg: handle it as a new-tip + // reorg from oldHeight, pointing at the + // actual new tip at newHeight. handleReorg + // emits ReorgEvent + TipBlock events + // itself. + tipHash, tipErr := t.esplora. + GetBlockHashByHeight( + context.Background(), newHeight, + ) + if tipErr != nil { + t.log.WarnS( + context.Background(), + "Tip poller "+ + "reorg tip "+ + "fetch failed", + tipErr, + slog.Int( + "height", + int(newHeight), + ), + ) + + return + } + + t.handleReorg(newHeight, tipHash) + + return + } + } + event := &TipBlock{ Height: height, Hash: hash, Header: header, } - // Hold t.mu across the {update tip + SendUpdate} - // pair so BestBlockAndSubscribe can serialize against - // it: a subscriber that acquires t.mu before us reads - // the OLD tip and is guaranteed to receive THIS event - // once it Subscribes (subscribe.Server's handler is - // single-threaded over Subscribe and SendUpdate, so - // our SendUpdate enqueues behind their Subscribe). A - // subscriber that acquires t.mu after us reads the - // NEW tip and will see only events strictly newer - // than this one. Without holding t.mu here a tip - // reader+subscriber pair has a small window where it - // can read the new tip but miss this event entirely. + if !t.broadcastTipBlock(event) { + return + } + } +} + +// broadcastTipBlock updates the cached tip, records the hash in the +// history map, and fans the TipBlock out to subscribers. Returns +// false if the send failed (server shutting down), telling the +// caller to abort the cycle so the cached tip is not advanced past +// an event subscribers did not receive. +func (t *TipPoller) broadcastTipBlock(event *TipBlock) bool { + // Hold t.mu across the {update tip + SendUpdate} pair so + // BestBlockAndSubscribe can serialize against it: a subscriber + // that acquires t.mu before us reads the OLD tip and is + // guaranteed to receive THIS event once it Subscribes + // (subscribe.Server's handler is single-threaded over + // Subscribe and SendUpdate, so our SendUpdate enqueues behind + // their Subscribe). A subscriber that acquires t.mu after us + // reads the NEW tip and will see only events strictly newer + // than this one. + t.mu.Lock() + t.tipHeight = event.Height + t.tipHash = event.Hash + if event.Header != nil { + t.tipTime = time.Unix(event.Header.Timestamp, 0) + } + t.recordHashLocked(event.Height, event.Hash) + tipErr := t.events.SendUpdate(event) + chainErr := t.chain.SendUpdate(&ChainEvent{Tip: event}) + t.mu.Unlock() + + if tipErr != nil { + t.log.WarnS( + context.Background(), + "Tip poller send update failed", + tipErr, + slog.Int("height", int(event.Height)), + ) + + return false + } + + if chainErr != nil { + t.log.WarnS( + context.Background(), + "Tip poller chain stream send failed", + chainErr, + slog.Int("height", int(event.Height)), + ) + + return false + } + + return true +} + +// handleReorg walks back through the recent-hash history map until it +// finds the fork point between the cached chain and the new chain +// whose tip is (newTipHeight, newTipHash). It then constructs a +// ReorgEvent listing the disconnected hashes and connected blocks, +// broadcasts the reorg, and finally fans out the connected blocks +// as TipBlock events in ascending height order so consumers can +// re-check registrations against each new block. +func (t *TipPoller) handleReorg(newTipHeight int32, newTipHash chainhash.Hash) { + t.log.InfoS(context.Background(), "Tip poller detected reorg", + slog.Int("new_tip_height", int(newTipHeight)), + slog.String("new_tip_hash", newTipHash.String()), + ) + + // Walk back to find the fork point. We probe heights from + // newTipHeight downwards, comparing the live Esplora hash at + // each height to the cached hash. The first height at which + // they agree is the fork point. We bound the search by the + // retained history depth so a misbehaving Esplora cannot drag + // us into an unbounded loop. + t.mu.Lock() + cachedTipHeight := t.tipHeight + cutoff := cachedTipHeight - int32(t.historySize) + t.mu.Unlock() + + if cutoff < 0 { + cutoff = 0 + } + + // connectedByHeight collects new-chain blocks discovered + // during walk-back so we can re-broadcast them in ascending + // order after the fork point is found. + connectedByHeight := make(map[int32]chainhash.Hash) + connectedByHeight[newTipHeight] = newTipHash + + forkHeight := int32(-1) + probeHeight := newTipHeight + for probeHeight > cutoff { + probeHeight-- + + // Heights strictly above our cached tip cannot be a fork + // point: we never broadcast a block there, so the live + // block is a NEW connected block on a longer chain (a + // forward reorg that both reorganized old blocks AND + // extended past our tip). Record it and keep walking + // down toward the real fork. Without this, the + // !haveCached branch below would mistake the first such + // height for the fork point and compute a forkHeight + // above the cached tip, underflowing the Disconnected + // slice capacity. + if probeHeight > cachedTipHeight { + liveHash, err := t.esplora.GetBlockHashByHeight( + context.Background(), probeHeight, + ) + if err != nil { + t.log.WarnS( + context.Background(), + "Tip poller reorg walk-back failed", + err, + slog.Int("height", int(probeHeight)), + ) + + return + } + + connectedByHeight[probeHeight] = liveHash + + continue + } + t.mu.Lock() - t.tipHeight = height - t.tipHash = hash - t.tipTime = time.Unix(header.Timestamp, 0) - sendErr := t.events.SendUpdate(event) + cachedHash, haveCached := t.hashAtHeightLocked(probeHeight) t.mu.Unlock() - // SendUpdate failures only happen when the embedded - // subscribe.Server is shutting down; log and exit so - // we do not advance the cached tip past an event we - // failed to fan out. - if sendErr != nil { + if !haveCached { + // We never broadcast a block at this height + // (typically because it's older than our + // retained history's start, e.g. on a fresh + // poller that only ever cached the initial + // tip). Treat probeHeight as the fork point: + // anything at or below it is not part of our + // old broadcast set, so it cannot be + // "disconnected" from a downstream consumer's + // point of view. The live hash at this height + // is not a new connected block we owe + // subscribers either; only blocks strictly + // above probeHeight that we previously + // broadcast (and that have now changed) form + // the reorg boundary. + forkHeight = probeHeight + + break + } + + liveHash, err := t.esplora.GetBlockHashByHeight( + context.Background(), probeHeight, + ) + if err != nil { t.log.WarnS( context.Background(), - "Tip poller send update failed", - sendErr, - slog.Int("height", int(height)), + "Tip poller reorg walk-back failed", + err, + slog.Int("height", int(probeHeight)), + ) + + return + } + + if liveHash == cachedHash { + // Genuine fork point: the new chain agrees with + // our cached canonical chain at this height. + forkHeight = probeHeight + + break + } + + // haveCached && liveHash != cachedHash: this height is + // part of the reorg. Record the live hash so we can + // rebroadcast it after the fork point is found. + connectedByHeight[probeHeight] = liveHash + } + + if forkHeight < 0 { + // We exhausted the retained history without finding a + // fork point. Be conservative: log and bail. A future + // tick will re-attempt with the now-pruned history; in + // practice a reorg deeper than DefaultHashHistorySize + // is a developer / operator problem, not a + // productionally recoverable condition. + t.log.WarnS( + context.Background(), + "Tip poller reorg deeper than retained history; "+ + "giving up walk-back", + fmt.Errorf("history exhausted"), + slog.Int("new_tip_height", int(newTipHeight)), + slog.Int("history_size", t.historySize), + ) + + return + } + + // Build the Disconnected slice from cached hashes between + // forkHeight+1 and the cached tipHeight in one critical + // section. The walk is already in ascending order so no sort + // is needed; we cap at the history size to bound the worst + // case under t.mu. + t.mu.Lock() + cachedTipHeight = t.tipHeight + disconnected := make( + []chainhash.Hash, 0, int(cachedTipHeight-forkHeight), + ) + for h := forkHeight + 1; h <= cachedTipHeight; h++ { + if hash, ok := t.hashAtHeightLocked(h); ok { + disconnected = append(disconnected, hash) + } + } + t.mu.Unlock() + + // Build the Connected slice. Heights range from forkHeight+1 + // to newTipHeight; for each, fetch the header so the + // TipBlock carries a populated Header field for downstream + // consumers. + connectedHeights := make([]int32, 0, len(connectedByHeight)) + for h := range connectedByHeight { + if h <= forkHeight { + continue + } + connectedHeights = append(connectedHeights, h) + } + sort.Slice(connectedHeights, func(i, j int) bool { + return connectedHeights[i] < connectedHeights[j] + }) + + connected := make([]*TipBlock, 0, len(connectedHeights)) + for _, h := range connectedHeights { + hash := connectedByHeight[h] + header, err := t.esplora.GetBlockHeader( + context.Background(), hash, + ) + if err != nil { + t.log.WarnS( + context.Background(), + "Tip poller reorg header fetch failed", + err, + slog.String("hash", hash.String()), ) return } + + connected = append(connected, &TipBlock{ + Height: h, + Hash: hash, + Header: header, + }) + } + + // Prune the now-stale hashes from the history before we + // broadcast the reorg so a subscriber that immediately calls + // back into BestBlock sees the new tip. + t.mu.Lock() + for h := forkHeight + 1; h <= cachedTipHeight; h++ { + delete(t.recentHashes, h) + } + t.mu.Unlock() + + reorgEvent := &ReorgEvent{ + ForkHeight: forkHeight, + Disconnected: disconnected, + Connected: connected, + } + + t.log.InfoS(context.Background(), "Tip poller emitting reorg", + slog.Int("fork_height", int(forkHeight)), + slog.Int("disconnected", len(disconnected)), + slog.Int("connected", len(connected)), + ) + + if err := t.reorgs.SendUpdate(reorgEvent); err != nil { + t.log.WarnS( + context.Background(), + "Tip poller reorg send failed", + err, + ) + + return + } + + // Also emit the reorg on the unified chain stream BEFORE any + // connected-block tip events land on it, so a single consumer + // reading the chain subscription sees Reorged before any of + // the replacement Connected blocks. This is the load-bearing + // ordering property the chain.Interface adapter needs to emit + // BlockDisconnected before BlockConnected, and the chainsource + // backend needs to reset registrations before block-epoch + // driven re-checks run on the replacement chain. + if err := t.chain.SendUpdate( + &ChainEvent{Reorg: reorgEvent}, + ); err != nil { + + t.log.WarnS( + context.Background(), + "Tip poller chain reorg send failed", + err, + ) + + return + } + + // Finally, fan the connected blocks out on the standard + // TipBlock stream so existing consumers re-check on each + // new block exactly as they would on a non-reorg advance. + // broadcastTipBlock also pushes each block onto the unified + // chain stream so the single-channel consumer observes the + // connected blocks immediately after the reorg event in the + // same producer-ordered sequence. + for _, block := range connected { + if !t.broadcastTipBlock(block) { + return + } } } diff --git a/lwwallet/tip_poller_reorg_test.go b/lwwallet/tip_poller_reorg_test.go new file mode 100644 index 000000000..0b9dd2cbc --- /dev/null +++ b/lwwallet/tip_poller_reorg_test.go @@ -0,0 +1,274 @@ +package lwwallet + +import ( + "fmt" + "testing" + "time" + + "github.com/btcsuite/btcd/chainhash/v2" + "github.com/btcsuite/btcd/wire/v2" + "github.com/btcsuite/btclog/v2" + "github.com/stretchr/testify/require" +) + +// mintStubHeaderGen is like mintStubHeader but mixes a fork generation +// into the salt so re-minting the same height on a forked chain yields +// a distinct BlockHash. This lets a test rebuild a height range on a +// different chain whose hashes provably diverge from the original. +func mintStubHeaderGen(height, gen int32, + prev chainhash.Hash) *wire.BlockHeader { + + salt := chainhash.HashH( + fmt.Appendf(nil, "stub-block-%d-gen-%d", height, gen), + ) + + return &wire.BlockHeader{ + Version: 1, + PrevBlock: prev, + MerkleRoot: chainhash.HashH( + fmt.Appendf(nil, "merkle-%d-gen-%d", height, gen), + ), + Timestamp: time.Unix(int64(height)*600, 0), + Bits: 0x207fffff, + Nonce: uint32(salt[0])<<24 | uint32(salt[1])<<16 | + uint32(salt[2])<<8 | uint32(salt[3]), + } +} + +// reorgTo rewrites the chain onto a fork that branches at forkHeight and +// extends to newTip. newTip may be lower than the current tip (a shorter +// but higher-work chain), equal to it (a same-height hash replacement), +// or higher (a deeper forward reorg). Heights above forkHeight are +// replaced with generation-salted blocks whose hashes diverge from the +// old chain; any old heights above newTip are orphaned (removed) so the +// stubbed backend no longer serves them. +func (c *stubChain) reorgTo(t *testing.T, forkHeight, newTip, gen int32) { + t.Helper() + + c.mu.Lock() + defer c.mu.Unlock() + + // Drop every height above the fork point; the new chain rebuilds + // forkHeight+1 .. newTip and orphans anything beyond newTip. + for h := range c.hashAt { + if h > forkHeight { + delete(c.hashAt, h) + delete(c.blocks, h) + } + } + + prev := c.hashAt[forkHeight] + for h := forkHeight + 1; h <= newTip; h++ { + hdr := mintStubHeaderGen(h, gen, prev) + c.blocks[h] = hdr + hash := hdr.BlockHash() + c.hashAt[h] = hash + prev = hash + } + + c.tipHeight = newTip +} + +// setTipHeight lowers (or raises) the reported tip height WITHOUT +// touching any cached hashes. It models a transient indexer hiccup +// where the remote momentarily reports fewer blocks than it served a +// moment ago, but is still on the same chain. +func (c *stubChain) setTipHeight(h int32) { + c.mu.Lock() + defer c.mu.Unlock() + + c.tipHeight = h +} + +// requireTipEventually blocks until the poller's BestBlock height +// reaches want, failing the test if it does not within the deadline. +func requireTipEventually(t *testing.T, tp *TipPoller, want int32) { + t.Helper() + + require.Eventually(t, func() bool { + h, _, _ := tp.BestBlock() + + return h == want + }, 2*time.Second, 5*time.Millisecond, + "poller never reached tip height %d", want) +} + +// newReorgTestPoller spins up a TipPoller over a stubChain seeded at the +// given height and returns both so a test can drive reorgs. +func newReorgTestPoller(t *testing.T, + seedHeight int32) (*TipPoller, *stubChain) { + + t.Helper() + + chain := newStubChain(seedHeight) + srv := mockEsploraServer(t, stubEsploraHandler(t, chain)) + + tp := NewTipPoller( + NewEsploraClient(srv.URL, btclog.Disabled), 10*time.Millisecond, + btclog.Disabled, + ) + require.NoError(t, tp.Start()) + t.Cleanup(tp.Stop) + + return tp, chain +} + +// TestTipPollerShorterChainReorg covers the case Roasbeef flagged: a +// reorg onto a SHORTER but higher-work chain, where blocks above the new +// tip are orphaned. The poller must detect this even though the remote +// reports a lower height than the cached tip, and roll the tip back to +// the shorter chain's tip. +func TestTipPollerShorterChainReorg(t *testing.T) { + t.Parallel() + + tp, chain := newReorgTestPoller(t, 100) + + reorgSub, err := tp.SubscribeReorgs() + require.NoError(t, err) + defer reorgSub.Cancel() + + // Grow to 110 so the poller caches 101..110, then confirm it + // observed the advance before we reorg out from under it. + chain.advance(t, 10) + requireTipEventually(t, tp, 110) + + // Fork at 105 onto a shorter chain that tips at 108 (< 110). + chain.reorgTo(t, 105, 108, 1) + + select { + case ev := <-reorgSub.Updates(): + require.NotNil(t, ev) + require.Equal(t, int32(105), ev.ForkHeight) + + // Old chain 106..110 disconnects (5 blocks), even though + // the new chain only carries 106..108. + require.Len(t, ev.Disconnected, 5) + + // New chain connects 106..108 (3 blocks) in ascending + // order, tipping at 108. + require.Len(t, ev.Connected, 3) + require.Equal( + t, int32(106), ev.Connected[0].Height, + ) + require.Equal( + t, int32(108), ev.Connected[len(ev.Connected)-1].Height, + ) + + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for shorter-chain reorg event") + } + + // The poller's tip must now be the shorter chain's tip. + requireTipEventually(t, tp, 108) +} + +// TestTipPollerShorterHeightSameChainNoReorg verifies the no-op half of +// the shorter-height branch: a transient indexer hiccup where the remote +// reports fewer blocks but is still on the same chain (the hash at the +// reported height matches our cache). No reorg must fire and the cached +// tip must NOT roll back. +func TestTipPollerShorterHeightSameChainNoReorg(t *testing.T) { + t.Parallel() + + tp, chain := newReorgTestPoller(t, 100) + + reorgSub, err := tp.SubscribeReorgs() + require.NoError(t, err) + defer reorgSub.Cancel() + + chain.advance(t, 10) + requireTipEventually(t, tp, 110) + + // Remote briefly reports height 108 with the SAME hash history + // (no fork). This is lag, not a reorg. + chain.setTipHeight(108) + + select { + case ev := <-reorgSub.Updates(): + t.Fatalf("unexpected reorg on transient lag: %+v", ev) + + case <-time.After(300 * time.Millisecond): + // No reorg fired: correct. + } + + // The tip must remain at 110: we never roll back on a lagging + // remote that is still on our chain. + h, _, _ := tp.BestBlock() + require.Equal(t, int32(110), h) +} + +// TestTipPollerSameHeightReorg covers a same-height hash replacement: the +// tip height does not change but the block at the tip is replaced by a +// different block on a competing chain. +func TestTipPollerSameHeightReorg(t *testing.T) { + t.Parallel() + + tp, chain := newReorgTestPoller(t, 100) + + reorgSub, err := tp.SubscribeReorgs() + require.NoError(t, err) + defer reorgSub.Cancel() + + chain.advance(t, 10) + requireTipEventually(t, tp, 110) + + oldHash := chain.hashAt[110] + + // Replace block 110 with a different block at the same height. + chain.reorgTo(t, 109, 110, 1) + + select { + case ev := <-reorgSub.Updates(): + require.NotNil(t, ev) + require.Equal(t, int32(109), ev.ForkHeight) + require.Len(t, ev.Disconnected, 1) + require.Equal(t, oldHash, ev.Disconnected[0]) + require.Len(t, ev.Connected, 1) + require.Equal(t, int32(110), ev.Connected[0].Height) + + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for same-height reorg event") + } + + requireTipEventually(t, tp, 110) +} + +// TestTipPollerDeeperForwardReorg covers a reorg that also extends the +// chain: the new chain forks below the old tip yet ends higher than it. +// The forward-advance path must notice the broken PrevBlock continuity +// at the old tip boundary and emit a reorg before the new tip events. +func TestTipPollerDeeperForwardReorg(t *testing.T) { + t.Parallel() + + tp, chain := newReorgTestPoller(t, 100) + + reorgSub, err := tp.SubscribeReorgs() + require.NoError(t, err) + defer reorgSub.Cancel() + + chain.advance(t, 10) + requireTipEventually(t, tp, 110) + + // Fork at 107 onto a longer chain that tips at 113. + chain.reorgTo(t, 107, 113, 1) + + select { + case ev := <-reorgSub.Updates(): + require.NotNil(t, ev) + require.Equal(t, int32(107), ev.ForkHeight) + + // Old chain 108..110 disconnects (3 blocks). + require.Len(t, ev.Disconnected, 3) + + // New chain connects 108..113 (6 blocks). + require.Len(t, ev.Connected, 6) + require.Equal( + t, int32(113), ev.Connected[len(ev.Connected)-1].Height, + ) + + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for deeper forward reorg event") + } + + requireTipEventually(t, tp, 113) +} diff --git a/lwwallet/tip_poller_test.go b/lwwallet/tip_poller_test.go index e4aed4a9c..cf75fa16b 100644 --- a/lwwallet/tip_poller_test.go +++ b/lwwallet/tip_poller_test.go @@ -1,6 +1,8 @@ package lwwallet import ( + "bytes" + "encoding/hex" "fmt" "net/http" "sync" @@ -9,6 +11,7 @@ import ( "time" "github.com/btcsuite/btcd/chainhash/v2" + "github.com/btcsuite/btcd/wire/v2" "github.com/btcsuite/btclog/v2" "github.com/stretchr/testify/require" ) @@ -16,28 +19,59 @@ import ( // stubChain is a tiny test fixture that simulates an Esplora chain // where the tip height can be advanced under test control. It is // independent of the larger mockEsploraServer helper so tip-poller -// tests can drive the chain forward synchronously. +// tests can drive the chain forward synchronously. Each height holds +// a real wire.BlockHeader whose PrevBlock chains back to the previous +// height; the poller's PrevBlock continuity check on advance therefore +// passes naturally rather than aborting. type stubChain struct { mu sync.Mutex tipHeight int32 - // hashAt[h] is the hash for height h. We pre-populate as the - // tip advances so GetBlockHashByHeight resolves consistently. + // blocks[h] holds the wire.BlockHeader for height h. Pre- + // populated through 0..tipHeight at construction and grown + // monotonically by advance. + blocks map[int32]*wire.BlockHeader + + // hashAt[h] is the chainhash.Hash for height h. hashAt map[int32]chainhash.Hash } +// mintStubHeader builds a deterministic wire.BlockHeader for a given +// height, chaining off the supplied previous hash. The salted nonce +// makes the resulting BlockHash stable per (height, prev) and unique +// across heights. +func mintStubHeader(height int32, prev chainhash.Hash) *wire.BlockHeader { + salt := chainhash.HashH([]byte(fmt.Sprintf("stub-block-%d", height))) + + return &wire.BlockHeader{ + Version: 1, + PrevBlock: prev, + MerkleRoot: chainhash.HashH( + []byte( + fmt.Sprintf("merkle-%d", height), + ), + ), + Timestamp: time.Unix(int64(height)*600, 0), + Bits: 0x207fffff, + Nonce: uint32(salt[0])<<24 | uint32(salt[1])<<16 | + uint32(salt[2])<<8 | uint32(salt[3]), + } +} + func newStubChain(tipHeight int32) *stubChain { c := &stubChain{ tipHeight: tipHeight, + blocks: make(map[int32]*wire.BlockHeader), hashAt: make(map[int32]chainhash.Hash), } + var prev chainhash.Hash for h := int32(0); h <= tipHeight; h++ { - c.hashAt[h] = chainhash.HashH( - []byte( - fmt.Sprintf("block-%d", h), - ), - ) + hdr := mintStubHeader(h, prev) + c.blocks[h] = hdr + hash := hdr.BlockHash() + c.hashAt[h] = hash + prev = hash } return c @@ -49,13 +83,14 @@ func (c *stubChain) advance(t *testing.T, n int32) { c.mu.Lock() defer c.mu.Unlock() + prev := c.hashAt[c.tipHeight] for i := int32(1); i <= n; i++ { h := c.tipHeight + i - c.hashAt[h] = chainhash.HashH( - []byte( - fmt.Sprintf("block-%d", h), - ), - ) + hdr := mintStubHeader(h, prev) + c.blocks[h] = hdr + hash := hdr.BlockHash() + c.hashAt[h] = hash + prev = hash } c.tipHeight += n @@ -147,19 +182,29 @@ func stubEsploraHandler(t *testing.T, chain *stubChain) http.HandlerFunc { h.String(), height, int64(height)*600) + case "/header": + // Raw 80-byte header, hex-encoded. + chain.mu.Lock() + hdr := chain.blocks[height] + chain.mu.Unlock() + if hdr == nil { + http.Error( + w, "not found", + http.StatusNotFound, + ) + + return + } + var buf bytes.Buffer + require.NoError(t, hdr.Serialize(&buf)) + _, _ = fmt.Fprint( + w, + hex.EncodeToString( + buf.Bytes(), + ), + ) + default: - // Raw header / raw block — synthesize a - // header whose serialized bytes hash to - // h. We take the simple route of using - // h's bytes themselves: the header - // hash-verifier compares header.BlockHash() - // to the requested h, so any header that - // happens to round-trip works. Synthesizing - // such a header from a target hash is - // effectively impossible, so for these - // suffix variants we return 501 — tests - // that need them must use the cache - // pre-fill path. http.Error( w, "not implemented", http.StatusNotImplemented, diff --git a/systest/reorg_test.go b/systest/reorg_test.go new file mode 100644 index 000000000..bdb19a64b --- /dev/null +++ b/systest/reorg_test.go @@ -0,0 +1,244 @@ +//go:build systest + +package systest + +import ( + "crypto/sha256" + "testing" + "time" + + btcaddr "github.com/btcsuite/btcd/address/v2" + "github.com/btcsuite/btcd/btcutil/v2" + "github.com/btcsuite/btcd/chaincfg/v2" + "github.com/btcsuite/btcd/chainhash/v2" + "github.com/btcsuite/btcd/txscript/v2" + "github.com/lightninglabs/darepo-client/baselib/actor" + "github.com/lightninglabs/darepo-client/chainsource" + "github.com/lightningnetwork/lnd/fn/v2" + "github.com/stretchr/testify/require" +) + +// reorgSystestEventTimeout is the per-step deadline used by the +// chainsource reorg systests. The chain-notification pipeline runs +// over gRPC to a real lnd instance, so the timeout is generous enough +// to absorb container startup variance and notifier wake-up latency. +const reorgSystestEventTimeout = 30 * time.Second + +// TestChainSourceConfReorgRoundTrip drives a real bitcoind reorg +// through the full chainsource pipeline: +// +// lnd chainntnfs (in-process) +// -> lndclient gRPC (WithReOrgChan) +// -> chainbackends.LndClientChainNotifier (bridge) +// -> chainbackends.LNDBackend (multi-shot forwarder) +// -> chainsource.ConfActor (reorg-aware mode) +// -> test actor refs +// +// The flow is: +// +// 1. Register a reorg-aware confirmation watch on a synthetic P2WPKH +// pkScript whose txid we know once we faucet to it. +// 2. Faucet + mine one block. Assert ConfirmationEvent arrives with +// the expected (txid, blockHeight, blockHash). +// 3. Drive a 1-block reorg via the harness helper, which invalidates +// the confirmation block and mines a strictly longer (2-block) +// replacement branch. Bitcoind preserves the original tx in its +// mempool, so the transaction re-confirms in the new chain at a +// different block hash (and potentially the same height). +// 4. Assert ConfReorgedEvent arrives. +// 5. Assert a fresh ConfirmationEvent arrives with the new chain's +// block hash (NOT the original one), demonstrating that lnd's +// chainntnfs dispatched a re-confirmation after the reorg and +// that every layer above it propagated the multi-shot signal +// correctly. +// +// This is the systest-level oracle for "the reorg-aware pipeline +// actually works over the real gRPC transport". The unit tests in +// chainsource/reorg_test.go and chainbackends/lnd_reorg_test.go prove +// the same lifecycle against mocks, but they cannot prove that +// lndclient.WithReOrgChan fires when wired to real lnd. Only this +// test does. +func TestChainSourceConfReorgRoundTrip(t *testing.T) { + ParallelN(t) + + h := NewSysTestHarness(t) + ctx := h.Context() + + // Spawn a real chainsource actor over the harness's LND. + chainSource := h.NewChainSourceActor() + + // Build a synthetic P2WPKH address from a deterministic per-test + // pubkey hash. The address does not need a controllable key (we + // never spend it in this test); we just need a known pkScript so + // we can register a confirmation watch. + pubKeyHash := sha256.Sum256([]byte(t.Name())) + addr, err := btcaddr.NewAddressWitnessPubKeyHash( + pubKeyHash[:20], &chaincfg.RegressionNetParams, + ) + require.NoError(t, err, "build synthetic P2WPKH address") + pkScript, err := txscript.PayToAddrScript(addr) + require.NoError(t, err, "derive pkScript for synthetic address") + + // Wire test refs for each event variant. The Reorged ref also + // has to be set for NotifyDone-or-NotifyReorged admission to flip + // the sub-actor into multi-shot mode; we leave Done unwired + // because the lndclient transport does not synthesize a Done + // signal so it would never fire over a real lnd run anyway. + confRef := actor.NewChannelTellOnlyRef[chainsource.ConfirmationEvent]( + "systest-conf", 8, + ) + reorgRef := actor.NewChannelTellOnlyRef[chainsource.ConfReorgedEvent]( + "systest-conf-reorged", 8, + ) + + // Register the watch BEFORE the tx hits the mempool so we + // exercise the live-detection path rather than the + // historical-backfill path. + heightHint := h.Harness.BlockCount() + amount := btcutil.Amount(btcutil.SatoshiPerBitcoin / 100) + + // We need the txid up front, which means faucet first, then + // register, then mine. The watch is on the txid + pkScript pair + // so the ordering between mempool entry and registration is OK; + // what must NOT happen is that we mine the block before + // registering, because lnd's notifier would then dispatch + // historical confirmation state and our test would be racing two + // delivery paths. + txidStr := h.Harness.Faucet(addr.String(), amount) + txid, err := chainhash.NewHashFromStr(txidStr) + require.NoError(t, err, "parse faucet txid") + + confNotify := actor.TellOnlyRef[chainsource.ConfirmationEvent]( + confRef, + ) + reorgNotify := actor.TellOnlyRef[chainsource.ConfReorgedEvent]( + reorgRef, + ) + + regResp := chainSource.Ask(ctx, &chainsource.RegisterConfRequest{ + CallerID: "test-reorg-conf-" + txidStr, + Txid: txid, + PkScript: pkScript, + TargetConfs: 1, + HeightHint: heightHint, + NotifyActor: fn.Some(confNotify), + NotifyReorged: fn.Some(reorgNotify), + }).Await(ctx) + require.True(t, regResp.IsOk(), "register reorg-aware conf watch") + resp, err := regResp.Unpack() + require.NoError(t, err) + _, ok := resp.(*chainsource.RegisterConfResponse) + require.True(t, ok, "unexpected register response type") + + // 1. Mine the block that confirms the faucet tx. + originalBlocks := h.Harness.Generate(1) + require.Len(t, originalBlocks, 1) + originalBlock := originalBlocks[0] + + originalHash, err := chainhash.NewHashFromStr(originalBlock.Hash) + require.NoError(t, err, "parse original block hash") + + // 2. Assert the first ConfirmationEvent. + firstConf := awaitConfEvent(t, confRef) + require.Equal(t, *txid, firstConf.Txid, "first conf txid mismatch") + require.Equal( + t, int32(originalBlock.Height), firstConf.BlockHeight, + "first conf block height should match the mined block", + ) + require.Equal( + t, *originalHash, firstConf.BlockHash, + "first conf block hash should match the mined block", + ) + t.Logf( + "first ConfirmationEvent: txid=%s height=%d hash=%s", + firstConf.Txid, firstConf.BlockHeight, firstConf.BlockHash, + ) + + // 3. Drive a reorg: invalidate the conf block, mine a strictly + // longer replacement branch. The harness Reorg helper waits for + // lnd's chain sync to catch up before returning. + reorg := h.Harness.Reorg(1, 2) + require.Equal( + t, originalBlock.Hash, reorg.Disconnected[0].Hash, + "the reorg should have disconnected the conf block", + ) + require.Len(t, reorg.Connected, 2) + t.Logf( + "reorg: disconnected=%d connected=%d fork=%d", + len(reorg.Disconnected), len(reorg.Connected), + reorg.ForkPoint.Height, + ) + + // 4. Assert the ConfReorgedEvent. Lnd's chainntnfs notifier + // dispatches NegativeConf for the disconnected confirmation + // asynchronously after processing the disconnect; the gRPC + // transport adds a further hop, so this can take longer than + // the initial confirmation event. + reorgEvt := awaitReorgEvent(t, reorgRef) + require.Equal(t, *txid, reorgEvt.Txid, "reorg event txid mismatch") + t.Logf("ConfReorgedEvent: txid=%s", reorgEvt.Txid) + + // 5. Assert a fresh ConfirmationEvent for the same tx, now in + // the replacement chain. Bitcoind keeps the tx in mempool across + // the invalidate, so generatetoaddress picks it back up on the + // first new block. The new block hash MUST differ from the + // original; the height may or may not match depending on where + // the tx landed in the replacement branch. + secondConf := awaitConfEvent(t, confRef) + require.Equal(t, *txid, secondConf.Txid, "re-conf txid mismatch") + require.NotEqual( + t, firstConf.BlockHash, secondConf.BlockHash, + "re-confirmation must arrive in a new block", + ) + t.Logf( + "second ConfirmationEvent: txid=%s height=%d hash=%s", + secondConf.Txid, secondConf.BlockHeight, secondConf.BlockHash, + ) + + // Sanity: the new block hash must be one of the harness-reported + // connected blocks. chainhash.Hash.String() renders the + // canonical big-endian hex form that bitcoind's RPCs emit, so + // the strings can be compared directly. + connectedHashes := make(map[string]struct{}, len(reorg.Connected)) + for _, blk := range reorg.Connected { + connectedHashes[blk.Hash] = struct{}{} + } + require.Contains( + t, connectedHashes, secondConf.BlockHash.String(), + "re-confirmation block must belong to the replacement branch", + ) +} + +// awaitConfEvent reads a single ConfirmationEvent from the test ref +// with a generous deadline, failing the test on timeout. +func awaitConfEvent(t *testing.T, + ref *actor.ChannelTellOnlyRef[chainsource.ConfirmationEvent], +) chainsource.ConfirmationEvent { + + t.Helper() + + evt, ok := ref.AwaitMessage(reorgSystestEventTimeout) + require.True( + t, ok, "timeout waiting for ConfirmationEvent (%s)", + reorgSystestEventTimeout, + ) + + return evt +} + +// awaitReorgEvent reads a single ConfReorgedEvent from the test ref +// with a generous deadline, failing the test on timeout. +func awaitReorgEvent(t *testing.T, + ref *actor.ChannelTellOnlyRef[chainsource.ConfReorgedEvent], +) chainsource.ConfReorgedEvent { + + t.Helper() + + evt, ok := ref.AwaitMessage(reorgSystestEventTimeout) + require.True( + t, ok, "timeout waiting for ConfReorgedEvent (%s)", + reorgSystestEventTimeout, + ) + + return evt +} diff --git a/systest/systest.go b/systest/systest.go index bfdc46a03..c2f264876 100644 --- a/systest/systest.go +++ b/systest/systest.go @@ -212,6 +212,13 @@ func (h *SysTestHarness) NewChainSourceActor() actor.ActorRef[ chainsource.ChainSourceConfig{ Backend: backend, System: h.actorSystem, + // Mirror the production wiring so systests that + // register reorg-aware conf/spend watches do not + // leak per-watch sub-actors past test teardown: + // the lndclient backend never writes the upstream + // Done channel, so without height-based synthesis + // reorg-aware watches would stay open forever. + FinalityDepth: chainsource.DefaultFinalityDepth, }.WithLogger( h.SubLogger(chainsource.Subsystem), ), diff --git a/systest/txconfirm_reorg_test.go b/systest/txconfirm_reorg_test.go new file mode 100644 index 000000000..e52bbae20 --- /dev/null +++ b/systest/txconfirm_reorg_test.go @@ -0,0 +1,221 @@ +//go:build systest + +package systest + +import ( + "context" + "crypto/sha256" + "testing" + "time" + + btcaddr "github.com/btcsuite/btcd/address/v2" + "github.com/btcsuite/btcd/btcutil/v2" + "github.com/btcsuite/btcd/chaincfg/v2" + "github.com/btcsuite/btcd/txscript/v2" + "github.com/lightninglabs/darepo-client/baselib/actor" + "github.com/lightninglabs/darepo-client/txconfirm" + "github.com/stretchr/testify/require" +) + +// txConfirmSystestEventTimeout is the per-step deadline used by the +// txconfirm reorg systest. Generous because the chain-notification +// pipeline runs over gRPC to a real lnd instance plus this test waits +// for txconfirm's tracked-tx FSM to forward each event. +const txConfirmSystestEventTimeout = 30 * time.Second + +// recordingNotificationRef captures every Notification delivered to a +// txconfirm subscriber so the test can assert on the order and shape +// of the lifecycle without racing concurrent delivery. +type recordingNotificationRef struct { + id string + msgs chan txconfirm.Notification +} + +func newRecordingNotificationRef(id string) *recordingNotificationRef { + return &recordingNotificationRef{ + id: id, + msgs: make(chan txconfirm.Notification, 16), + } +} + +// ID returns the subscriber identifier. +func (r *recordingNotificationRef) ID() string { + return r.id +} + +// Tell records the inbound notification on the channel for the test to +// consume. +func (r *recordingNotificationRef) Tell(_ context.Context, + msg txconfirm.Notification) error { + + r.msgs <- msg + + return nil +} + +// await pulls the next notification or fails the test on timeout. +func (r *recordingNotificationRef) await(t *testing.T) txconfirm.Notification { + t.Helper() + + select { + case msg := <-r.msgs: + return msg + + case <-time.After(txConfirmSystestEventTimeout): + t.Fatalf("timeout waiting for txconfirm notification (%s)", + txConfirmSystestEventTimeout) + + return nil + } +} + +// TestTxConfirmReorgRoundTrip drives a real bitcoind reorg through +// the full txconfirm pipeline: +// +// lnd chainntnfs (in-process) +// -> lndclient gRPC (WithReOrgChan) +// -> chainbackends.LndClientChainNotifier (bridge) +// -> chainbackends.LNDBackend (multi-shot forwarder) +// -> chainsource.ConfActor (reorg-aware mode + finality synth) +// -> txconfirm.TxBroadcasterActor (tracked-tx FSM) +// -> recording subscriber +// +// This is the systest-level oracle for the layer that unroll consumes. +// The chainsource-level systest (TestChainSourceConfReorgRoundTrip) +// proves the chain-event plumbing; this one proves the tracked-tx FSM +// transitions Confirmed -> AwaitingConfirmation -> Confirmed correctly +// on real reorgs and that subscribers see TxConfirmed -> TxReorged -> +// TxConfirmed in order, with TxFinalized arriving once the +// height-based safety depth is reached. +// +// Full daemon-level end-to-end coverage (the VTXOUnrollActor walking +// a real proof through a real wallet under a real reorg) belongs in +// itest; the unroll FSM's reducer behavior on these events is already +// covered by unit tests in unroll/reorg_safety_test.go. +func TestTxConfirmReorgRoundTrip(t *testing.T) { + ParallelN(t) + + h := NewSysTestHarness(t) + ctx := h.Context() + + chainSource := h.NewChainSourceActor() + + // Spawn a txconfirm actor over the real chainsource. Wallet is + // nil because the faucet tx is non-anchor and we never trigger + // CPFP fee-input selection in this test. + txconfBehavior := txconfirm.NewTxBroadcasterActor(txconfirm.Config{ + ChainSource: chainSource, + }) + txconfInstance := actor.NewActor(actor.ActorConfig[ + txconfirm.Msg, txconfirm.Resp, + ]{ + ID: "txconfirm-systest", + Behavior: txconfBehavior, + MailboxSize: 64, + }) + txconfBehavior.SetSelfRef(txconfInstance.TellRef()) + txconfInstance.Start() + t.Cleanup(txconfInstance.Stop) + + // Synthetic watched address: deterministic P2WPKH derived from + // the test name. We faucet to it so we have a known txid and + // pkScript for txconfirm to track. The address is never spent. + pubKeyHash := sha256.Sum256([]byte(t.Name())) + addr, err := btcaddr.NewAddressWitnessPubKeyHash( + pubKeyHash[:20], &chaincfg.RegressionNetParams, + ) + require.NoError(t, err, "build synthetic P2WPKH address") + pkScript, err := txscript.PayToAddrScript(addr) + require.NoError(t, err, "derive pkScript for synthetic address") + + heightHint := h.Harness.BlockCount() + + // txconfirm's CPFP broadcaster enforces v3/TRUC at the version + // gate, and the standard bitcoind faucet returns v2 txs. Build a + // v3 tx that pays our synthetic address from bitcoind's wallet, + // sign it, but do NOT broadcast — txconfirm will handle the + // first broadcast on EnsureConfirmedReq. + signedTx := h.Harness.SignedV3Tx( + pkScript, btcutil.Amount(btcutil.SatoshiPerBitcoin/100), + ) + txidVal := signedTx.TxHash() + txid := &txidVal + t.Logf("constructed v3 tx: txid=%s", txid) + + subscriber := newRecordingNotificationRef("txconfirm-sub") + var subRef actor.TellOnlyRef[txconfirm.Notification] = subscriber + + // Register the EnsureConfirmedReq BEFORE mining so we exercise + // the live-detection path and the tracked-tx FSM walks the full + // Broadcasting -> AwaitingConfirmation -> Confirmed transitions. + ensureResp, err := txconfInstance.Ref().Ask( + ctx, &txconfirm.EnsureConfirmedReq{ + Tx: signedTx, + ConfirmationPkScript: pkScript, + Label: "systest-reorg", + HeightHint: heightHint, + TargetConfs: 1, + Subscriber: subRef, + }, + ).Await(ctx).Unpack() + require.NoError(t, err, "EnsureConfirmedReq failed") + require.IsType(t, &txconfirm.EnsureConfirmedResp{}, ensureResp) + + // 1. Mine the block that confirms the faucet tx. + originalBlocks := h.Harness.Generate(1) + require.Len(t, originalBlocks, 1) + originalBlock := originalBlocks[0] + + // 2. Expect TxConfirmed. + first := subscriber.await(t) + firstConfirmed, ok := first.(*txconfirm.TxConfirmed) + require.True( + t, ok, "first notification must be TxConfirmed, got %T", first, + ) + require.Equal(t, *txid, firstConfirmed.Txid) + require.Equal( + t, int32(originalBlock.Height), firstConfirmed.BlockHeight, + "first conf block height should match the mined block", + ) + t.Logf( + "first TxConfirmed: txid=%s height=%d", firstConfirmed.Txid, + firstConfirmed.BlockHeight, + ) + + // 3. Reorg the conf block out, mine a strictly longer + // replacement branch, wait for lnd to chain-sync. + reorg := h.Harness.Reorg(1, 2) + require.Equal( + t, originalBlock.Hash, reorg.Disconnected[0].Hash, + "the reorg should have disconnected the conf block", + ) + require.Len(t, reorg.Connected, 2) + t.Logf( + "reorg: disconnected=%d connected=%d fork=%d", + len(reorg.Disconnected), len(reorg.Connected), + reorg.ForkPoint.Height, + ) + + // 4. Expect TxReorged. + second := subscriber.await(t) + reorgedMsg, ok := second.(*txconfirm.TxReorged) + require.True( + t, ok, "second notification must be TxReorged, got %T", second, + ) + require.Equal(t, *txid, reorgedMsg.Txid) + t.Logf("TxReorged: txid=%s", reorgedMsg.Txid) + + // 5. Expect a fresh TxConfirmed on the replacement chain. The + // faucet tx stays in mempool across the invalidate so it + // re-confirms in the first new block. + third := subscriber.await(t) + secondConfirmed, ok := third.(*txconfirm.TxConfirmed) + require.True( + t, ok, "third notification must be TxConfirmed, got %T", third, + ) + require.Equal(t, *txid, secondConfirmed.Txid) + t.Logf( + "second TxConfirmed: txid=%s height=%d", secondConfirmed.Txid, + secondConfirmed.BlockHeight, + ) +} diff --git a/txconfirm/actor.go b/txconfirm/actor.go index a2d88a491..0fff91500 100644 --- a/txconfirm/actor.go +++ b/txconfirm/actor.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "log/slog" + "sync/atomic" "time" "github.com/btcsuite/btcd/chainhash/v2" @@ -52,6 +53,14 @@ var ( // waiting longer only risks blocking unrelated confirmation work behind // a durable subscriber's DB writer. terminalNotifyTimeout = time.Second + + // reversibleNotifyTimeout bounds how long a fire-and-forget reversible + // notification goroutine (TxConfirmed, TxReorged) is willing to wait + // on a slow subscriber's mailbox before logging the drop and returning. + // Reversible deliveries are best-effort: the next state transition on + // the same tracked tx supersedes the missed event, so we trade a stale + // notification for keeping txconfirm's actor loop unblocked. + reversibleNotifyTimeout = time.Second ) // ErrEnsureParamsMismatch is returned by EnsureConfirmedReq when a second @@ -185,6 +194,24 @@ type TxBroadcasterActor struct { blockSubscriptionActive bool } +// trackedSubscriber is one attached subscriber to a tracked tx. Every +// subscriber receives the full reorg-aware lifecycle (TxConfirmed, +// TxReorged, re-TxConfirmed, TxFinalized, TxFailed) and remains +// attached until TxFinalized or TxFailed is acknowledged. +// +// pendingConfirmed reports whether the INITIAL TxConfirmed delivery +// is still owed to this subscriber. It is true at admission, flipped +// false the first time notifyOneConfirmed lands successfully, and +// drives retryConfirmedRedelivery's per-tick retry loop so the +// at-least-once initial-TxConfirmed contract holds even when the +// subscriber's mailbox is briefly slow. Re-confirmations after a +// reorg are delivered best-effort because the eventual TxFinalized +// reliably carries the final height/numConfs. +type trackedSubscriber struct { + Ref actor.TellOnlyRef[Notification] + pendingConfirmed bool +} + // trackedTx stores the actor-owned handle for one tracked txid. // // The struct is the actor's single source of truth about a tracked @@ -195,7 +222,7 @@ type trackedTx struct { data trackedTxData fsm *trackedTxStateMachine - subscribers map[string]actor.TellOnlyRef[Notification] + subscribers map[string]trackedSubscriber // escalateLog rate-limits the operator-facing escalation that fires // once a tx has failed to reach any mempool repeatedly. It @@ -219,6 +246,17 @@ type trackedTx struct { // subsequent interval-paced bumps fall back to the estimator. Zero // means "no override pending". pendingTargetFeeRate int64 + + // sealed is set true the moment terminal delivery (TxFinalized / + // TxFailed) begins for this entry. Reversible deliveries + // (TxReorged / re-TxConfirmed) run on detached fire-and-forget + // goroutines, so a reversible spawned just before finality could + // otherwise Tell its subscriber after the terminal notification and + // resurrect reorg-recovery bookkeeping the consumer already dropped. + // Each reversible goroutine checks this flag immediately before its + // Tell and skips if the entry has sealed. It is atomic because it is + // read off the actor goroutine; only the actor goroutine writes it. + sealed atomic.Bool } // confirmationObservedMsg routes a chainsource confirmation callback back into @@ -239,14 +277,53 @@ func (m *confirmationObservedMsg) MessageType() string { // set. func (m *confirmationObservedMsg) txConfirmMsgSealed() {} -// terminalNotifyResultMsg returns the result of a terminal notification that -// outlived the actor-path wait budget. +// confirmationReorgedMsg routes a chainsource ConfReorgedEvent back into +// the actor mailbox. +type confirmationReorgedMsg struct { + actor.BaseMessage + txid chainhash.Hash +} + +// MessageType returns the stable message type identifier. +func (m *confirmationReorgedMsg) MessageType() string { + return "confirmationReorgedMsg" +} + +// txConfirmMsgSealed seals confirmationReorgedMsg into the package message +// set. +func (m *confirmationReorgedMsg) txConfirmMsgSealed() {} + +// confirmationDoneMsg routes a chainsource ConfDoneEvent back into the +// actor mailbox. +type confirmationDoneMsg struct { + actor.BaseMessage + txid chainhash.Hash +} + +// MessageType returns the stable message type identifier. +func (m *confirmationDoneMsg) MessageType() string { + return "confirmationDoneMsg" +} + +// txConfirmMsgSealed seals confirmationDoneMsg into the package message +// set. +func (m *confirmationDoneMsg) txConfirmMsgSealed() {} + +// terminalNotifyResultMsg returns the result of a terminal-shape +// notification that outlived the actor-path wait budget. kind carries the +// original delivery kind ("confirmed" / "finalized" / "failed") so +// handleTerminalNotifyResult can distinguish a mid-lifecycle +// initial-TxConfirmed redelivery (where the subscriber must stay attached +// to receive later TxReorged / TxFinalized) from a truly terminal +// notification (where the subscriber should be removed and the entry can +// evict once empty). type terminalNotifyResultMsg struct { actor.BaseMessage txid chainhash.Hash subscriberID string inflightKey string + kind string err error } @@ -354,6 +431,22 @@ func (a *TxBroadcasterActor) Receive(ctx context.Context, State: TxStateConfirmed, }) + case *confirmationReorgedMsg: + a.handleConfirmationReorged(ctx, req) + + return fn.Ok[Resp](&EnsureConfirmedResp{ + Txid: req.txid, + State: TxStateAwaitingConfirmation, + }) + + case *confirmationDoneMsg: + a.handleConfirmationDone(ctx, req) + + return fn.Ok[Resp](&EnsureConfirmedResp{ + Txid: req.txid, + State: TxStateFinalized, + }) + case *blockEpochObservedMsg: a.handleBlockObserved(ctx, req) @@ -401,7 +494,23 @@ func (a *TxBroadcasterActor) OnStop(ctx context.Context) error { continue } - if state == TxStateConfirmed || state == TxStateFailed { + if isTerminalTxState(state) { + // A terminal entry can still hold a registered conf + // watch: a Failed entry never received Done, and a + // Finalized entry whose Done-driven release raced + // OnStop may not have torn it down yet. Release it so + // the chainsource sub-actor does not leak for the + // daemon's lifetime. confWatchRegistered guards the + // common case where Done already released the watch. + if entry.confWatchRegistered { + if err := a.unregisterConfWatch( + ctx, entry, + ); err != nil && firstErr == nil { + + firstErr = err + } + } + if entry.fsm != nil { entry.fsm.Stop() } @@ -476,7 +585,10 @@ func (a *TxBroadcasterActor) handleEnsure(ctx context.Context, } return a.attachExistingSubscriber( - ctx, existing, req.Subscriber, + ctx, existing, trackedSubscriber{ + Ref: req.Subscriber, + pendingConfirmed: true, + }, ), nil } @@ -700,7 +812,7 @@ func (a *TxBroadcasterActor) handleCancel(ctx context.Context, return nil, err } - if state == TxStateConfirmed || state == TxStateFailed { + if isTerminalTxState(state) { a.evictTerminal(ctx, entry) return resp, nil @@ -920,8 +1032,13 @@ func (a *TxBroadcasterActor) handleBumpNow(ctx context.Context, }, nil } -// handleConfirmationObserved marks a tracked txid as confirmed and fans the -// result out to all subscribers. +// handleConfirmationObserved advances a tracked txid into the reversible +// Confirmed state and fans TxConfirmed out to all subscribers. The +// confirmation watch is intentionally kept alive: a subsequent reorg +// would arrive on the same registration, and the entry must stay +// observable so the chainsource sub-actor's reorg/done channels reach +// this layer. The terminal Finalized state is reached separately when +// the backend emits a Done signal. func (a *TxBroadcasterActor) handleConfirmationObserved(ctx context.Context, msg *confirmationObservedMsg) { @@ -942,7 +1059,9 @@ func (a *TxBroadcasterActor) handleConfirmationObserved(ctx context.Context, return } - if state == TxStateConfirmed || state == TxStateFailed { + // Terminal entries (Failed / Finalized) are sticky; if delivery to + // any subscriber was deferred, retry it and evict on success. + if isTerminalTxState(state) { if a.retryTerminalNotifications(ctx, entry) { a.evictTerminal(ctx, entry) } @@ -950,6 +1069,17 @@ func (a *TxBroadcasterActor) handleConfirmationObserved(ctx context.Context, return } + // An already-Confirmed entry receiving another Confirmed event + // without an intervening Reorged is unexpected on a well-behaved + // backend (the chainsource sub-actor only re-fires Confirmed after + // a reorg), but we tolerate it by re-delivering TxConfirmed rather + // than failing the FSM. + if state == TxStateConfirmed { + a.notifyConfirmed(ctx, entry, msg.blockHeight, msg.numConfs) + + return + } + if err := a.advanceTrackedTxFSM(ctx, entry, &trackedTxConfirmed{ BlockHeight: msg.blockHeight, }); err != nil { @@ -960,13 +1090,153 @@ func (a *TxBroadcasterActor) handleConfirmationObserved(ctx context.Context, return } + a.notifyConfirmed(ctx, entry, msg.blockHeight, msg.numConfs) +} + +// handleConfirmationReorged moves a Confirmed tracked txid back into +// AwaitingConfirmation and fans TxReorged out to all subscribers. The +// confirmation watch stays alive on the chainsource side, so a +// subsequent re-confirmation arrives on the same registration and drives +// another handleConfirmationObserved. +func (a *TxBroadcasterActor) handleConfirmationReorged(ctx context.Context, + msg *confirmationReorgedMsg) { + + entry, ok := a.tracked[msg.txid] + if !ok { + return + } + + state, err := entry.currentTxState() + if err != nil { + a.log.WarnS(ctx, "Failed to read tracked tx state", + err, "txid", entry.data.Txid) + + return + } + + // Only Confirmed entries can be reorged. A reorg ping in any other + // state is benign and dropped: it may be a late notification for an + // entry the actor has already evicted or terminally failed. + if state != TxStateConfirmed { + return + } + + if err := a.advanceTrackedTxFSM( + ctx, entry, &trackedTxReorged{}, + ); err != nil { + + a.log.WarnS(ctx, "Failed to apply reorg to tracked tx", + err, "txid", entry.data.Txid) + + return + } + + a.notifyReorged(ctx, entry) +} + +// handleConfirmationDone moves a Confirmed tracked txid into the terminal +// Finalized state, fans TxFinalized out to all subscribers, releases the +// confirmation watch, and evicts the entry. +func (a *TxBroadcasterActor) handleConfirmationDone(ctx context.Context, + msg *confirmationDoneMsg) { + + entry, ok := a.tracked[msg.txid] + if !ok { + return + } + + state, err := entry.currentTxState() + if err != nil { + a.log.WarnS(ctx, "Failed to read tracked tx state", + err, "txid", entry.data.Txid) + + return + } + + // Only Confirmed entries can be finalized. A Done ping for an entry + // that is not Confirmed has three possible origins: + // + // - Idempotent re-delivery for a tx that is already Finalized: a + // benign no-op. + // + // - Late arrival while the entry is in AwaitingConfirmation + // after a reorg: structurally anomalous for the current + // backends — chainntnfs (in-process lnd) only writes Done + // after the tx has matured past the safety depth, and the + // lndclient adapter never writes the channel at all and + // relies on height-based synthesis (which is gated on + // confirmHeight, reset to 0 on reorg). A future backend that + // DID fire Done during the reorg gap would land here, the + // chainsource sub-actor would already have exited (Done is + // one-shot), and this txid would stop receiving new events. + // The watch is unrecoverable without a fresh RegisterConf + // from this layer. + // + // We log + drop rather than failing the entry: the realistic + // backends do not produce this state, and failing on a hypothetical + // edge case would break callers that rely on the FSM staying live + // for re-confirmation after a reorg. If the log starts firing in + // the wild, the right follow-up is to re-register the conf watch + // from here (see registerConfWatch) rather than relax the guard. + if state != TxStateConfirmed { + a.log.WarnS(ctx, "Dropping confirmation Done for non-Confirmed "+ + "entry; chainsource watch is gone, entry will not "+ + "receive further reorg/finality events", + fmt.Errorf("state %s", state), + "txid", entry.data.Txid) + + return + } + + // Snapshot the confirmation height BEFORE advancing the FSM so we + // can attach it to the outgoing TxFinalized event; the new state + // preserves it but reading from the FSM state directly avoids a + // second state-lookup roundtrip. + fsmState, err := entry.currentFSMState() + if err != nil { + a.log.WarnS(ctx, "Failed to read tracked tx FSM state", + err, "txid", entry.data.Txid) + + return + } + confirmHeight, _ := trackedTxConfirmHeight(fsmState) + + if err := a.advanceTrackedTxFSM( + ctx, entry, &trackedTxFinalized{}, + ); err != nil { + + a.log.WarnS(ctx, "Failed to finalize tracked tx", + err, "txid", entry.data.Txid) + + return + } + + // Only release the conf watch and evict once every subscriber has + // acknowledged the terminal TxFinalized notification. Failed + // deliveries leave the entry in place so retryTerminalNotifications + // can resend on a later actor tick. + if !a.notifyFinalized(ctx, entry, confirmHeight) { + return + } + if err := a.unregisterConfWatch(ctx, entry); err != nil { a.log.WarnS(ctx, "Failed to unregister confirmation watch", err, "txid", entry.data.Txid) } - if a.notifyConfirmed(ctx, entry, msg.blockHeight, msg.numConfs) { - a.evictTerminal(ctx, entry) + a.evictTerminal(ctx, entry) +} + +// isTerminalTxState reports whether a public TxState value represents a +// terminal lifecycle stage that the actor will not advance further on +// its own. +func isTerminalTxState(state TxState) bool { + switch state { + case TxStateFinalized, TxStateFailed: + return true + + default: + return false } } @@ -1003,7 +1273,7 @@ func (a *TxBroadcasterActor) handleBlockObserved(ctx context.Context, continue } - if state == TxStateConfirmed || state == TxStateFailed { + if isTerminalTxState(state) { if a.retryTerminalNotifications(ctx, entry) { a.evictTerminal(ctx, entry) } @@ -1057,13 +1327,13 @@ func (a *TxBroadcasterActor) handleBlockObserved(ctx context.Context, // txid or immediately replays a terminal result. func (a *TxBroadcasterActor) attachExistingSubscriber( ctx context.Context, entry *trackedTx, - subscriber actor.TellOnlyRef[Notification], + subscriber trackedSubscriber, ) *EnsureConfirmedResp { state, err := entry.currentFSMState() if err != nil { a.notifyOneFailed( - ctx, subscriber, entry.data.Txid, + ctx, subscriber.Ref, entry.data.Txid, fmt.Sprintf("tracked tx state: %v", err), ) @@ -1073,27 +1343,57 @@ func (a *TxBroadcasterActor) attachExistingSubscriber( } } + subID := subscriber.Ref.ID() switch state := state.(type) { case *trackedTxStateConfirmed: confirmHeight, _ := trackedTxConfirmHeight(state) - if !a.notifyOneConfirmed( - ctx, subscriber, entry.data.Txid, confirmHeight, - entry.data.TargetConfs, + txConfirmed := &TxConfirmed{ + Txid: entry.data.Txid, + BlockHeight: confirmHeight, + NumConfs: entry.data.TargetConfs, + } + + // Reliable replay of the at-least-once TxConfirmed contract. + // The subscriber is retained in the map regardless of + // delivery outcome so it also receives any later TxReorged + // or TxFinalized on this entry, and on timeout the per-tick + // retryTerminalNotifications path re-attempts delivery via + // the still-true pendingConfirmed flag until it lands. + if a.notifyOneConfirmed( + ctx, subscriber.Ref, entry.data.Txid, txConfirmed, + ) { + + subscriber.pendingConfirmed = false + } + entry.subscribers[subID] = subscriber + + case *trackedTxStateFinalized: + // Finalized is terminal. Deliver TxFinalized reliably and + // retain on timeout so the per-tick retry path can finish + // the handoff. Note that a late-attaching subscriber has + // not yet received TxConfirmed; TxFinalized carries the + // authoritative confirmation height so consumers that need + // it (sweep-finality gates etc.) can recover without an + // out-of-band lookup. + if !a.notifyOneFinalized( + ctx, subscriber.Ref, entry.data.Txid, + state.ConfirmHeight, entry.data.TargetConfs, ) { - entry.subscribers[subscriber.ID()] = subscriber + entry.subscribers[subID] = subscriber } case *trackedTxStateFailed: reason, _ := trackedTxFailureReason(state) - if !a.notifyOneFailed(ctx, subscriber, entry.data.Txid, - reason) { + if !a.notifyOneFailed( + ctx, subscriber.Ref, entry.data.Txid, reason, + ) { - entry.subscribers[subscriber.ID()] = subscriber + entry.subscribers[subID] = subscriber } default: - entry.subscribers[subscriber.ID()] = subscriber + entry.subscribers[subID] = subscriber } return a.ensureResp(entry, false) @@ -1150,8 +1450,11 @@ func (a *TxBroadcasterActor) newTrackedTx(ctx context.Context, return &trackedTx{ data: data, fsm: fsm, - subscribers: map[string]actor.TellOnlyRef[Notification]{ - req.Subscriber.ID(): req.Subscriber, + subscribers: map[string]trackedSubscriber{ + req.Subscriber.ID(): { + Ref: req.Subscriber, + pendingConfirmed: true, + }, }, escalateLog: rate.Sometimes{ First: 1, @@ -1289,6 +1592,9 @@ func (a *TxBroadcasterActor) ensureBlockSubscription( } // registerConfWatch registers a confirmation watch for one tracked txid. +// The watch is registered in reorg-aware mode so the tracked entry can +// observe a confirmation being reorged out and a confirmation maturing +// past the backend's reorg-safety depth. func (a *TxBroadcasterActor) registerConfWatch(ctx context.Context, entry *trackedTx) error { @@ -1303,6 +1609,18 @@ func (a *TxBroadcasterActor) registerConfWatch(ctx context.Context, } }, ) + reorgRef := chainsource.MapConfReorgedEvent( + a.selfRef, + func(event chainsource.ConfReorgedEvent) Msg { + return &confirmationReorgedMsg{txid: event.Txid} + }, + ) + doneRef := chainsource.MapConfDoneEvent( + a.selfRef, + func(event chainsource.ConfDoneEvent) Msg { + return &confirmationDoneMsg{txid: event.Txid} + }, + ) _, err := a.cfg.ChainSource.Ask( ctx, &chainsource.RegisterConfRequest{ @@ -1311,9 +1629,11 @@ func (a *TxBroadcasterActor) registerConfWatch(ctx context.Context, PkScript: append( []byte(nil), entry.data.ConfirmationPkScript..., ), - TargetConfs: entry.data.TargetConfs, - HeightHint: entry.data.HeightHint, - NotifyActor: fn.Some(notifyRef), + TargetConfs: entry.data.TargetConfs, + HeightHint: entry.data.HeightHint, + NotifyActor: fn.Some(notifyRef), + NotifyReorged: fn.Some(reorgRef), + NotifyDone: fn.Some(doneRef), }, ).Await(ctx).Unpack() if err != nil { @@ -1570,11 +1890,18 @@ func (a *TxBroadcasterActor) retryTerminalNotifications(ctx context.Context, switch state := state.(type) { case *trackedTxStateConfirmed: - confirmHeight, _ := trackedTxConfirmHeight(state) + // Retry deferred TxConfirmed deliveries to legacy + // subscribers whose first attempt timed out. Reorg-aware + // subscribers receive TxConfirmed fire-and-forget and have + // no retry tracking; they are skipped here. Returning false + // keeps the entry alive — the caller never evicts on + // Confirmed, only on terminal states. + a.retryConfirmedRedelivery(ctx, entry, state.ConfirmHeight) - return a.notifyConfirmed( - ctx, entry, confirmHeight, entry.data.TargetConfs, - ) + return false + + case *trackedTxStateFinalized: + return a.notifyFinalized(ctx, entry, state.ConfirmHeight) case *trackedTxStateFailed: reason, _ := trackedTxFailureReason(state) @@ -1586,8 +1913,57 @@ func (a *TxBroadcasterActor) retryTerminalNotifications(ctx context.Context, } } -// handleTerminalNotifyResult applies the result of a terminal subscriber -// notification that continued after txconfirm returned to its actor mailbox. +// retryConfirmedRedelivery retries TxConfirmed delivery to every +// subscriber whose initial delivery timed out. The at-least-once +// initial-TxConfirmed contract means a missed delivery must be +// landed before the lifecycle moves on, so this helper runs every +// actor tick from retryTerminalNotifications until every +// pendingConfirmed subscriber has been notified. On successful +// delivery the subscriber's pendingConfirmed flag is cleared and +// the subscriber stays in the map so it continues to receive any +// later TxReorged / TxFinalized on this entry — eviction is +// reserved for the truly terminal Finalized / Failed states. +func (a *TxBroadcasterActor) retryConfirmedRedelivery(ctx context.Context, + entry *trackedTx, confirmHeight int32) { + + for id, subscriber := range entry.subscribers { + if !subscriber.pendingConfirmed { + continue + } + + txConfirmed := &TxConfirmed{ + Txid: entry.data.Txid, + BlockHeight: confirmHeight, + NumConfs: entry.data.TargetConfs, + } + if a.notifyOneConfirmed( + ctx, subscriber.Ref, entry.data.Txid, txConfirmed, + ) { + + subscriber.pendingConfirmed = false + entry.subscribers[id] = subscriber + } + } +} + +// handleTerminalNotifyResult applies the result of a terminal-shape +// subscriber notification that continued after txconfirm returned to its +// actor mailbox. +// +// The kind field distinguishes mid-lifecycle from truly terminal +// deliveries: +// +// - kind == "confirmed": the late-landing delivery is the INITIAL +// TxConfirmed for this subscriber. Clear pendingConfirmed and KEEP +// the subscriber attached to the entry so later TxReorged / +// TxFinalized still reach it. Removing the subscriber here would +// silently drop the entire reorg-aware tail of its lifecycle — +// exactly the kind of failure the unified reliable-delivery path is +// supposed to prevent. +// +// - kind == "finalized" or "failed": the lifecycle is genuinely over +// for this subscriber. Remove it, and evict the entry once every +// subscriber has been notified. func (a *TxBroadcasterActor) handleTerminalNotifyResult(ctx context.Context, msg *terminalNotifyResultMsg) { @@ -1596,7 +1972,8 @@ func (a *TxBroadcasterActor) handleTerminalNotifyResult(ctx context.Context, if msg.err != nil { a.log.WarnS(ctx, "Terminal notification failed after "+ "actor-path timeout", msg.err, "txid", msg.txid, - "subscriber_id", msg.subscriberID) + "subscriber_id", msg.subscriberID, + "notification_kind", msg.kind) return } @@ -1606,6 +1983,40 @@ func (a *TxBroadcasterActor) handleTerminalNotifyResult(ctx context.Context, return } + if msg.kind == "confirmed" { + subscriber, ok := entry.subscribers[msg.subscriberID] + if !ok { + return + } + subscriber.pendingConfirmed = false + entry.subscribers[msg.subscriberID] = subscriber + + // If the tx reorged out of its confirmation while this + // subscriber's initial TxConfirmed was still parked on the + // async notify path, notifyReorged skipped it: a TxReorged must + // never precede the TxConfirmed it reverses. Now that the + // initial delivery has completed and pendingConfirmed is + // cleared, deliver the catch-up TxReorged so the subscriber is + // not left believing a reorged-out tx is confirmed. A tx that + // reorged and then re-confirmed during the window is back in + // Confirmed, so the delivered TxConfirmed is already accurate + // and no catch-up is owed; terminal states own their own + // notifications. + state, err := entry.currentTxState() + if err == nil && state != TxStateConfirmed && + !isTerminalTxState(state) { + + a.notifyReversibleAsync( + ctx, &entry.sealed, subscriber.Ref, + entry.data.Txid, "TxReorged", &TxReorged{ + Txid: entry.data.Txid, + }, + ) + } + + return + } + delete(entry.subscribers, msg.subscriberID) if len(entry.subscribers) != 0 { return @@ -1619,21 +2030,205 @@ func (a *TxBroadcasterActor) handleTerminalNotifyResult(ctx context.Context, return } - if state == TxStateConfirmed || state == TxStateFailed { + if isTerminalTxState(state) { a.evictTerminal(ctx, entry) } } -// notifyConfirmed fans a confirmation result out to all current subscribers. -// It returns true only after every subscriber accepted the terminal -// notification. Failed deliveries are left in the subscriber map so a later -// actor tick can retry instead of permanently losing the confirmation. +// notifyConfirmed fans a TxConfirmed notification out to every current +// subscriber. The initial TxConfirmed (subscriber.pendingConfirmed == +// true) goes through the reliable terminal-delivery path: on success +// pendingConfirmed flips false and the subscriber is retained so it +// keeps receiving the reorg-aware lifecycle (TxReorged / TxFinalized); +// on timeout pendingConfirmed stays true and retryConfirmedRedelivery +// re-attempts on the next actor tick, so the at-least-once initial- +// TxConfirmed guarantee holds even for slow durable subscribers. +// +// Re-confirmations (pendingConfirmed == false, i.e. this subscriber +// has already received the initial TxConfirmed and the chain reorged +// then re-confirmed) are best-effort. The subscriber already knows +// the tx confirmed at least once; the eventual TxFinalized carries +// the authoritative height / numConfs so a missed re-confirmation is +// not load-bearing. +// +// Subscribers are never deleted from the map by this function — that +// happens only on TxFinalized / TxFailed acknowledgment, which is +// when the reorg-aware lifecycle reaches a truly terminal state. func (a *TxBroadcasterActor) notifyConfirmed(ctx context.Context, - entry *trackedTx, blockHeight int32, numConfs uint32) bool { + entry *trackedTx, blockHeight int32, numConfs uint32) { + + for id, subscriber := range entry.subscribers { + txConfirmed := &TxConfirmed{ + Txid: entry.data.Txid, + BlockHeight: blockHeight, + NumConfs: numConfs, + } + + if !subscriber.pendingConfirmed { + a.notifyReversibleAsync( + ctx, &entry.sealed, subscriber.Ref, + entry.data.Txid, "TxConfirmed", txConfirmed, + ) + + continue + } + + if a.notifyOneConfirmed( + ctx, subscriber.Ref, entry.data.Txid, txConfirmed, + ) { + + subscriber.pendingConfirmed = false + entry.subscribers[id] = subscriber + } + } +} + +// notifyOneConfirmed delivers one TxConfirmed notification to a legacy +// (non-opt-in) subscriber via the reliable terminal-delivery path so a +// slow durable subscriber cannot block the actor while still +// preserving the pre-reorg-aware contract of guaranteed-at-least-once +// TxConfirmed delivery. +func (a *TxBroadcasterActor) notifyOneConfirmed(ctx context.Context, + subscriber actor.TellOnlyRef[Notification], txid chainhash.Hash, + notification *TxConfirmed) bool { + + return a.notifyOneTerminal( + ctx, subscriber, txid, "confirmed", + func(notifyCtx context.Context) error { + return subscriber.Tell(notifyCtx, notification) + }, + ) +} + +// notifyReorged fans a TxReorged notification out to every retained +// subscriber whose initial TxConfirmed has already landed. Delivery is +// fire-and-forget: a slow subscriber that misses the reorg is recovered +// by the next lifecycle event (re-TxConfirmed, TxFinalized, or TxFailed) +// since the actor stays attached until one of those terminal events lands. +// +// Subscribers still owed their initial TxConfirmed (pendingConfirmed) are +// skipped: that delivery is deferred to the reliable retry path while a +// reorg's TxReorged goes out fire-and-forget, so notifying them here would +// race the two and could surface TxReorged before — or without — the +// initial TxConfirmed, violating the at-least-once contract and leaving a +// consumer believing a reorged-out tx is confirmed. They observe the live +// state on their eventual initial delivery instead. +func (a *TxBroadcasterActor) notifyReorged(ctx context.Context, + entry *trackedTx) { + + for _, subscriber := range entry.subscribers { + if subscriber.pendingConfirmed { + continue + } + + a.notifyReversibleAsync( + ctx, &entry.sealed, subscriber.Ref, entry.data.Txid, + "TxReorged", + &TxReorged{ + Txid: entry.data.Txid, + }, + ) + } +} + +// notifyReversibleAsync delivers one reversible (non-terminal) +// Notification to a subscriber on a fresh goroutine so txconfirm's +// actor loop does not block on a slow durable subscriber's mailbox. +// Failures are logged but not retried: the next event in the tracked +// tx's lifecycle (re-TxConfirmed after a TxReorged, TxFinalized, or +// eventual TxFailed) carries the live state and supersedes any +// dropped reversible delivery. +// +// The delivery context is detached from the actor transaction (so the +// txconfirm goroutine's DB tx is not held open behind a subscriber +// roundtrip) and from ctx cancellation (so a per-message context +// expiring mid-flight does not race the actor releasing its mailbox). +// Each goroutine is bounded by reversibleNotifyTimeout so a stuck +// subscriber does not leak unbounded goroutines on every block. +func (a *TxBroadcasterActor) notifyReversibleAsync(ctx context.Context, + sealed *atomic.Bool, subscriber actor.TellOnlyRef[Notification], + txid chainhash.Hash, kind string, notification Notification) { + + notifyCtx := actor.WithoutTx(context.WithoutCancel(ctx)) + notifyCtx, cancel := context.WithTimeout( + notifyCtx, reversibleNotifyTimeout, + ) + + subscriberID := subscriber.ID() + go func() { + defer cancel() + + // Drop the reversible delivery if the entry has sealed (a + // terminal TxFinalized/TxFailed has begun). Checked here, just + // before the Tell, so a reversible spawned before finality + // cannot trail the terminal notification into the subscriber's + // mailbox and resurrect bookkeeping it already released. + if sealed != nil && sealed.Load() { + return + } + + if err := subscriber.Tell(notifyCtx, notification); err != nil { + a.log.WarnS(notifyCtx, + "Failed to deliver reversible notification", + err, "txid", txid, + "subscriber_id", subscriberID, + "notification_kind", kind) + } + }() +} + +// notifyFinalized fans a TxFinalized notification out to every +// retained subscriber. The map may still contain subscribers whose +// initial reliable TxConfirmed delivery timed out +// (pendingConfirmed=true) — those still need TxConfirmed before they +// see TxFinalized, otherwise the at-least-once TxConfirmed contract +// is violated. For each pending subscriber we make one TxConfirmed +// attempt, flip pendingConfirmed on success, and either way leave +// them attached for the next finalize retry (the per-tick retry path +// will keep cycling through this function until both deliveries +// land). +// +// On successful TxFinalized delivery the subscriber is removed and +// the caller may evict the tracked entry once every subscriber has +// acknowledged. Failed deliveries are left in the subscriber map for +// retry from the next actor tick. +// +// confirmHeight is the height observed at finalization time, replayed +// onto TxFinalized so consumers that dropped the fire-and-forget +// re-TxConfirmed can recover the authoritative confirmation height +// without an out-of-band lookup. +func (a *TxBroadcasterActor) notifyFinalized(ctx context.Context, + entry *trackedTx, confirmHeight int32) bool { + + // Seal the entry so any in-flight reversible delivery skips its Tell + // rather than trailing this terminal notification into a subscriber's + // mailbox. Idempotent across the per-tick finalize retries. + entry.sealed.Store(true) for id, subscriber := range entry.subscribers { - ok := a.notifyOneConfirmed( - ctx, subscriber, entry.data.Txid, blockHeight, numConfs, + if subscriber.pendingConfirmed { + if a.notifyOneConfirmed( + ctx, subscriber.Ref, entry.data.Txid, + &TxConfirmed{ + Txid: entry.data.Txid, + BlockHeight: confirmHeight, + NumConfs: entry.data.TargetConfs, + }, + ) { + + subscriber.pendingConfirmed = false + entry.subscribers[id] = subscriber + } else { + // Hold until the next tick — we must not + // deliver TxFinalized before the documented + // initial TxConfirmed has landed. + continue + } + } + + ok := a.notifyOneFinalized( + ctx, subscriber.Ref, entry.data.Txid, confirmHeight, + entry.data.TargetConfs, ) if !ok { continue @@ -1652,9 +2247,14 @@ func (a *TxBroadcasterActor) notifyConfirmed(ctx context.Context, func (a *TxBroadcasterActor) notifyFailed(ctx context.Context, entry *trackedTx, reason string) bool { + // Seal the entry so any in-flight reversible delivery skips its Tell + // rather than trailing this terminal failure into a subscriber's + // mailbox. Idempotent across the per-tick retries. + entry.sealed.Store(true) + for id, subscriber := range entry.subscribers { ok := a.notifyOneFailed( - ctx, subscriber, entry.data.Txid, reason, + ctx, subscriber.Ref, entry.data.Txid, reason, ) if !ok { continue @@ -1666,15 +2266,20 @@ func (a *TxBroadcasterActor) notifyFailed(ctx context.Context, entry *trackedTx, return len(entry.subscribers) == 0 } -// notifyOneConfirmed delivers one confirmation notification. -func (a *TxBroadcasterActor) notifyOneConfirmed(ctx context.Context, +// notifyOneFinalized delivers one TxFinalized notification through the +// terminal-delivery path so a slow subscriber cannot block the actor +// loop. blockHeight and numConfs are replayed onto the notification +// from the last observed confirmation so opt-in subscribers that +// dropped the (fire-and-forget) TxConfirmed event can still recover +// the authoritative confirmation height. +func (a *TxBroadcasterActor) notifyOneFinalized(ctx context.Context, subscriber actor.TellOnlyRef[Notification], txid chainhash.Hash, blockHeight int32, numConfs uint32) bool { return a.notifyOneTerminal( - ctx, subscriber, txid, "confirmed", + ctx, subscriber, txid, "finalized", func(notifyCtx context.Context) error { - return subscriber.Tell(notifyCtx, &TxConfirmed{ + return subscriber.Tell(notifyCtx, &TxFinalized{ Txid: txid, BlockHeight: blockHeight, NumConfs: numConfs, @@ -1735,7 +2340,7 @@ func (a *TxBroadcasterActor) notifyOneTerminal(ctx context.Context, a.terminalNotifyInflight[inflightKey] = struct{}{} //nolint:contextcheck // async result outlives ctx a.completeTerminalNotifyAsync( - inflightKey, txid, subscriberID, errChan, cancel, + inflightKey, txid, subscriberID, kind, errChan, cancel, ) a.log.DebugS(ctx, "Terminal tx notification deferred", @@ -1748,10 +2353,13 @@ func (a *TxBroadcasterActor) notifyOneTerminal(ctx context.Context, } } -// completeTerminalNotifyAsync reports a timed-out terminal delivery back to the -// txconfirm actor once the underlying Tell returns. +// completeTerminalNotifyAsync reports a timed-out terminal-shape delivery +// back to the txconfirm actor once the underlying Tell returns. kind +// records the notification variant ("confirmed" / "finalized" / "failed") +// so the actor-side handler can apply the correct post-delivery state +// transition. func (a *TxBroadcasterActor) completeTerminalNotifyAsync(inflightKey string, - txid chainhash.Hash, subscriberID string, errChan <-chan error, + txid chainhash.Hash, subscriberID, kind string, errChan <-chan error, cancel context.CancelFunc) { if a.selfRef == nil { @@ -1768,6 +2376,7 @@ func (a *TxBroadcasterActor) completeTerminalNotifyAsync(inflightKey string, txid: txid, subscriberID: subscriberID, inflightKey: inflightKey, + kind: kind, err: err, } bgCtx := context.Background() diff --git a/txconfirm/actor_test.go b/txconfirm/actor_test.go index 688dcb64f..eebe1d0b4 100644 --- a/txconfirm/actor_test.go +++ b/txconfirm/actor_test.go @@ -29,6 +29,11 @@ import ( // testTimeout is the default timeout used by txconfirm actor tests. const testTimeout = time.Second +// confReorgedRef / confDoneRef are the reorg / done notification ref +// types used in the fake chain-source ref. +type confReorgedRef = actor.TellOnlyRef[chainsource.ConfReorgedEvent] +type confDoneRef = actor.TellOnlyRef[chainsource.ConfDoneEvent] + // confNotifyRef is the confirmation-event notification ref type used in // the fake chainsource test double. type confNotifyRef = actor.TellOnlyRef[chainsource.ConfirmationEvent] @@ -54,6 +59,8 @@ type fakeChainSourceRef struct { blockNotify actor.TellOnlyRef[chainsource.BlockEpoch] confNotify map[chainhash.Hash]confNotifyRef + confReorged map[chainhash.Hash]confReorgedRef + confDone map[chainhash.Hash]confDoneRef confConfs map[chainhash.Hash]uint32 alreadyConfirmed map[chainhash.Hash]chainsource.ConfirmationEvent @@ -148,6 +155,93 @@ func (b *blockingNotifyRef) attemptsCount() int { return b.attempts } +// deferringNotifyRef blocks Tell until released, then completes +// successfully (returns nil regardless of ctx state). Used to drive the +// async-completion path of notifyOneTerminal: Tell exceeds the actor-path +// budget, so the caller observes a timeout and parks the result on the +// completeTerminalNotifyAsync goroutine; the test then releases the Tell +// to simulate the underlying mailbox eventually accepting the +// notification, producing a terminalNotifyResultMsg{err: nil} that the +// actor mailbox processes via handleTerminalNotifyResult. +type deferringNotifyRef struct { + id string + + started chan struct{} + release chan struct{} + once sync.Once + + mu sync.Mutex + attempts int + msgs []Notification +} + +// newDeferringNotifyRef creates a subscriber that blocks on the first Tell +// until release is closed. +func newDeferringNotifyRef(id string) *deferringNotifyRef { + return &deferringNotifyRef{ + id: id, + started: make(chan struct{}), + release: make(chan struct{}), + } +} + +// ID returns the fake subscriber ID. +func (d *deferringNotifyRef) ID() string { + return d.id +} + +// Tell blocks until release is closed, records the notification, and +// returns nil. ctx cancellation is intentionally ignored so the test can +// drive the "Tell eventually succeeded" path even after the actor-side +// notifyCtx timed out. +func (d *deferringNotifyRef) Tell(_ context.Context, n Notification) error { + d.mu.Lock() + d.attempts++ + d.mu.Unlock() + + d.once.Do(func() { + close(d.started) + }) + + <-d.release + + d.mu.Lock() + d.msgs = append(d.msgs, n) + d.mu.Unlock() + + return nil +} + +// waitStarted blocks until the first Tell has begun, so the test can +// release after the actor-side timeout has fired. +func (d *deferringNotifyRef) waitStarted(t *testing.T) { + t.Helper() + + select { + case <-d.started: + case <-time.After(testTimeout): + t.Fatal("deferringNotifyRef Tell never started") + } +} + +// releaseTell unblocks every parked Tell call so the deferred deliveries +// complete and the test observes the async-completion path. +func (d *deferringNotifyRef) releaseTell() { + close(d.release) +} + +// snapshotMessages returns a defensive copy of every notification that +// has landed in the subscriber's mailbox so far. +func (d *deferringNotifyRef) snapshotMessages() []Notification { + d.mu.Lock() + defer d.mu.Unlock() + + out := make([]Notification, len(d.msgs)) + copy(out, d.msgs) + + return out +} + // ID returns the fake subscriber ID. func (r *contextInspectNotifyRef) ID() string { return r.id @@ -232,10 +326,12 @@ func (r *retryNotifyRef) awaitMessage(timeout time.Duration) (Notification, // newFakeChainSourceRef creates a new controllable chainsource test double. func newFakeChainSourceRef(bestHeight int32) *fakeChainSourceRef { return &fakeChainSourceRef{ - bestHeight: bestHeight, - feeRate: 5, - confNotify: make(map[chainhash.Hash]confNotifyRef), - confConfs: make(map[chainhash.Hash]uint32), + bestHeight: bestHeight, + feeRate: 5, + confNotify: make(map[chainhash.Hash]confNotifyRef), + confReorged: make(map[chainhash.Hash]confReorgedRef), + confDone: make(map[chainhash.Hash]confDoneRef), + confConfs: make(map[chainhash.Hash]uint32), alreadyConfirmed: make( map[chainhash.Hash]chainsource.ConfirmationEvent, ), @@ -344,6 +440,12 @@ func (f *fakeChainSourceRef) handleAsk(_ context.Context, if req.Txid != nil && req.NotifyActor.IsSome() { f.confNotify[*req.Txid] = req.NotifyActor.UnwrapOr(nil) f.confConfs[*req.Txid] = req.TargetConfs + req.NotifyReorged.WhenSome(func(r confReorgedRef) { + f.confReorged[*req.Txid] = r + }) + req.NotifyDone.WhenSome(func(r confDoneRef) { + f.confDone[*req.Txid] = r + }) if event, ok := f.alreadyConfirmed[*req.Txid]; ok { notifyRef := req.NotifyActor.UnwrapOr(nil) //nolint:contextcheck // fake backend @@ -358,6 +460,8 @@ func (f *fakeChainSourceRef) handleAsk(_ context.Context, if req.Txid != nil { delete(f.confNotify, *req.Txid) delete(f.confConfs, *req.Txid) + delete(f.confReorged, *req.Txid) + delete(f.confDone, *req.Txid) } return &chainsource.UnregisterConfResponse{}, nil @@ -417,6 +521,34 @@ func (f *fakeChainSourceRef) emitConfirmation(t *testing.T, txid chainhash.Hash, require.NoError(t, err) } +// emitConfReorged delivers a reorg event for one tracked txid. +func (f *fakeChainSourceRef) emitConfReorged(t *testing.T, + txid chainhash.Hash) { + + t.Helper() + + f.mu.Lock() + ref := f.confReorged[txid] + f.mu.Unlock() + + require.NotNil(t, ref) + err := ref.Tell(t.Context(), chainsource.ConfReorgedEvent{Txid: txid}) + require.NoError(t, err) +} + +// emitConfDone delivers a finality event for one tracked txid. +func (f *fakeChainSourceRef) emitConfDone(t *testing.T, txid chainhash.Hash) { + t.Helper() + + f.mu.Lock() + ref := f.confDone[txid] + f.mu.Unlock() + + require.NotNil(t, ref) + err := ref.Tell(t.Context(), chainsource.ConfDoneEvent{Txid: txid}) + require.NoError(t, err) +} + // emitBlock delivers a new block epoch to the shared block subscriber. func (f *fakeChainSourceRef) emitBlock(t *testing.T, height int32) { t.Helper() @@ -815,15 +947,27 @@ func TestEnsureConfirmedDedupesTwoSubscribers(t *testing.T) { require.IsType(t, &TxConfirmed{}, confirmedA) require.IsType(t, &TxConfirmed{}, confirmedB) + + // TxConfirmed alone does not release the chainsource conf watch in + // the reorg-aware model: the watch stays alive until the backend + // finalizes the confirmation. Driving Done evicts the tracked + // entry and unregisters. + chain.emitConfDone(t, tx.TxHash()) + mustAwaitNotification(t, subA) + mustAwaitNotification(t, subB) mustEventually(t, func() bool { return chain.unregisterConfCount() == 1 }) } -// TestConfirmationDeliveryRetriesAfterTellFailure verifies that a transient -// subscriber delivery failure does not permanently drop a terminal -// confirmation notification. -func TestConfirmationDeliveryRetriesAfterTellFailure(t *testing.T) { +// TestLifecycleDeliveryRetriesAfterTellFailure verifies that a transient +// subscriber delivery failure does not permanently drop a lifecycle +// notification. Both the initial TxConfirmed and the terminal +// TxFinalized are reliable: if the first Tell attempt fails, the +// per-tick retry path keeps re-attempting until delivery lands, and +// the entry only evicts after every retained subscriber has +// acknowledged both events. +func TestLifecycleDeliveryRetriesAfterTellFailure(t *testing.T) { chain := newFakeChainSourceRef(100) ref, _ := newTestActor(t, Config{ ChainSource: chain, @@ -831,6 +975,9 @@ func TestConfirmationDeliveryRetriesAfterTellFailure(t *testing.T) { tx := makeTestTx(false) txid := tx.TxHash() + // Fail the first Tell attempt: TxConfirmed delivery has to be + // retried before pendingConfirmed clears, after which TxFinalized + // can be attempted. sub := newRetryNotifyRef("sub-retry", 1) resp := mustEnsure(t, ref.Ref(), &EnsureConfirmedReq{ @@ -844,31 +991,40 @@ func TestConfirmationDeliveryRetriesAfterTellFailure(t *testing.T) { return sub.attemptsCount() == 1 }) + // First TxConfirmed attempt failed; the entry stays alive with + // pendingConfirmed=true and no notification has reached the + // subscriber mailbox yet. msg, ok := sub.awaitMessage(100 * time.Millisecond) - require.False(t, ok, "unexpected notification: %v", msg) - mustEventually(t, func() bool { - return chain.unregisterConfCount() == 1 - }) - - chain.emitBlock(t, 102) - msg, ok = sub.awaitMessage(testTimeout) - require.True(t, ok, "expected retried notification") - - confirmed, ok := msg.(*TxConfirmed) - require.True(t, ok) + require.False(t, ok, "unexpected early notification: %v", msg) + require.Equal(t, 0, chain.unregisterConfCount()) + + // Finalization fires. notifyFinalized retries TxConfirmed first + // (the at-least-once initial-confirmation contract must land + // before TxFinalized is allowed through). The second Tell + // succeeds, so TxConfirmed lands and TxFinalized follows in the + // same notifyFinalized pass. + chain.emitConfDone(t, txid) + + first, ok := sub.awaitMessage(testTimeout) + require.True(t, ok, "expected retried TxConfirmed") + confirmed, ok := first.(*TxConfirmed) + require.True( + t, ok, "first notification must be TxConfirmed, got %T", first, + ) require.Equal(t, txid, confirmed.Txid) - require.Equal(t, int32(101), confirmed.BlockHeight) - require.Equal(t, uint32(1), confirmed.NumConfs) - require.Equal(t, 2, sub.attemptsCount()) - freshSub := actor.NewChannelTellOnlyRef[Notification]("sub-fresh", 4) - replayResp := mustEnsure(t, ref.Ref(), &EnsureConfirmedReq{ - Tx: tx, - Subscriber: freshSub, + second, ok := sub.awaitMessage(testTimeout) + require.True(t, ok, "expected TxFinalized") + finalized, ok := second.(*TxFinalized) + require.True( + t, ok, "second notification must be TxFinalized, got %T", + second, + ) + require.Equal(t, txid, finalized.Txid) + + mustEventually(t, func() bool { + return chain.unregisterConfCount() == 1 }) - require.True(t, replayResp.Created) - require.Equal(t, 2, chain.registerConfCount()) - require.Equal(t, 2, chain.broadcastCallCount()) } // TestTerminalNotificationsDoNotInheritCallerContext verifies that terminal @@ -884,17 +1040,17 @@ func TestTerminalNotificationsDoNotInheritCallerContext(t *testing.T) { cancel() txid := chainhash.Hash{1} - confirmedSub := &contextInspectNotifyRef{id: "confirmed-sub"} - ok := behavior.notifyOneConfirmed( - ctx, confirmedSub, txid, 101, 1, + finalizedSub := &contextInspectNotifyRef{id: "finalized-sub"} + ok := behavior.notifyOneFinalized( + ctx, finalizedSub, txid, 101, 1, ) require.True(t, ok) - hasTx, ctxErr, msgs := confirmedSub.snapshot() + hasTx, ctxErr, msgs := finalizedSub.snapshot() require.False(t, hasTx) require.NoError(t, ctxErr) require.Len(t, msgs, 1) - require.IsType(t, &TxConfirmed{}, msgs[0]) + require.IsType(t, &TxFinalized{}, msgs[0]) failedSub := &contextInspectNotifyRef{id: "failed-sub"} ok = behavior.notifyOneFailed(ctx, failedSub, txid, "boom") @@ -937,14 +1093,14 @@ func TestTerminalNotificationTimeoutDoesNotBlockActor(t *testing.T) { }() start := time.Now() - ok := behavior.notifyOneConfirmed( + ok := behavior.notifyOneFinalized( context.Background(), sub, txid, 101, 1, ) require.False(t, ok) require.Less(t, time.Since(start), testTimeout) require.True(t, <-started) - key := terminalNotifyKey(txid, sub.ID(), "confirmed") + key := terminalNotifyKey(txid, sub.ID(), "finalized") _, inflight := behavior.terminalNotifyInflight[key] require.True(t, inflight) require.Equal(t, 1, sub.attemptsCount()) @@ -1051,6 +1207,9 @@ func TestEnsureConfirmedAlreadyConfirmedUsesSuccessPath(t *testing.T) { subA := actor.NewChannelTellOnlyRef[Notification]("sub-a", 4) subB := actor.NewChannelTellOnlyRef[Notification]("sub-b", 4) + // Opt in to reorg-aware notifications so the tracked entry stays + // alive past TxConfirmed and the second EnsureConfirmedReq can + // attach to the existing tracking state. resp := mustEnsure(t, ref.Ref(), &EnsureConfirmedReq{ Tx: tx, Subscriber: subA, @@ -1062,21 +1221,21 @@ func TestEnsureConfirmedAlreadyConfirmedUsesSuccessPath(t *testing.T) { require.True(t, ok) require.Equal(t, int32(99), confirmed.BlockHeight) - // Once subA's TxConfirmed has been delivered, terminal eviction drops - // the tracked entry. A subsequent EnsureConfirmedReq for the same - // txid therefore starts fresh tracking rather than replaying cached - // state. Chainsource immediately re-fires the confirmation for the - // already-confirmed tx, so subB still receives TxConfirmed. + // TxConfirmed is no longer terminal: the tracked entry remains alive + // (still subscribed to chainsource reorg/done) until a Done event + // fires. A subsequent EnsureConfirmedReq therefore attaches to the + // existing tracking state and replays TxConfirmed without + // re-broadcasting or re-registering with chainsource. replayResp := mustEnsure(t, ref.Ref(), &EnsureConfirmedReq{ Tx: tx, Subscriber: subB, }) - require.True(t, replayResp.Created) + require.False(t, replayResp.Created) replayed := mustAwaitNotification(t, subB) require.IsType(t, &TxConfirmed{}, replayed) - require.Equal(t, 2, chain.broadcastCallCount()) - require.Equal(t, 2, chain.registerConfCount()) + require.Equal(t, 1, chain.broadcastCallCount()) + require.Equal(t, 1, chain.registerConfCount()) } // TestEnsureConfirmedBroadcastFailureNotifiesFailure verifies that terminal @@ -1462,6 +1621,11 @@ func TestUnregisterConfMatchesRegisterServiceKey(t *testing.T) { confirmed := mustAwaitNotification(t, sub) require.IsType(t, &TxConfirmed{}, confirmed) + // The conf watch is only released once chainsource fires Done. + chain.emitConfDone(t, tx.TxHash()) + finalized := mustAwaitNotification(t, sub) + require.IsType(t, &TxFinalized{}, finalized) + mustEventually(t, func() bool { return chain.unregisterConfCount() == 1 }) @@ -1531,7 +1695,17 @@ func TestTerminalEntriesEvictedAfterConfirmation(t *testing.T) { require.IsType(t, &TxConfirmed{}, confirmed) } - // Every confirmation should have produced exactly one unregister. + // Finalization drives terminal eviction: TxConfirmed alone is now + // reversible and the tracked entry stays alive until the backend + // fires Done. + for i := 0; i < numTxs; i++ { + chain.emitConfDone(t, txids[i]) + + finalized := mustAwaitNotification(t, subs[i]) + require.IsType(t, &TxFinalized{}, finalized) + } + + // Every finalized entry should have produced exactly one unregister. mustEventually(t, func() bool { return chain.unregisterConfCount() == numTxs }) @@ -1907,3 +2081,81 @@ func TestEnsureConfirmedFailsPermanentBroadcastError(t *testing.T) { failed := mustAwaitNotification(t, sub) require.IsType(t, &TxFailed{}, failed) } + +// TestInitialConfirmedAsyncDeliveryRetainsSubscriber regression-tests an +// initial TxConfirmed delivery whose Tell exceeds terminalNotifyTimeout +// before landing successfully via the async-completion goroutine. The +// subscriber must remain attached to the entry afterwards so the +// reorg-aware tail of its lifecycle (TxReorged / TxFinalized) still +// reaches it; the previous handleTerminalNotifyResult deleted the +// subscriber on any successful async terminal completion, silently +// dropping every subsequent reorg event for slow mailboxes. +func TestInitialConfirmedAsyncDeliveryRetainsSubscriber(t *testing.T) { + // Shorten the actor-path budget so the test reliably exercises the + // async-completion code path within a normal test runtime. + oldTimeout := terminalNotifyTimeout + terminalNotifyTimeout = 20 * time.Millisecond + t.Cleanup(func() { + terminalNotifyTimeout = oldTimeout + }) + + chain := newFakeChainSourceRef(100) + ref, _ := newTestActor(t, Config{ + ChainSource: chain, + }) + + tx := makeTestTx(false) + txid := tx.TxHash() + sub := newDeferringNotifyRef("sub-async-confirmed") + + resp := mustEnsure(t, ref.Ref(), &EnsureConfirmedReq{ + Tx: tx, + Subscriber: sub, + }) + require.True(t, resp.Created) + + // Fire the confirmation. notifyOneConfirmed -> notifyOneTerminal + // will park on sub.Tell, blow through terminalNotifyTimeout, and + // hand the in-flight Tell off to completeTerminalNotifyAsync. + chain.emitConfirmation(t, txid, 101) + sub.waitStarted(t) + + // Wait long enough that the actor-side notifyCtx is guaranteed to + // have timed out and the async-completion goroutine is in flight. + // Without this delay the Tell could land synchronously and the + // async path the regression targets would not be exercised. + time.Sleep(10 * terminalNotifyTimeout) + + // Release the parked Tell so the async goroutine reports back to + // the txconfirm actor with a successful terminalNotifyResultMsg. + sub.releaseTell() + + // Wait for TxConfirmed to actually land in the subscriber mailbox. + require.Eventually(t, func() bool { + for _, m := range sub.snapshotMessages() { + if _, ok := m.(*TxConfirmed); ok { + return true + } + } + + return false + }, testTimeout, 10*time.Millisecond, "TxConfirmed never delivered") + + // Drive a reorg. If the subscriber was wrongly evicted on the + // async-confirmed completion, notifyReorged has nobody to fan to + // and the TxReorged below never arrives. + chain.emitConfReorged(t, txid) + + require.Eventually(t, func() bool { + for _, m := range sub.snapshotMessages() { + if _, ok := m.(*TxReorged); ok { + return true + } + } + + return false + }, testTimeout, 10*time.Millisecond, + "TxReorged never reached subscriber whose initial TxConfirmed "+ + "completed via the async path — handleTerminalNotify"+ + "Result probably evicted it on completion") +} diff --git a/txconfirm/broadcaster_test.go b/txconfirm/broadcaster_test.go index cbf638e3e..716348281 100644 --- a/txconfirm/broadcaster_test.go +++ b/txconfirm/broadcaster_test.go @@ -361,7 +361,7 @@ func newTrackedTxForState(t *testing.T, state trackedTxState) *trackedTx { return &trackedTx{ data: data, fsm: &fsm, - subscribers: make(map[string]actor.TellOnlyRef[Notification]), + subscribers: make(map[string]trackedSubscriber), } } @@ -1619,11 +1619,13 @@ func TestActorValidationAndCleanup(t *testing.T) { }, } entry := newTrackedTxForState(t, awaitConf) - entry.subscribers["fail"] = &failingNotifyRef{} + entry.subscribers["fail"] = trackedSubscriber{ + Ref: &failingNotifyRef{}, + } behavior.tracked[entry.data.Txid] = entry - behavior.notifyOneConfirmed( - t.Context(), &failingNotifyRef{}, entry.data.Txid, 1, 1, + behavior.notifyOneFinalized( + t.Context(), &failingNotifyRef{}, entry.data.Txid, 0, 0, ) behavior.notifyOneFailed( t.Context(), &failingNotifyRef{}, entry.data.Txid, diff --git a/txconfirm/fsm_types.go b/txconfirm/fsm_types.go index 387962b43..3cde4011f 100644 --- a/txconfirm/fsm_types.go +++ b/txconfirm/fsm_types.go @@ -168,6 +168,21 @@ type trackedTxFailed struct { // trackedTxEventSealed marks trackedTxFailed as a tracked-tx event. func (e *trackedTxFailed) trackedTxEventSealed() {} +// trackedTxReorged records that a previously delivered confirmation was +// reorged out of the canonical chain. Only valid from +// trackedTxStateConfirmed. +type trackedTxReorged struct{} + +// trackedTxEventSealed marks trackedTxReorged as a tracked-tx event. +func (e *trackedTxReorged) trackedTxEventSealed() {} + +// trackedTxFinalized records that a confirmation is past the backend's +// reorg-safety depth. Only valid from trackedTxStateConfirmed. +type trackedTxFinalized struct{} + +// trackedTxEventSealed marks trackedTxFinalized as a tracked-tx event. +func (e *trackedTxFinalized) trackedTxEventSealed() {} + // trackedTxErrorReporter reports tracked-tx FSM errors through the package // logger. type trackedTxErrorReporter struct { @@ -225,6 +240,9 @@ func txStateFromTrackedState(state trackedTxState) TxState { case *trackedTxStateConfirmed: return TxStateConfirmed + case *trackedTxStateFinalized: + return TxStateFinalized + case *trackedTxStateFailed: return TxStateFailed @@ -279,6 +297,9 @@ func trackedTxLastBroadcastHeight(state trackedTxState) fn.Option[int32] { case *trackedTxStateConfirmed: return s.LastBroadcastHeight + case *trackedTxStateFinalized: + return s.LastBroadcastHeight + case *trackedTxStateFailed: return s.LastBroadcastHeight @@ -302,12 +323,16 @@ func trackedTxBroadcastFailures(state trackedTxState) int { // trackedTxConfirmHeight returns the state's confirmation height if the // transaction has already confirmed. func trackedTxConfirmHeight(state trackedTxState) (int32, bool) { - confirmed, ok := state.(*trackedTxStateConfirmed) - if !ok { + switch s := state.(type) { + case *trackedTxStateConfirmed: + return s.ConfirmHeight, true + + case *trackedTxStateFinalized: + return s.ConfirmHeight, true + + default: return 0, false } - - return confirmed.ConfirmHeight, true } // trackedTxFailureReason returns the state's terminal failure reason when diff --git a/txconfirm/funded_anchor_test.go b/txconfirm/funded_anchor_test.go index 53c07bb46..f0f51e992 100644 --- a/txconfirm/funded_anchor_test.go +++ b/txconfirm/funded_anchor_test.go @@ -575,9 +575,12 @@ func TestBumpNowParentFeeSufficient(t *testing.T) { } // TestBumpNowUntrackedAndTerminal covers the remaining no-op guards: an -// untracked txid, and a transaction that confirmed and was evicted from -// tracking (a confirmed entry whose terminal notification is delivered is -// dropped from the map, so a late bump lands in the untracked branch). +// untracked txid, and a transaction that has reached its confirmation target. +// Under the reorg-aware lifecycle a confirmed entry is NOT evicted at its +// confirmation — it stays tracked in the reversible Confirmed state until the +// backend signals finality (Done), so it can observe a later reorg — so a bump +// of a confirmed-but-not-final tx lands in the "already confirmed" no-op +// branch rather than the untracked branch. func TestBumpNowUntrackedAndTerminal(t *testing.T) { t.Parallel() @@ -608,9 +611,10 @@ func TestBumpNowUntrackedAndTerminal(t *testing.T) { Subscriber: sub, }) - // Drain the confirmation callback the fake chain delivered to the - // self ref so the entry reaches its terminal state and, with its - // notification delivered, is evicted from tracking. + // Drain the confirmation callback the fake chain delivered to the self + // ref so the entry advances into the reversible Confirmed state and its + // TxConfirmed notification is delivered. It is NOT evicted: the entry + // stays tracked until finality so a reorg can still be observed. selfRef, ok := behavior.selfRef.(*actor.ChannelTellOnlyRef[Msg]) require.True(t, ok) for { @@ -622,11 +626,13 @@ func TestBumpNowUntrackedAndTerminal(t *testing.T) { } require.NotNil(t, mustAwaitNotification(t, sub)) - // A bump after eviction reports the untracked no-op: there is no - // entry left to bump, which is the correct answer for a confirmed tx. + // A bump of the confirmed (but not-yet-final) entry is the confirmed + // no-op: it is still tracked, so the handler reports "already + // confirmed" rather than the untracked branch. Either way a confirmed + // tx cannot be fee-bumped. bumpResp = mustReceiveBump(t, behavior, &BumpNowReq{ Txid: tx.TxHash(), }) require.False(t, bumpResp.Bumped) - require.Contains(t, bumpResp.Reason, "not tracked") + require.Contains(t, bumpResp.Reason, "already confirmed") } diff --git a/txconfirm/messages.go b/txconfirm/messages.go index 2208788f2..e595e6401 100644 --- a/txconfirm/messages.go +++ b/txconfirm/messages.go @@ -58,9 +58,15 @@ const ( TxStateFeeBumping // TxStateConfirmed indicates the tracked transaction reached its target - // confirmation count. + // confirmation count on the canonical chain. This state is reversible: + // a reorg moves the tracked entry back to TxStateAwaitingConfirmation, + // and finality moves it to TxStateFinalized. TxStateConfirmed + // TxStateFinalized indicates the tracked transaction confirmed and is + // past the backend's reorg-safety depth. Terminal. + TxStateFinalized + // TxStateFailed indicates the actor encountered a terminal // failure while // trying to confirm the transaction. @@ -85,6 +91,9 @@ func (s TxState) String() string { case TxStateConfirmed: return "confirmed" + case TxStateFinalized: + return "finalized" + case TxStateFailed: return "failed" @@ -126,8 +135,17 @@ type EnsureConfirmedReq struct { // parents and for callers that do not fee-bump. ParentFee btcutil.Amount - // Subscriber receives TxConfirmed or TxFailed notifications for this - // request. + // Subscriber receives the full reorg-aware notification lifecycle + // for this request: TxConfirmed when the tx reaches the + // confirmation target, TxReorged if a previously delivered + // TxConfirmed is reorged out, re-TxConfirmed on the new canonical + // chain, TxFinalized once the confirmation is past the backend's + // reorg-safety depth, and TxFailed on terminal failure. + // TxConfirmed and TxFailed deliveries are reliable (retry on + // timeout from the per-tick retry path); TxReorged is best-effort + // because the next lifecycle event (re-TxConfirmed / TxFinalized / + // TxFailed) re-establishes state. TxFinalized is reliable so the + // caller can free reorg-recovery bookkeeping deterministically. Subscriber actor.TellOnlyRef[Notification] } @@ -306,6 +324,65 @@ func (m *TxConfirmed) MessageType() string { // set. func (m *TxConfirmed) txConfirmNotificationSealed() {} +// TxReorged notifies a subscriber that a previously delivered TxConfirmed +// has been reorged out of the canonical chain. After receiving this event +// a consumer should consider the prior confirmation no longer valid; if +// the transaction re-confirms on the new canonical chain a fresh +// TxConfirmed will follow on the same subscription. +type TxReorged struct { + actor.BaseMessage + + // Txid identifies the reorged transaction. + Txid chainhash.Hash +} + +// MessageType returns the stable message type identifier. +func (m *TxReorged) MessageType() string { + return "TxReorged" +} + +// txConfirmNotificationSealed seals TxReorged into the package notification +// set. +func (m *TxReorged) txConfirmNotificationSealed() {} + +// TxFinalized notifies a subscriber that the tracked transaction is past +// the backend's reorg-safety depth and will receive no further events. +// Subscribers may use this signal to drop any reorg-recovery bookkeeping +// they were holding for the registration. Not all backends synthesize +// this event (the lndclient transport does not), so consumers must treat +// its absence as a normal operating condition rather than an error. +// +// BlockHeight and NumConfs replay the authoritative confirmation +// numbers carried by the last TxConfirmed before finalization. Because +// reversible TxConfirmed deliveries are fire-and-forget for opt-in +// subscribers and may be dropped on a momentarily-full mailbox, the +// finalization event must carry enough information for height-dependent +// consumers to recover without out-of-band lookups. +type TxFinalized struct { + actor.BaseMessage + + // Txid identifies the finalized transaction. + Txid chainhash.Hash + + // BlockHeight is the height of the block carrying the latest + // observed confirmation (post-any-reorg) at finalization time. + BlockHeight int32 + + // NumConfs is the confirmation count that triggered the + // finalization event — typically the EnsureConfirmedReq's + // TargetConfs. + NumConfs uint32 +} + +// MessageType returns the stable message type identifier. +func (m *TxFinalized) MessageType() string { + return "TxFinalized" +} + +// txConfirmNotificationSealed seals TxFinalized into the package +// notification set. +func (m *TxFinalized) txConfirmNotificationSealed() {} + // TxFailed notifies a subscriber that the actor encountered a terminal // failure while trying to confirm the tracked transaction. type TxFailed struct { diff --git a/txconfirm/reorg_test.go b/txconfirm/reorg_test.go new file mode 100644 index 000000000..dcc49e76e --- /dev/null +++ b/txconfirm/reorg_test.go @@ -0,0 +1,147 @@ +package txconfirm + +import ( + "testing" + + "github.com/lightninglabs/darepo-client/baselib/actor" + "github.com/stretchr/testify/require" +) + +// TestEnsureConfirmedReorgLifecycle drives the full reorg-aware lifecycle +// (Confirmed -> Reorged -> Confirmed -> Finalized) through the +// TxBroadcasterActor and asserts that: +// +// - Each chainsource event reaches the subscriber as a matching public +// notification, in order. +// - The conf watch is held open across the reorg-out / re-confirm +// bounce (no unregister fires before Done). +// - Finalization releases the conf watch and evicts the tracked entry. +func TestEnsureConfirmedReorgLifecycle(t *testing.T) { + chain := newFakeChainSourceRef(100) + ref, _ := newTestActor(t, Config{ + ChainSource: chain, + }) + + tx := makeTestTx(false) + txid := tx.TxHash() + sub := actor.NewChannelTellOnlyRef[Notification]("sub", 16) + + resp := mustEnsure(t, ref.Ref(), &EnsureConfirmedReq{ + Tx: tx, + Subscriber: sub, + }) + require.True(t, resp.Created) + require.Equal(t, TxStateAwaitingConfirmation, resp.State) + require.Equal(t, 1, chain.registerConfCount()) + + // 1. First confirmation on the canonical chain. + chain.emitConfirmation(t, txid, 101) + first := mustAwaitNotification(t, sub) + confirmed, ok := first.(*TxConfirmed) + require.True(t, ok, "first event must be TxConfirmed") + require.Equal(t, txid, confirmed.Txid) + require.Equal(t, int32(101), confirmed.BlockHeight) + + // The conf watch must NOT have been released yet: the entry is + // still in the reversible Confirmed state. + require.Equal(t, 0, chain.unregisterConfCount()) + + // 2. Reorg evicts that confirmation. + chain.emitConfReorged(t, txid) + second := mustAwaitNotification(t, sub) + reorged, ok := second.(*TxReorged) + require.True(t, ok, "second event must be TxReorged") + require.Equal(t, txid, reorged.Txid) + require.Equal(t, 0, chain.unregisterConfCount()) + + // 3. Transaction re-confirms on the new tip. + chain.emitConfirmation(t, txid, 102) + third := mustAwaitNotification(t, sub) + reConfirmed, ok := third.(*TxConfirmed) + require.True(t, ok, "third event must be TxConfirmed") + require.Equal(t, int32(102), reConfirmed.BlockHeight) + require.Equal(t, 0, chain.unregisterConfCount()) + + // 4. Finality. After TxFinalized the entry evicts. + chain.emitConfDone(t, txid) + fourth := mustAwaitNotification(t, sub) + finalized, ok := fourth.(*TxFinalized) + require.True(t, ok, "fourth event must be TxFinalized") + require.Equal(t, txid, finalized.Txid) + + mustEventually(t, func() bool { + return chain.unregisterConfCount() == 1 + }) + + // Cancel for the now-evicted entry must observe an empty map: Removed + // is false because there is nothing to remove. + cancelResp := mustCancel(t, ref.Ref(), &CancelInterestReq{ + Txid: txid, + SubscriberID: sub.ID(), + }) + require.False(t, cancelResp.Removed) +} + +// TestEnsureConfirmedDoneDuringReorgGapDropped pins the documented +// edge-case semantics: if the chainsource backend fires Done while the +// tracked entry is in AwaitingConfirmation (post-reorg, pre-re-confirm), +// txconfirm drops the Done rather than advancing to Finalized. The +// realistic backends (chainntnfs, lndclient) do not fire Done during a +// reorg gap, but this test pins the guard so a future backend change +// cannot silently land the entry in Finalized off a non-Confirmed +// state, and so an accidental relaxation of the guard fails loudly. +func TestEnsureConfirmedDoneDuringReorgGapDropped(t *testing.T) { + chain := newFakeChainSourceRef(100) + ref, _ := newTestActor(t, Config{ + ChainSource: chain, + }) + + tx := makeTestTx(false) + txid := tx.TxHash() + sub := actor.NewChannelTellOnlyRef[Notification]("sub", 16) + + mustEnsure(t, ref.Ref(), &EnsureConfirmedReq{ + Tx: tx, + Subscriber: sub, + }) + + // Confirm, then reorg out — entry is now in AwaitingConfirmation. + chain.emitConfirmation(t, txid, 101) + first := mustAwaitNotification(t, sub) + _, ok := first.(*TxConfirmed) + require.True(t, ok, "first event must be TxConfirmed") + + chain.emitConfReorged(t, txid) + second := mustAwaitNotification(t, sub) + _, ok = second.(*TxReorged) + require.True(t, ok, "second event must be TxReorged") + + // Fire Done out-of-band, before any re-confirmation. txconfirm + // must NOT advance the entry to Finalized, must NOT deliver + // TxFinalized to the subscriber, and must NOT unregister the + // conf watch. + chain.emitConfDone(t, txid) + + // Re-issuing EnsureConfirmedReq for the same (txid, params) is a + // no-op attach that flushes the mailbox: by the time the response + // returns, the queued confirmationDoneMsg has been processed (or + // in this case, logged and dropped). The reported state must + // remain AwaitingConfirmation. + probe := mustEnsure(t, ref.Ref(), &EnsureConfirmedReq{ + Tx: tx, + Subscriber: sub, + }) + require.False(t, probe.Created) + require.Equal( + t, TxStateAwaitingConfirmation, probe.State, + "Done during reorg gap must not promote entry to Finalized", + ) + + // No TxFinalized notification should have been delivered. + mustHaveNoNotification(t, sub) + + require.Equal( + t, 0, chain.unregisterConfCount(), + "dropped Done must not release the conf watch", + ) +} diff --git a/txconfirm/states.go b/txconfirm/states.go index 3555c5ce7..1b173fdad 100644 --- a/txconfirm/states.go +++ b/txconfirm/states.go @@ -249,7 +249,10 @@ func (s *trackedTxStateFeeBumping) ProcessEvent(_ context.Context, } } -// trackedTxStateConfirmed is the terminal confirmed state. +// trackedTxStateConfirmed is the reorg-reversible confirmed state. A +// transaction in this state has reached its target confirmation count on +// the canonical chain; a subsequent reorg moves it back to +// AwaitingConfirmation, and finality moves it to Finalized. type trackedTxStateConfirmed struct { trackedTxData trackedTxProgress @@ -263,19 +266,74 @@ func (s *trackedTxStateConfirmed) String() string { return "Confirmed" } -// IsTerminal returns true because confirmed is terminal. +// IsTerminal returns false because the confirmation is reversible until +// the backend reports finality via trackedTxFinalized. func (s *trackedTxStateConfirmed) IsTerminal() bool { - return true + return false } // trackedTxStateSealed marks trackedTxStateConfirmed as a tracked-tx state. func (s *trackedTxStateConfirmed) trackedTxStateSealed() {} -// ProcessEvent rejects unexpected events in the terminal confirmed state. +// ProcessEvent applies one event to the confirmed state. A reorg moves +// the FSM back to AwaitingConfirmation; finality moves it to the terminal +// Finalized state. func (s *trackedTxStateConfirmed) ProcessEvent(_ context.Context, event trackedTxEvent, _ *trackedTxEnvironment) ( *trackedTxStateTransition, error) { + switch event.(type) { + case *trackedTxReorged: + return &trackedTxStateTransition{ + NextState: &trackedTxStateAwaitingConfirmation{ + trackedTxData: s.trackedTxData, + trackedTxProgress: s.trackedTxProgress, + }, + }, nil + + case *trackedTxFinalized: + return &trackedTxStateTransition{ + NextState: &trackedTxStateFinalized{ + trackedTxData: s.trackedTxData, + trackedTxProgress: s.trackedTxProgress, + ConfirmHeight: s.ConfirmHeight, + }, + }, nil + + default: + return nil, fmt.Errorf("unexpected event %T in %s", event, s) + } +} + +// trackedTxStateFinalized is the terminal "confirmed past reorg safety +// depth" state. No further events are accepted. +type trackedTxStateFinalized struct { + trackedTxData + trackedTxProgress + + // ConfirmHeight is the block height where the tx confirmed before + // being finalized. + ConfirmHeight int32 +} + +// String returns a human-readable representation of the finalized state. +func (s *trackedTxStateFinalized) String() string { + return "Finalized" +} + +// IsTerminal returns true because finalized is terminal. +func (s *trackedTxStateFinalized) IsTerminal() bool { + return true +} + +// trackedTxStateSealed marks trackedTxStateFinalized as a tracked-tx state. +func (s *trackedTxStateFinalized) trackedTxStateSealed() {} + +// ProcessEvent rejects unexpected events in the terminal finalized state. +func (s *trackedTxStateFinalized) ProcessEvent(_ context.Context, + event trackedTxEvent, _ *trackedTxEnvironment) ( + *trackedTxStateTransition, error) { + return nil, fmt.Errorf("unexpected event %T in %s", event, s) } diff --git a/wallet/boarding_sweep_actor.go b/wallet/boarding_sweep_actor.go index 255e54fab..2620ee413 100644 --- a/wallet/boarding_sweep_actor.go +++ b/wallet/boarding_sweep_actor.go @@ -233,29 +233,154 @@ func (m BoardingSweepSpendNotification) MessageType() string { func (m BoardingSweepSpendNotification) walletMsgSealed() {} -// BoardingSweepTxNotification is a Tell carrying a txconfirm terminal -// notification (confirmation or failure) for a tracked sweep tx, -// re-wrapped from txconfirm.TxConfirmed / txconfirm.TxFailed via +// BoardingSweepTxStatus identifies which point in the txconfirm +// reorg-aware lifecycle drove this notification. Splitting the status +// out (instead of a single `Confirmed bool`) makes it impossible for +// the handler to confuse a reorg-out with a terminal failure: txconfirm +// fan-outs a TxReorged whenever a previously delivered TxConfirmed is +// rolled back on chain, and we must NOT treat that as a sweep failure. +type BoardingSweepTxStatus int + +const ( + // BoardingSweepTxStatusUnknown is the zero value; receiving it + // indicates a programmer error (MapNotification missed a kind). + BoardingSweepTxStatusUnknown BoardingSweepTxStatus = iota + + // BoardingSweepTxStatusConfirmed reports that the sweep + // transaction has been observed on the canonical chain at + // BlockHeight with NumConfs confirmations. The observation is + // provisional until BoardingSweepTxStatusFinalized arrives — a + // reorg can roll it back via BoardingSweepTxStatusReorged. + BoardingSweepTxStatusConfirmed + + // BoardingSweepTxStatusReorged reports that a previously + // delivered TxConfirmed for this sweep was reorged out. The + // handler must NOT call MarkBoardingSweepFailed; instead it + // leaves the spend watches and pending state armed so the next + // TxConfirmed on the new canonical chain (or an eventual + // TxFailed) drives the terminal decision. + BoardingSweepTxStatusReorged + + // BoardingSweepTxStatusFinalized reports that the sweep + // confirmation is past the chainsource backend's reorg-safety + // depth and is no longer reversible. The handler can release + // any reorg-recovery bookkeeping for this sweep. + BoardingSweepTxStatusFinalized + + // BoardingSweepTxStatusFailed reports a terminal failure from + // txconfirm (broadcast rejected, retry budget exhausted, etc.). + // The handler marks the sweep failed in the store and cleans up + // pending state. + BoardingSweepTxStatusFailed +) + +// classifyTxconfirmNotificationForBoardingSweep maps one event from +// the txconfirm reorg-aware lifecycle into the wallet-domain +// BoardingSweepTxNotification shape consumed by +// handleSweepTxNotification. Pulled out of submitSweepConfirmer so the +// systest (TestBoardingSweepReorgRoundTrip) can reuse the exact same +// classifier the production wiring uses; if the production wiring +// drifts from the systest classifier, the test stops validating +// production behavior. +func classifyTxconfirmNotificationForBoardingSweep( + n txconfirm.Notification) BoardingSweepTxNotification { + + switch ev := n.(type) { + case *txconfirm.TxConfirmed: + return BoardingSweepTxNotification{ + Status: BoardingSweepTxStatusConfirmed, + + Txid: ev.Txid, + BlockHeight: ev.BlockHeight, + NumConfs: ev.NumConfs, + } + + case *txconfirm.TxReorged: + // Reorg-out: do NOT mark the sweep as failed in the + // handler. Leave spend watches and pending state armed so + // the next TxConfirmed on the new canonical chain (or an + // eventual TxFailed) drives the terminal decision. + return BoardingSweepTxNotification{ + Status: BoardingSweepTxStatusReorged, + Txid: ev.Txid, + } + + case *txconfirm.TxFinalized: + // Reorg-safety horizon reached. The sweep observation is + // no longer reversible; the handler may release reorg- + // recovery bookkeeping. Audit/ledger emission already + // fired on the original Confirmed (idempotent on replay). + return BoardingSweepTxNotification{ + Status: BoardingSweepTxStatusFinalized, + + Txid: ev.Txid, + BlockHeight: ev.BlockHeight, + NumConfs: ev.NumConfs, + } + + case *txconfirm.TxFailed: + return BoardingSweepTxNotification{ + Status: BoardingSweepTxStatusFailed, + Txid: ev.Txid, + Reason: ev.Reason, + } + } + + return BoardingSweepTxNotification{} +} + +// NewBoardingSweepTxconfirmSubscriber wires the production boarding- +// sweep classifier onto a txconfirm.MapNotification anchored at +// selfRef. The returned ref can be set as the Subscriber field on a +// txconfirm.EnsureConfirmedReq; every TxConfirmed / TxReorged / +// TxFinalized / TxFailed delivered for that sweep will be classified +// into a BoardingSweepTxNotification and Tell'd to selfRef as a +// WalletMsg, which the wallet actor's Receive arm dispatches to +// handleSweepTxNotification. +// +// Exported so the systest can construct the exact same subscriber +// chain that submitSweepConfirmer uses in production. +func NewBoardingSweepTxconfirmSubscriber( + selfRef actor.TellOnlyRef[WalletMsg], +) actor.TellOnlyRef[txconfirm.Notification] { + + walletNotif := actor.NewMapInputRef[ + BoardingSweepTxNotification, WalletMsg, + ]( + selfRef, + func(n BoardingSweepTxNotification) WalletMsg { + return n + }, + ) + + return txconfirm.MapNotification( + walletNotif, classifyTxconfirmNotificationForBoardingSweep, + ) +} + +// BoardingSweepTxNotification is a Tell carrying one event from the +// txconfirm reorg-aware lifecycle for a tracked sweep tx, re-wrapped +// from txconfirm.{TxConfirmed,TxReorged,TxFinalized,TxFailed} via // txconfirm.MapNotification. type BoardingSweepTxNotification struct { actor.BaseMessage - // Confirmed is true when the underlying txconfirm.TxConfirmed event - // fired; false when it was txconfirm.TxFailed. - Confirmed bool + // Status identifies which lifecycle event this notification + // carries. See BoardingSweepTxStatus. + Status BoardingSweepTxStatus // Txid identifies the tracked sweep transaction. Txid chainhash.Hash // BlockHeight is the height at which the sweep confirmed when - // Confirmed=true; zero otherwise. + // Status=Confirmed or Finalized; zero otherwise. BlockHeight int32 - // NumConfs is the confirmation count when Confirmed=true; zero - // otherwise. + // NumConfs is the confirmation count when Status=Confirmed or + // Finalized; zero otherwise. NumConfs uint32 - // Reason is the human-readable failure reason when Confirmed=false. + // Reason is the human-readable failure reason when Status=Failed. Reason string } @@ -430,6 +555,16 @@ func (a *Ark) loadSweepCandidates(ctx context.Context, return intents, nil } +// isTerminalSuccessSweepStatus reports whether a persisted boarding-sweep +// status represents a resolved sweep whose accounting is already booked and +// must not be rolled back to failed: confirmed (our sweep landed) or +// external_resolved (the input was spent by another path). A spurious +// TxFailed for such a sweep is ignored. +func isTerminalSuccessSweepStatus(status string) bool { + return status == BoardingSweepStatusConfirmed || + status == BoardingSweepStatusExternalResolved +} + // defaultBoardingSweepStatuses are the boarding-intent statuses considered // candidates for an aggregate timeout sweep when no outpoint set is // supplied. @@ -868,37 +1003,11 @@ func (a *Ark) cancelSweepSpendWatches(ctx context.Context, func (a *Ark) submitSweepConfirmer(ctx context.Context, tx *wire.MsgTx, pkScript []byte, heightHint uint32) error { - walletNotif := actor.NewMapInputRef[ - BoardingSweepTxNotification, WalletMsg, - ]( - a.selfRef, - func(n BoardingSweepTxNotification) WalletMsg { - return n - }, - ) - - subscriber := txconfirm.MapNotification(walletNotif, - func(n txconfirm.Notification) BoardingSweepTxNotification { - switch ev := n.(type) { - case *txconfirm.TxConfirmed: - return BoardingSweepTxNotification{ - Confirmed: true, - Txid: ev.Txid, - BlockHeight: ev.BlockHeight, - NumConfs: ev.NumConfs, - } - - case *txconfirm.TxFailed: - return BoardingSweepTxNotification{ - Confirmed: false, - Txid: ev.Txid, - Reason: ev.Reason, - } - } + if a.actorSystem == nil { + return fmt.Errorf("actor system unavailable") + } - return BoardingSweepTxNotification{} - }, - ) + subscriber := NewBoardingSweepTxconfirmSubscriber(a.selfRef) return a.submitSweepToConfirm( ctx, tx, pkScript, heightHint, boardingSweepBroadcastLabel, @@ -1061,8 +1170,8 @@ func (a *Ark) handleSweepTxNotification(ctx context.Context, return fn.Ok[WalletResp](&BoardingSweepNotificationAck{}) } - switch { - case notif.Confirmed: + switch notif.Status { + case BoardingSweepTxStatusConfirmed: a.logger(ctx).DebugS( ctx, "Boarding sweep confirmation observed by broadcaster", @@ -1074,7 +1183,74 @@ func (a *Ark) handleSweepTxNotification(ctx context.Context, a.emitSweepConfirmedLedger(ctx, notif) - default: + case BoardingSweepTxStatusReorged: + // A previously delivered TxConfirmed for this sweep was + // reorged out. Do NOT mark the sweep as failed and do NOT + // tear down the pending entry: txconfirm keeps the watch + // alive, so a subsequent TxConfirmed on the new canonical + // chain will re-run reconcileSweepInputsOnConfirm with the + // new height. MarkBoardingSweepInputSpent is idempotent on + // (outpoint, txid), and the ledger emissions use txid-keyed + // idempotency, so the second confirmation will not produce + // duplicate audit/balance entries. + // + // Best-effort backstop only: the per-input chainsource + // spend watch fires handleSweepSpendNotification if some + // other party spends an input. That spend-watch path is + // not itself reorg-symmetric — MapSpendEvent collapses the + // chainsource SpendEvent lifecycle to a single + // BoardingSweepSpendNotification without surfacing + // Reorged / Done — so a reorged-out external spender will + // leave the input row marked external_spent until manual + // reconciliation. Closing that gap is tracked separately. + a.logger(ctx).WarnS( + ctx, + "Boarding sweep confirmation reorged out; waiting "+ + "for re-confirmation or external spend", + nil, + slog.String("txid", notif.Txid.String()), + ) + + case BoardingSweepTxStatusFinalized: + // Confirmation is past the backend's reorg-safety depth. + // No further reversible events will fire for this sweep — + // reorg-recovery bookkeeping can drop. Audit/ledger + // emission already ran on Confirmed and is idempotent on + // replay, so we do not re-emit here. + a.logger(ctx).DebugS( + ctx, + "Boarding sweep confirmation finalized", + nil, + slog.String("txid", notif.Txid.String()), + slog.Int("block_height", int(notif.BlockHeight)), + ) + + case BoardingSweepTxStatusFailed: + // Defensive guard against a failure arriving for a sweep that + // already confirmed. txconfirm's contract is that TxFailed + // never follows a real TxConfirmed, but the confirmed sweep's + // ledger legs (fee + per-input + destination) are irreversible + // and txid-keyed, so rolling the intents back to failed here + // would diverge the store from the ledger. If the persisted + // record shows the sweep already reached a terminal-success + // status, ignore the failure rather than undo a booked sweep. + if rec, ok := a.lookupSweepRecord(ctx, notif.Txid); ok && + isTerminalSuccessSweepStatus(rec.Status) { + + a.logger(ctx).WarnS( + ctx, + "Ignoring boarding sweep failure for an "+ + "already-resolved sweep", + errors.New(notif.Reason), + slog.String("txid", notif.Txid.String()), + slog.String("status", rec.Status), + ) + + return fn.Ok[WalletResp]( + &BoardingSweepNotificationAck{}, + ) + } + a.logger(ctx).WarnS( ctx, "Boarding sweep broadcaster reported failure", @@ -1104,6 +1280,14 @@ func (a *Ark) handleSweepTxNotification(ctx context.Context, if pending != nil { a.cancelSweepSpendWatches(ctx, pending) } + + default: + a.logger(ctx).WarnS( + ctx, + "Boarding sweep tx notification with unknown status", + fmt.Errorf("status=%d", notif.Status), + slog.String("txid", notif.Txid.String()), + ) } return fn.Ok[WalletResp](&BoardingSweepNotificationAck{}) @@ -1116,6 +1300,18 @@ func (a *Ark) handleSweepTxNotification(ctx context.Context, // the spending txid is the sweep's own txid. MarkBoardingSweepInputSpent // is idempotent, so inputs already resolved via the spend-notification path // are left untouched. +// +// Under the reorg-aware lifecycle a second TxConfirmed can land after a +// TxReorged / re-confirmation cycle. The store's status guard rejects +// the redundant transition with sql.ErrNoRows; that signals "input row +// already advanced past pending/published" and is treated as a benign +// no-op, mirroring how handleSweepSpendNotification classifies the +// same error. Other errors still log at warn because they indicate a +// real persistence problem. Note: the input row's confirmed_height +// stays pinned to the FIRST Confirmed observation — a deliberate +// first-observation-wins audit policy that survives reorg-reconfirm +// at a different height; the reorg-safety horizon is reported +// separately via the Finalized lifecycle event. func (a *Ark) reconcileSweepInputsOnConfirm(ctx context.Context, notif BoardingSweepTxNotification) { @@ -1128,7 +1324,17 @@ func (a *Ark) reconcileSweepInputsOnConfirm(ctx context.Context, _, err := a.sweepStore.MarkBoardingSweepInputSpent( ctx, op, notif.Txid, notif.BlockHeight, ) - if err != nil { + switch { + case err == nil: + // Success. + + case errors.Is(err, sql.ErrNoRows): + // Row already past pending/published — likely + // resolved via handleSweepSpendNotification or a + // previous Confirmed in a reorg-reconfirm cycle. + // Idempotent no-op. + + default: a.logger(ctx).WarnS( ctx, "Failed to mark sweep input spent on confirm", diff --git a/wallet/boarding_sweep_actor_test.go b/wallet/boarding_sweep_actor_test.go index a1030f27f..a8d103eae 100644 --- a/wallet/boarding_sweep_actor_test.go +++ b/wallet/boarding_sweep_actor_test.go @@ -599,7 +599,7 @@ func TestSweepTxNotificationConfirmedEmitsLedger(t *testing.T) { result := a.handleSweepTxNotification( t.Context(), BoardingSweepTxNotification{ - Confirmed: true, + Status: BoardingSweepTxStatusConfirmed, Txid: swept, BlockHeight: 800_650, NumConfs: 1, @@ -719,7 +719,7 @@ func TestSweepTxNotificationConfirmedExternalDestSkipsCreated(t *testing.T) { result := a.handleSweepTxNotification( t.Context(), BoardingSweepTxNotification{ - Confirmed: true, + Status: BoardingSweepTxStatusConfirmed, Txid: swept, BlockHeight: 800_700, }, @@ -854,12 +854,13 @@ func TestSweepLedgerClearingNetsToZero(t *testing.T) { ), ) + notif := BoardingSweepTxNotification{ + Status: BoardingSweepTxStatusConfirmed, + Txid: swept, + BlockHeight: 800_800, + } result := a.handleSweepTxNotification( - t.Context(), BoardingSweepTxNotification{ - Confirmed: true, - Txid: swept, - BlockHeight: 800_800, - }, + t.Context(), notif, ) require.True(t, result.IsOk()) @@ -934,7 +935,7 @@ func TestSweepTxNotificationMissingTxSkipsLegs(t *testing.T) { result := a.handleSweepTxNotification( t.Context(), BoardingSweepTxNotification{ - Confirmed: true, + Status: BoardingSweepTxStatusConfirmed, Txid: swept, BlockHeight: 800_900, }, @@ -962,6 +963,12 @@ func TestSweepTxNotificationFailedMarksFailed(t *testing.T) { mock.Anything, ).Return(nil) + // The Failed arm looks the sweep up first to ignore failures for an + // already-resolved sweep; absent record means the failure proceeds. + store.On( + "GetBoardingSweep", mock.Anything, failedTxid, + ).Return(nil, nil) + a := newSweepTestArk(t, store, nil, 0, 0) a.pendingSweeps[failedTxid] = &pendingSweepState{ txid: failedTxid, @@ -970,9 +977,9 @@ func TestSweepTxNotificationFailedMarksFailed(t *testing.T) { result := a.handleSweepTxNotification( t.Context(), BoardingSweepTxNotification{ - Confirmed: false, - Txid: failedTxid, - Reason: "test failure", + Status: BoardingSweepTxStatusFailed, + Txid: failedTxid, + Reason: "test failure", }, ) require.True(t, result.IsOk()) @@ -980,3 +987,245 @@ func TestSweepTxNotificationFailedMarksFailed(t *testing.T) { store.AssertExpectations(t) } + +// TestSweepTxNotificationReorgedDoesNotMarkFailed verifies that a +// TxReorged event from txconfirm — which arrives whenever a previously +// observed confirmation is rolled back on chain — must NOT be treated +// as a sweep failure. The handler should leave pendingSweeps and the +// persistent sweep record intact so that the next TxConfirmed on the +// new canonical chain (or, in the worst case, a chainsource spend +// notification for some other spender of the inputs) drives the +// terminal decision. +func TestSweepTxNotificationReorgedDoesNotMarkFailed(t *testing.T) { + t.Parallel() + + reorgedTxid := chainhash.Hash{0xa1} + store := &MockBoardingSweepStore{} + // CRITICAL: MarkBoardingSweepFailed must NOT be called on reorg. + // testify/mock will fail the test if any unexpected method is + // invoked, so we simply do not register MarkBoardingSweepFailed + // here and rely on AssertExpectations to verify the negative. + + a := newSweepTestArk(t, store, nil, 0, 0) + pending := &pendingSweepState{ + txid: reorgedTxid, + inputs: map[wire.OutPoint]string{}, + } + a.pendingSweeps[reorgedTxid] = pending + + result := a.handleSweepTxNotification( + t.Context(), BoardingSweepTxNotification{ + Status: BoardingSweepTxStatusReorged, + Txid: reorgedTxid, + }, + ) + require.True(t, result.IsOk()) + + // Pending state must remain intact: a re-confirmation on the new + // canonical chain has to find the same entry to drive its + // reconcileSweepInputsOnConfirm pass. + require.Same( + t, pending, a.pendingSweeps[reorgedTxid], + "reorg must not evict the pending sweep entry", + ) + + // Mock did not register MarkBoardingSweepFailed; AssertExpectations + // passes vacuously, but any call would have failed the mock. + store.AssertExpectations(t) +} + +// TestSweepTxNotificationFinalizedIsBenign verifies that a TxFinalized +// event (the chainsource reorg-safety horizon is reached) does not +// mark the sweep failed, does not re-emit ledger entries, and does +// not tear down pending state. Pending state remains because the +// terminal release path is the same as the legacy confirmation path — +// post-finalization cleanup is a no-op since reconcileSweepInputsOnConfirm +// already ran on the original TxConfirmed. +func TestSweepTxNotificationFinalizedIsBenign(t *testing.T) { + t.Parallel() + + finalizedTxid := chainhash.Hash{0xa2} + store := &MockBoardingSweepStore{} + // No expectations registered — finalized must not call any + // store mutation method. + + a := newSweepTestArk(t, store, nil, 0, 0) + pending := &pendingSweepState{ + txid: finalizedTxid, + inputs: map[wire.OutPoint]string{}, + } + a.pendingSweeps[finalizedTxid] = pending + + result := a.handleSweepTxNotification( + t.Context(), BoardingSweepTxNotification{ + Status: BoardingSweepTxStatusFinalized, + Txid: finalizedTxid, + BlockHeight: 800_750, + NumConfs: 6, + }, + ) + require.True(t, result.IsOk()) + + // Pending state should still be present; finalized is informational. + require.Same(t, pending, a.pendingSweeps[finalizedTxid]) + + store.AssertExpectations(t) +} + +// TestSweepTxNotificationReorgedAfterPendingCleared verifies the +// Reorged handler arm survives a missing pendingSweeps entry (which +// can happen when every per-input spend notification has already +// resolved and the entry was cleaned up by handleSweepSpendNotification +// before the tx-level reorg notification arrives). +func TestSweepTxNotificationReorgedAfterPendingCleared(t *testing.T) { + t.Parallel() + + reorgedTxid := chainhash.Hash{0xa3} + store := &MockBoardingSweepStore{} + // No store expectations — reorg with no pending entry must touch + // nothing. + + a := newSweepTestArk(t, store, nil, 0, 0) + // Note: pendingSweeps is intentionally empty. + + result := a.handleSweepTxNotification( + t.Context(), BoardingSweepTxNotification{ + Status: BoardingSweepTxStatusReorged, + Txid: reorgedTxid, + }, + ) + require.True(t, result.IsOk()) + require.Empty(t, a.pendingSweeps) + + store.AssertExpectations(t) +} + +// TestSweepTxNotificationFailedAfterReorgedStillTerminates verifies +// the Reorged arm does NOT suppress a subsequent terminal Failed +// notification: a sequence of Reorged then Failed must still call +// MarkBoardingSweepFailed and drop the pending entry, otherwise a +// reorg followed by a hard broadcast failure would be silently +// stranded. +func TestSweepTxNotificationFailedAfterReorgedStillTerminates(t *testing.T) { + t.Parallel() + + txid := chainhash.Hash{0xa4} + store := &MockBoardingSweepStore{} + store.On( + "MarkBoardingSweepFailed", mock.Anything, txid, + mock.Anything, + ).Return(nil) + + // The Failed arm now looks the sweep up first to ignore a spurious + // failure for an already-resolved sweep. Here the record is absent + // (not terminal-success), so the failure path proceeds as before. + store.On( + "GetBoardingSweep", mock.Anything, txid, + ).Return(nil, nil) + + a := newSweepTestArk(t, store, nil, 0, 0) + pending := &pendingSweepState{ + txid: txid, + inputs: map[wire.OutPoint]string{}, + } + a.pendingSweeps[txid] = pending + + // Step 1: Reorged — pending should remain, store should not be + // touched. + reorgResult := a.handleSweepTxNotification( + t.Context(), BoardingSweepTxNotification{ + Status: BoardingSweepTxStatusReorged, + Txid: txid, + }, + ) + require.True(t, reorgResult.IsOk()) + require.Same(t, pending, a.pendingSweeps[txid]) + + // Step 2: Failed — terminal path must still fire even though we + // passed through Reorged first. + failResult := a.handleSweepTxNotification( + t.Context(), BoardingSweepTxNotification{ + Status: BoardingSweepTxStatusFailed, + Txid: txid, + Reason: "post-reorg broadcast failure", + }, + ) + require.True(t, failResult.IsOk()) + require.Empty( + t, a.pendingSweeps, + "Failed after Reorged must still tear down pendingSweeps", + ) + + store.AssertExpectations(t) +} + +// TestSweepTxNotificationFailedIgnoredForConfirmedSweep verifies the +// defensive guard on the Failed arm: a spurious Failed notification for a +// sweep whose persisted record already shows a terminal-success status +// (confirmed) must NOT roll the sweep back to failed, because the +// confirmed sweep's ledger legs are irreversible and txid-keyed. +func TestSweepTxNotificationFailedIgnoredForConfirmedSweep(t *testing.T) { + t.Parallel() + + txid := chainhash.Hash{0xa6} + store := &MockBoardingSweepStore{} + + // The record is already confirmed, so the guard must short-circuit + // before MarkBoardingSweepFailed; no expectation is set for it, so a + // call would fail the test. + store.On( + "GetBoardingSweep", mock.Anything, txid, + ).Return(&BoardingSweepRecord{ + Status: BoardingSweepStatusConfirmed, + }, nil) + + a := newSweepTestArk(t, store, nil, 0, 0) + pending := &pendingSweepState{ + txid: txid, + inputs: map[wire.OutPoint]string{}, + } + a.pendingSweeps[txid] = pending + + failResult := a.handleSweepTxNotification( + t.Context(), BoardingSweepTxNotification{ + Status: BoardingSweepTxStatusFailed, + Txid: txid, + Reason: "spurious failure after confirmation", + }, + ) + require.True(t, failResult.IsOk()) + + store.AssertNotCalled(t, "MarkBoardingSweepFailed") + store.AssertExpectations(t) +} + +// TestSweepTxNotificationUnknownStatusIsBenign verifies the default +// arm of handleSweepTxNotification handles an unrecognised status +// without touching the store or pendingSweeps. This guards against a +// future txconfirm lifecycle event being added without a matching +// MapNotification arm. +func TestSweepTxNotificationUnknownStatusIsBenign(t *testing.T) { + t.Parallel() + + txid := chainhash.Hash{0xa5} + store := &MockBoardingSweepStore{} + // No expectations — unknown must touch nothing. + + a := newSweepTestArk(t, store, nil, 0, 0) + pending := &pendingSweepState{ + txid: txid, + inputs: map[wire.OutPoint]string{}, + } + a.pendingSweeps[txid] = pending + + result := a.handleSweepTxNotification( + t.Context(), BoardingSweepTxNotification{ + Status: BoardingSweepTxStatusUnknown, + Txid: txid, + }, + ) + require.True(t, result.IsOk()) + require.Same(t, pending, a.pendingSweeps[txid]) + + store.AssertExpectations(t) +}