diff --git a/adapters.go b/adapters.go index 42f63921..18f2bbf5 100644 --- a/adapters.go +++ b/adapters.go @@ -19,6 +19,15 @@ type Communication struct { Broadcaster } +func newCommunication(sender Sender, broadcaster Broadcaster, validators common.Nodes) *Communication { + c := &Communication{ + Sender: sender, + Broadcaster: broadcaster, + } + c.SetValidators(validators) + return c +} + func (c *Communication) SetValidators(nodes common.Nodes) { c.nodes.Store(nodes) } @@ -31,45 +40,52 @@ func (c *Communication) Validators() common.Nodes { return nodes } -// EpochAwareStorage is a wrapper around Storage that is aware of epoch changes. -// Upon an epoch change, it will ignore blocks from previous epochs -// and will call the onEpochChange callback when a new epoch is detected. -type EpochAwareStorage struct { - msm *metadata.StateMachine - onEpochChange func(seq uint64, validators common.Nodes) error +// InstanceStorage is a wrapper around Storage that skips indexing Telocks +// and delegates post-index handling to a caller-provided onIndex hook. +type InstanceStorage struct { Storage - epoch uint64 + + msm *metadata.StateMachine + + onIndex func(block *ParsedBlock) error } -func (e *EpochAwareStorage) Retrieve(seq uint64) (common.VerifiedBlock, common.Finalization, error) { - block, finalization, err := e.Storage.GetBlock(seq) +func NewInstanceStorage(storage Storage, msm *metadata.StateMachine, onIndex func(block *ParsedBlock) error) *InstanceStorage { + return &InstanceStorage{ + Storage: storage, + msm: msm, + onIndex: onIndex, + } +} + +func (s *InstanceStorage) Retrieve(seq uint64) (common.VerifiedBlock, common.Finalization, error) { + block, finalization, err := s.Storage.GetBlock(seq) if err != nil { return nil, common.Finalization{}, err } parsedBlock := &ParsedBlock{ - msm: e.msm, + msm: s.msm, StateMachineBlock: block, } return parsedBlock, *finalization, nil } -func (e *EpochAwareStorage) Index(ctx context.Context, block common.VerifiedBlock, certificate common.Finalization) error { - if block.BlockHeader().Epoch < e.epoch { - // This is a Telock from a previous epoch, so we ignore it and do not index it. +func (s *InstanceStorage) Index(ctx context.Context, block common.VerifiedBlock, certificate common.Finalization) error { + pb, ok := block.(*ParsedBlock) + if !ok { + return fmt.Errorf("expected ParsedBlock, got %T", block) + } + + // A Telock only extends time until the epoch transition finalizes, so we never index it. + if pb.Type() == metadata.BlockTypeTelock { return nil } - if err := e.Storage.Index(ctx, block, certificate); err != nil { + + if err := s.Storage.Index(ctx, block, certificate); err != nil { return err } - // This is a sealing block, and it is not the zero block - if block.SealingBlockInfo() != nil && block.SealingBlockInfo().PrevSealingBlockHash != [32]byte{} { - if err := e.onEpochChange(block.BlockHeader().Seq, block.SealingBlockInfo().ValidatorSet); err != nil { - return err - } - // We are now in a new epoch, so we update the epoch number to prevent indexing Telocks from the previous epoch. - e.epoch = block.BlockHeader().Seq - } - return nil + + return s.onIndex(pb) } // cachedBlock is a wrapper around ParsedBlock that caches the block in the CachedStorage upon verification. @@ -114,6 +130,16 @@ func (cs *CachedStorage) RetrieveBlock(seq uint64, digest common.Digest) (metada func (cs *CachedStorage) Retrieve(seq uint64, digest common.Digest) (common.VerifiedBlock, *common.Finalization, error) { cs.lock.RLock() item, exists := cs.cache[digest] + if !exists && digest == (common.Digest{}) { + // Seq-only lookups pass a zero digest, so scan the cache by seq. + // Otherwise a verified but not yet finalized block is invisible to them. + for _, cb := range cs.cache { + if cb.Metadata.SimplexProtocolMetadata.Seq == seq { + item, exists = cb, true + break + } + } + } if exists { cs.lock.RUnlock() // If the block is cached, it means it's not finalized yet, because upon finalizing the block (indexing) @@ -229,17 +255,17 @@ func (bw *BlockBuilderWaiter) BuildBlock(ctx context.Context, metadata common.Pr } type blockDeserializer struct { - vm VM - msm *metadata.StateMachine + deserializer BlockDeserializer + msm *metadata.StateMachine } -func (bp *blockDeserializer) DeserializeBlock(ctx context.Context, bytes []byte) (common.Block, error) { +func (bd *blockDeserializer) DeserializeBlock(ctx context.Context, bytes []byte) (common.Block, error) { var rawBlock metadata.RawBlock if err := rawBlock.UnmarshalCanoto(bytes); err != nil { return nil, err } - block, err := bp.vm.ParseBlock(ctx, rawBlock.InnerBlockBytes) + block, err := bd.deserializer.ParseBlock(ctx, rawBlock.InnerBlockBytes) if err != nil { return nil, err } @@ -248,6 +274,6 @@ func (bp *blockDeserializer) DeserializeBlock(ctx context.Context, bytes []byte) InnerBlock: block, Metadata: rawBlock.Metadata, }, - msm: bp.msm, + msm: bd.msm, }, nil } diff --git a/common/msg.canoto.go b/common/msg.canoto.go new file mode 100644 index 00000000..f3f6a49b --- /dev/null +++ b/common/msg.canoto.go @@ -0,0 +1,194 @@ +// Code generated by canoto. DO NOT EDIT. +// versions: +// canoto v0.19.0 +// source: msg.go + +package common + +import ( + "io" + "reflect" + "sync/atomic" + + "github.com/StephenButtolph/canoto" +) + +// Ensure that the generated code is compatible with the library version. +const ( + _ uint = canoto.VersionCompatibility - 1 + _ uint = 1 - canoto.VersionCompatibility +) + +// Ensure that unused imports do not error +var _ = io.ErrUnexpectedEOF + +const ( + canotoNumber_AuxiliaryInfo__Version = 1 + canotoNumber_AuxiliaryInfo__Data = 2 + + canotoTag_AuxiliaryInfo__Version = "\x08" // canoto.Tag(canotoNumber_AuxiliaryInfo__Version, canoto.Varint) + canotoTag_AuxiliaryInfo__Data = "\x12" // canoto.Tag(canotoNumber_AuxiliaryInfo__Data, canoto.Len) +) + +type canotoData_AuxiliaryInfo struct { + size uint64 +} + +// CanotoSpec returns the specification of this canoto message. +func (*AuxiliaryInfo) CanotoSpec(...reflect.Type) *canoto.Spec { + var zero AuxiliaryInfo + s := &canoto.Spec{ + Name: "AuxiliaryInfo", + Fields: []canoto.FieldType{ + { + FieldNumber: canotoNumber_AuxiliaryInfo__Version, + Name: "Version", + OneOf: "", + TypeUint: canoto.SizeOf(zero.Version), + }, + { + FieldNumber: canotoNumber_AuxiliaryInfo__Data, + Name: "Data", + OneOf: "", + TypeBytes: true, + }, + }, + } + s.CalculateCanotoCache() + return s +} + +// UnmarshalCanoto unmarshals a Canoto-encoded byte slice into the struct. +// +// During parsing, the canoto cache is saved. +func (c *AuxiliaryInfo) UnmarshalCanoto(bytes []byte) error { + r := canoto.Reader{ + B: bytes, + } + return c.UnmarshalCanotoFrom(r) +} + +// UnmarshalCanotoFrom populates the struct from a [canoto.Reader]. Most users +// should just use UnmarshalCanoto. +// +// During parsing, the canoto cache is saved. +// +// This function enables configuration of reader options. +func (c *AuxiliaryInfo) UnmarshalCanotoFrom(r canoto.Reader) error { + // Zero the struct before unmarshaling. + *c = AuxiliaryInfo{} + atomic.StoreUint64(&c.canotoData.size, uint64(len(r.B))) + + var minField uint32 + for canoto.HasNext(&r) { + field, wireType, err := canoto.ReadTag(&r) + if err != nil { + return err + } + if field < minField { + return canoto.ErrInvalidFieldOrder + } + + switch field { + case canotoNumber_AuxiliaryInfo__Version: + if wireType != canoto.Varint { + return canoto.ErrUnexpectedWireType + } + + if err := canoto.ReadUint(&r, &c.Version); err != nil { + return err + } + if canoto.IsZero(c.Version) { + return canoto.ErrZeroValue + } + case canotoNumber_AuxiliaryInfo__Data: + if wireType != canoto.Len { + return canoto.ErrUnexpectedWireType + } + + if err := canoto.ReadBytes(&r, &c.Data); err != nil { + return err + } + if len(c.Data) == 0 { + return canoto.ErrZeroValue + } + default: + return canoto.ErrUnknownField + } + + minField = field + 1 + } + return nil +} + +// ValidCanoto validates that the struct can be correctly marshaled into the +// Canoto format. +// +// Specifically, ValidCanoto ensures: +// 1. All OneOfs are specified at most once. +// 2. All strings are valid utf-8. +// 3. All custom fields are ValidCanoto. +func (c *AuxiliaryInfo) ValidCanoto() bool { + return true +} + +// CalculateCanotoCache populates size and OneOf caches based on the current +// values in the struct. +// +// It is not safe to copy this struct concurrently. +func (c *AuxiliaryInfo) CalculateCanotoCache() { + var size uint64 + if !canoto.IsZero(c.Version) { + size += uint64(len(canotoTag_AuxiliaryInfo__Version)) + canoto.SizeUint(c.Version) + } + if len(c.Data) != 0 { + size += uint64(len(canotoTag_AuxiliaryInfo__Data)) + canoto.SizeBytes(c.Data) + } + atomic.StoreUint64(&c.canotoData.size, size) +} + +// CachedCanotoSize returns the previously calculated size of the Canoto +// representation from CalculateCanotoCache. +// +// If CalculateCanotoCache has not yet been called, it will return 0. +// +// If the struct has been modified since the last call to CalculateCanotoCache, +// the returned size may be incorrect. +func (c *AuxiliaryInfo) CachedCanotoSize() uint64 { + return atomic.LoadUint64(&c.canotoData.size) +} + +// MarshalCanoto returns the Canoto representation of this struct. +// +// It is assumed that this struct is ValidCanoto. +// +// It is not safe to copy this struct concurrently. +func (c *AuxiliaryInfo) MarshalCanoto() []byte { + c.CalculateCanotoCache() + w := canoto.Writer{ + B: make([]byte, 0, c.CachedCanotoSize()), + } + w = c.MarshalCanotoInto(w) + return w.B +} + +// MarshalCanotoInto writes the struct into a [canoto.Writer] and returns the +// resulting [canoto.Writer]. Most users should just use MarshalCanoto. +// +// It is assumed that CalculateCanotoCache has been called since the last +// modification to this struct. +// +// It is assumed that this struct is ValidCanoto. +// +// It is not safe to copy this struct concurrently. +func (c *AuxiliaryInfo) MarshalCanotoInto(w canoto.Writer) canoto.Writer { + if !canoto.IsZero(c.Version) { + canoto.Append(&w, canotoTag_AuxiliaryInfo__Version) + canoto.AppendUint(&w, c.Version) + } + if len(c.Data) != 0 { + canoto.Append(&w, canotoTag_AuxiliaryInfo__Data) + canoto.AppendBytes(&w, c.Data) + } + return w +} diff --git a/common/msg.go b/common/msg.go index 59b7b1af..e28d8ca6 100644 --- a/common/msg.go +++ b/common/msg.go @@ -30,6 +30,10 @@ type Message struct { // Verified Messages VerifiedBlockMessage *VerifiedBlockMessage VerifiedReplicationResponse *VerifiedReplicationResponse + + // Epoch Transition Messages + AuxiliaryInfo *AuxiliaryInfo + EpochTransitionApproval *ValidatorSetApproval } func (m *Message) IsReplicationMessage() bool { @@ -432,6 +436,23 @@ type BlockDigestRequest struct { // VersionID is an identifier for applications that care about epoch changes. type VersionID uint32 +//go:generate go run github.com/StephenButtolph/canoto/canoto msg.go + +// AuxiliaryInfo defines application-specific information for applications that might care about epoch change, +// such as threshold distributed public key generation. +type AuxiliaryInfo struct { + // VersionID is an identifier that identifies the application. + // Can be used for backward-compatibility and upgrade purposes. + Version VersionID `canoto:"uint,1"` + + // Info is opaque bytes that can be used by applications to encode any information that describes + // the current state for the application. + Data []byte `canoto:"bytes,2"` + + canotoData canotoData_AuxiliaryInfo +} + +// ValidatorSetApproval is an approval from a validator type ValidatorSetApproval struct { NodeID avalanchego.NodeID AuxInfoDigest [32]byte diff --git a/config.go b/config.go index a47474f5..1aba21aa 100644 --- a/config.go +++ b/config.go @@ -53,12 +53,11 @@ type VM interface { // WaitForPendingBlock returns when either the given context is cancelled, // or when the VM signals that a block should be built. WaitForPendingBlock(ctx context.Context) +} +type BlockDeserializer interface { // ParseBlock parses the given block bytes into a VMBlock. ParseBlock(context.Context, []byte) (avalanchego.VMBlock, error) - - // ComputeICMEpoch computes the ICM epoch transition given the input parameters. - ComputeICMEpoch(input metadata.ICMEpochInput) metadata.ICMEpochInfo } type Storage interface { diff --git a/external_test.go b/external_test.go new file mode 100644 index 00000000..933867a2 --- /dev/null +++ b/external_test.go @@ -0,0 +1,112 @@ +package simplex + +import ( + "sync" + "testing" + "time" + + "github.com/ava-labs/simplex/common" + metadata "github.com/ava-labs/simplex/msm" + "github.com/stretchr/testify/require" +) + +func TestParseBlockSizeMatchesBytes(t *testing.T) { + // Case 1: Bytes() first, Size() second, size returns the cached length. + pb := &ParsedBlock{ + StateMachineBlock: metadata.StateMachineBlock{ + Metadata: metadata.StateMachineMetadata{ + SimplexProtocolMetadata: common.ProtocolMetadata{ + Version: 1, + Prev: common.Digest{}, + Round: 1, + Epoch: 4, + Seq: 2, + }, + SimplexBlacklist: common.Blacklist{ + Updates: common.BlacklistUpdates{{NodeIndex: 1, Type: 1}}, + NodeCount: 2, + }, + PChainHeight: 6, + }, + InnerBlock: &testInnerBlock{ + Height_: 7, + TS: time.UnixMilli(8), + Payload: []byte("payload"), + }, + }, + } + bytes := pb.Bytes() + require.Equal(t, len(bytes), pb.Size()) + + // Case 2: Size() first on a non serialized block. it will + // compute the size and match a later Byte() call. + pb2 := &ParsedBlock{ + StateMachineBlock: metadata.StateMachineBlock{ + Metadata: metadata.StateMachineMetadata{ + SimplexProtocolMetadata: common.ProtocolMetadata{ + Version: 1, + Prev: common.Digest{}, + Round: 1, + Epoch: 4, + Seq: 2, + }, + SimplexBlacklist: common.Blacklist{ + Updates: common.BlacklistUpdates{{NodeIndex: 1, Type: 1}}, + NodeCount: 2, + }, + PChainHeight: 6, + }, + InnerBlock: &testInnerBlock{ + Height_: 9, + TS: time.UnixMilli(10), + Payload: []byte("other payload"), + }, + }, + } + size := pb2.Size() + require.NotZero(t, size) + bytes2 := pb2.Bytes() + require.Equal(t, len(bytes2), size) + + // case 3: concurrent Size() calls on a block that was never serialized. + // the goroutines rase to compute the size, the lock must make this + // safe and every call must return the correct value + + pb3 := &ParsedBlock{ + StateMachineBlock: metadata.StateMachineBlock{ + Metadata: metadata.StateMachineMetadata{ + SimplexProtocolMetadata: common.ProtocolMetadata{ + Version: 1, + Prev: common.Digest{}, + Round: 1, + Epoch: 4, + Seq: 2, + }, + SimplexBlacklist: common.Blacklist{ + Updates: common.BlacklistUpdates{{NodeIndex: 1, Type: 1}}, + NodeCount: 2, + }, + PChainHeight: 6, + }, + InnerBlock: &testInnerBlock{ + Height_: 11, + TS: time.UnixMilli(12), + Payload: []byte("concurrent"), + }, + }, + } + var wg sync.WaitGroup + sizes := make([]int, 4) + for i := range sizes { + wg.Add(1) + go func() { + defer wg.Done() + sizes[i] = pb3.Size() + }() + } + wg.Wait() + bytes3 := pb3.Bytes() + for _, size := range sizes { + require.Equal(t, len(bytes3), size) + } +} diff --git a/instance.go b/instance.go index d53e6f5e..980ccdee 100644 --- a/instance.go +++ b/instance.go @@ -5,6 +5,7 @@ package simplex import ( "context" + "errors" "fmt" "math" "sync" @@ -19,6 +20,8 @@ import ( "go.uber.org/zap" ) +var errAlreadyStarted = errors.New("instance already started") + const ( // tickInterval is the interval at which the instance will call AdvanceTime on the current epoch or non-validator. tickInterval = time.Millisecond * 100 @@ -26,7 +29,7 @@ const ( type Config struct { // LastNonSimplexInnerBlock is the last non-simplex inner block that was persisted to storage. - // This is used to determine the current epoch and validator set. + // The genesis validator state from pchain is used to determine the current epoch and validator set. Can be the genesis block LastNonSimplexInnerBlock avalanchego.VMBlock // ParameterConfig is the configuration for the simplex instance. ParameterConfig ParameterConfig @@ -34,38 +37,33 @@ type Config struct { PlatformChain PlatformChain // Broadcaster is the interface to broadcast messages to other nodes in the network. Broadcaster Broadcaster + // Sender is an interface to send messages to a specific node in the network + Sender Sender // CryptoOps is the interface to the cryptographic operations needed by the simplex instance. CryptoOps CryptoOps // WalCreator is the interface to create new write-ahead logs for the simplex instance. WalCreator wal.Creator // Storage is the interface to the block storage layer for the simplex instance. - Storage Storage - Logger common.Logger - Sender Sender - WALs []wal.DeletableWAL - VM VM - ID common.NodeID + Storage Storage + Logger common.Logger + WALs []wal.DeletableWAL + VM VM + ICMETransition metadata.ICMEpochTransition + BlockDeserializer BlockDeserializer + ID common.NodeID } -type nodeRole byte - -const ( - nonValidator nodeRole = iota - validator -) - type epochChange struct { - epochNum uint64 + epoch uint64 validators common.Nodes - nodeRole nodeRole } - type timeAdvancer interface { AdvanceTime(t time.Time) } type Instance struct { - Config Config + Config Config + lock sync.Mutex started bool cs *CachedStorage @@ -93,21 +91,14 @@ func (i *Instance) Start(ctx context.Context) error { defer i.lock.Unlock() if i.started { - return fmt.Errorf("instance already started") + return errAlreadyStarted } i.started = true context.AfterFunc(ctx, i.Stop) - lastBlock, numBlocks, err := i.lastBlock() - if err != nil { - return fmt.Errorf("error retrieving last block: %w", err) - } - - lastNonSimplexHeight := i.Config.LastNonSimplexInnerBlock.Height() - genesisValidatorSet := i.Config.PlatformChain.GenesisValidatorSet() - nodes, epochNum, err := constructEpochAndValidatorSet(i.Config.Logger, lastNonSimplexHeight, genesisValidatorSet, numBlocks, &ParsedBlock{StateMachineBlock: lastBlock}, i.Config.Storage) + nodes, epochNum, err := getLastAcceptedEpochAndValidatorSet(&i.Config) if err != nil { return fmt.Errorf("error determining latest epoch and validator set: %w", err) } @@ -122,16 +113,16 @@ func (i *Instance) Start(ctx context.Context) error { return nil } -func (i *Instance) startValidator() error { - epochConfig, err := i.createEpochConfig() +func (i *Instance) startValidator(epoch uint64, validators common.Nodes) error { + epochConfig, err := i.createEpochConfig(epoch, validators) if err != nil { return err } return i.startEpoch(epochConfig) } -func (i *Instance) startNonValidator(epochNum uint64, validators common.Nodes) error { - config, err := i.createNonValidatorConfig(epochNum, validators) +func (i *Instance) startNonValidator() error { + config, err := i.createNonValidatorConfig() if err != nil { return err } @@ -146,35 +137,41 @@ func (i *Instance) startNonValidator(epochNum uint64, validators common.Nodes) e return nil } -func (i *Instance) createNonValidatorConfig(epochNum uint64, validators common.Nodes) (nonvalidator.Config, error) { +func (i *Instance) createNonValidatorConfig() (nonvalidator.Config, error) { source, err := simplex.NewRandomSource() if err != nil { return nonvalidator.Config{}, err } - comm := &Communication{Sender: i.Config.Sender, Broadcaster: i.Config.Broadcaster} - comm.SetValidators(validators) - - epochAwareStorage := &EpochAwareStorage{ - epoch: epochNum, - Storage: i.Config.Storage, - onEpochChange: func(epoch uint64, validators common.Nodes) error { - height := i.Config.PlatformChain.GetCurrentHeight() - vdrs, err := i.Config.PlatformChain.GetValidatorSet(height) + nodes, err := GetHighestValidatorSet(i.Config.PlatformChain) + if err != nil { + return nonvalidator.Config{}, err + } + comm := newCommunication(i.Config.Sender, i.Config.Broadcaster, nodes) + + // Non-validators have no block builder, so they pass a nil approval handler: + // they broadcast approvals but do not need to record their own locally. + transitionListener := newEpochTransitionListener( + i.Config.Logger, + comm, + avalanchego.NodeID(i.Config.ID), + i.Config.PlatformChain.GetValidatorSet, + i.cs.RetrieveBlock, + i.Config.CryptoOps, + &NoopAuxiliaryInfoApp{}, // TODO: set this in the config + nil, + func(epoch uint64, validators common.Nodes) error { + // set the communication to the highest validator set, since this node is a non-validator and may be behind. + nodes, err := GetHighestValidatorSet(i.Config.PlatformChain) if err != nil { - i.Config.Logger.Error("error getting validator set", zap.Error(err)) - return fmt.Errorf("error getting validator set from platform chain: %w", err) - } - comm.SetValidators(validators) - if i.iAmValidator(vdrs.Nodes()) { - i.notifyEpochChange(epoch, validators, nonValidator) - } else { - i.Config.Logger.Debug("I am still a non-validator at the tip of the P-chain, skipping role change", - zap.Uint64("height", height)) + return err } + comm.SetValidators(nodes) + i.notifyEpochChange(epoch, validators) + return nil }, - } + ) // Plant an artificial MSM that just skips verification. i.msm = &metadata.StateMachine{ @@ -183,11 +180,12 @@ func (i *Instance) createNonValidatorConfig(epochNum uint64, validators common.N }, } i.cs.msm = i.msm + instanceStorage := NewInstanceStorage(i.cs, i.msm, transitionListener.onIndex) config := nonvalidator.Config{ ID: i.Config.ID, RandomSource: source, - Storage: epochAwareStorage, + Storage: instanceStorage, Comm: comm, Logger: i.Config.Logger, StartTime: time.Now(), @@ -197,16 +195,25 @@ func (i *Instance) createNonValidatorConfig(epochNum uint64, validators common.N return config, nil } -func (i *Instance) notifyEpochChange(epoch uint64, validators common.Nodes, role nodeRole) { - select { - case i.epochChanges <- epochChange{ - epochNum: epoch, +// notifyEpochChange hands the latest epoch change to listenForEpochChanges. +func (i *Instance) notifyEpochChange(epoch uint64, validators common.Nodes) { + ec := epochChange{ + epoch: epoch, validators: validators, - nodeRole: role, - }: - case <-i.stopCh: - // If the instance is stopped, we don't need to notify about epoch changes. - return + } + + for { + select { + case i.epochChanges <- ec: + return + case <-i.stopCh: + return + case pending := <-i.epochChanges: + // The slot holds a stale epoch change: take it and keep the newer of the two. + if pending.epoch > ec.epoch { + ec = pending + } + } } } @@ -290,6 +297,13 @@ func (i *Instance) HandleMessage(msg *common.Message, from common.NodeID) error } if i.e != nil { + switch { + case msg.AuxiliaryInfo != nil: + i.msm.HandleAuxiliaryInfo(*msg.AuxiliaryInfo, avalanchego.NodeID(from)) + case msg.EpochTransitionApproval != nil: + // TODO: pass in time.Now() rather than uint64 + i.msm.HandleApproval(msg.EpochTransitionApproval, uint64(time.Now().Unix())) + } return i.e.HandleMessage(msg, from) } @@ -353,8 +367,8 @@ func (i *Instance) wireBlockMessage(msg *common.Message) error { func (i *Instance) listenForEpochChanges() { for { select { - case epochChange := <-i.epochChanges: - i.processEpochChange(epochChange) + case newEpoch := <-i.epochChanges: + i.processEpochChange(newEpoch) case <-i.stopCh: return } @@ -362,23 +376,41 @@ func (i *Instance) listenForEpochChanges() { } func (i *Instance) processEpochChange(epochChange epochChange) { + if i.isStopped() { + i.Config.Logger.Info("instance is already stopped, skipping epoch change") + return + } + var err error - switch epochChange.nodeRole { - case nonValidator: + isValidator, isNonValidator := i.IsValidator(), i.isNonValidator() + + switch { + case isValidator && isNonValidator: + i.Config.Logger.Fatal("We are running both a validator and non-validator") + return + case isNonValidator: err = i.transitionEpochNonValidator(epochChange) - case validator: + case isValidator: err = i.transitionEpochValidator(epochChange) default: // This should never happen, but we log it just in case. - i.Config.Logger.Fatal("Unknown node role on epoch change", - zap.String("role", fmt.Sprintf("%v", epochChange.nodeRole))) + i.Config.Logger.Fatal("We are not running either a validator or non-validator") return } + if err != nil { - i.Config.Logger.Error("Error transitioning epoch", zap.Uint8("role", uint8(epochChange.nodeRole)), zap.Error(err)) + i.Config.Logger.Error("Error transitioning epoch", zap.Error(err)) i.Stop() } } +func (i *Instance) IsValidator() bool { + return i.e != nil +} + +func (i *Instance) isNonValidator() bool { + return i.nv != nil +} + // startEpoch starts a new epoch with the given configuration. // Must be called under the lock, and assumes that the previous epoch has been stopped (if any). func (i *Instance) startEpoch(epochConfig simplex.EpochConfig) error { @@ -393,42 +425,7 @@ func (i *Instance) startEpoch(epochConfig simplex.EpochConfig) error { return epoch.Start() } -func (i *Instance) lastBlock() (metadata.StateMachineBlock, uint64, error) { - numBlocks := i.Config.Storage.NumBlocks() - if numBlocks == 0 { - return metadata.StateMachineBlock{}, 0, fmt.Errorf("no genesis block found in storage") - } - - lastBlock, _, err := i.Config.Storage.GetBlock(numBlocks - 1) - if err != nil { - return metadata.StateMachineBlock{}, 0, fmt.Errorf("error retrieving last block from storage: %w", err) - } - - return lastBlock, numBlocks, nil -} - -func (i *Instance) iAmValidator(nodes common.Nodes) bool { - for _, node := range nodes { - if i.Config.ID.Equals(node.Id) { - return true - } - } - return false -} - -func (i *Instance) createEpochConfig() (simplex.EpochConfig, error) { - lastBlock, numBlocks, err := i.lastBlock() - if err != nil { - return simplex.EpochConfig{}, err - } - - lastNonSimplexHeight := i.Config.LastNonSimplexInnerBlock.Height() - genesisValidatorSet := i.Config.PlatformChain.GenesisValidatorSet() - nodes, epochNum, err := constructEpochAndValidatorSet(i.Config.Logger, lastNonSimplexHeight, genesisValidatorSet, numBlocks, &ParsedBlock{StateMachineBlock: lastBlock}, i.Config.Storage) - if err != nil { - return simplex.EpochConfig{}, err - } - +func (i *Instance) createEpochConfig(epoch uint64, validators common.Nodes) (simplex.EpochConfig, error) { wal, err := wal.NewGarbageCollectedWAL(i.Config.WALs, i.Config.WalCreator, &common.WALRetentionReader{}, i.Config.ParameterConfig.WALMaxEntryCount) if err != nil { return simplex.EpochConfig{}, fmt.Errorf("error creating garbage collected wal: %w", err) @@ -438,7 +435,7 @@ func (i *Instance) createEpochConfig() (simplex.EpochConfig, error) { // We might have crashed right after a sealing block was persisted to storage, // but before the WAL was garbage collected. // In that case, we need to garbage collect the WAL to remove all entries from previous epochs. - if err := i.maybeGarbageCollectWAL(lastBlock); err != nil { + if err := i.maybeGarbageCollectWAL(); err != nil { return simplex.EpochConfig{}, err } @@ -453,15 +450,15 @@ func (i *Instance) createEpochConfig() (simplex.EpochConfig, error) { MaxBlockBuildingWaitTime: i.Config.ParameterConfig.MaxNetworkDelay, Logger: i.Config.Logger, Signer: i.Config.CryptoOps, - GenesisValidatorSet: genesisValidatorSet, - LastNonSimplexBlockPChainHeight: lastNonSimplexHeight, + GenesisValidatorSet: i.Config.PlatformChain.GenesisValidatorSet(), + LastNonSimplexBlockPChainHeight: i.Config.LastNonSimplexInnerBlock.Height(), SignatureAggregatorCreator: i.Config.CryptoOps.CreateSignatureAggregator, BlockBuilder: i.Config.VM, LastNonSimplexInnerBlock: i.Config.LastNonSimplexInnerBlock, GetPChainHeightForProposing: i.Config.PlatformChain.GetMinimumHeight, GetPChainHeightForVerifying: i.Config.PlatformChain.GetCurrentHeight, AuxiliaryInfoApp: &NoopAuxiliaryInfoApp{}, - ComputeICMEpoch: i.Config.VM.ComputeICMEpoch, + ComputeICMEpoch: i.Config.ICMETransition, GetBlock: i.cs.RetrieveBlock, }) if err != nil { @@ -478,26 +475,31 @@ func (i *Instance) createEpochConfig() (simplex.EpochConfig, error) { blockBuilder := &BlockBuilderWaiter{vm: i.Config.VM, msm: msm} - comm := &Communication{Sender: i.Config.Sender, Broadcaster: i.Config.Broadcaster} - comm.SetValidators(nodes) - - epochAwareStorage := &EpochAwareStorage{ - msm: msm, - epoch: epochNum, - Storage: i.cs, - onEpochChange: func(epoch uint64, validators common.Nodes) error { + comm := newCommunication(i.Config.Sender, i.Config.Broadcaster, validators) + + transitionListener := newEpochTransitionListener( + i.Config.Logger, + comm, + avalanchego.NodeID(i.Config.ID), + i.Config.PlatformChain.GetValidatorSet, + i.cs.RetrieveBlock, + i.Config.CryptoOps, + &NoopAuxiliaryInfoApp{}, + msm.HandleApproval, + func(epoch uint64, validators common.Nodes) error { blockBuilder.stop() comm.SetValidators(validators) - i.notifyEpochChange(epoch, validators, validator) + i.notifyEpochChange(epoch, validators) return nil }, - } + ) + instanceStorage := NewInstanceStorage(i.cs, msm, transitionListener.onIndex) epochConfig := simplex.EpochConfig{ - Epoch: epochNum, + Epoch: epoch, ReplicationEnabled: true, StartTime: time.Now(), - // TODO: For simpicity, we use the same value for all timeouts. If needed we can expand the config. + // TODO: For simplicity, we use the same value for all timeouts. If needed we can expand the config. MaxProposalWait: i.Config.ParameterConfig.MaxNetworkDelay * 2, // 1 proposal + 1 vote MaxRebroadcastWait: i.Config.ParameterConfig.MaxNetworkDelay * 2, FinalizeRebroadcastTimeout: i.Config.ParameterConfig.MaxNetworkDelay * 2, @@ -510,15 +512,20 @@ func (i *Instance) createEpochConfig() (simplex.EpochConfig, error) { QCDeserializer: i.Config.CryptoOps, Signer: i.Config.CryptoOps, Verifier: i.Config.CryptoOps, - Storage: epochAwareStorage, + Storage: instanceStorage, Comm: comm, BlockBuilder: blockBuilder, - BlockDeserializer: &blockDeserializer{vm: i.Config.VM, msm: msm}, + BlockDeserializer: &blockDeserializer{deserializer: i.Config.BlockDeserializer, msm: msm}, } return epochConfig, nil } -func (i *Instance) maybeGarbageCollectWAL(lastBlock metadata.StateMachineBlock) error { +func (i *Instance) maybeGarbageCollectWAL() error { + lastBlock, _, err := LastBlock(i.Config.Storage) + if err != nil { + return fmt.Errorf("error retrieving last block: %w", err) + } + if lastBlock.Metadata.SimplexEpochInfo.BlockValidationDescriptor != nil { i.Config.Logger.Info("Last block is a sealing block, garbage collecting all WALs preceding it to start a new epoch") // We figure out the round number of the latest block and garbage collect all WALs preceding it. @@ -537,12 +544,7 @@ func (i *Instance) transitionEpochNonValidator(epochChange epochChange) error { i.lock.Lock() defer i.lock.Unlock() - if i.isStopped() { - i.Config.Logger.Info("instance is already stopped, skipping epoch change") - return nil - } - - if !i.iAmValidator(epochChange.validators) { + if !i.isValidatorForLatestEpoch(epochChange.epoch, epochChange.validators) { i.Config.Logger.Debug("Skipping restarting a non-validator because I am not a validator yet") return nil } @@ -550,19 +552,32 @@ func (i *Instance) transitionEpochNonValidator(epochChange epochChange) error { // Stop the non-validator before doing anything else, so that we don't process any more messages while we are changing epochs. i.stopNonValidator() - return i.startAtEpoch(epochChange.validators, epochChange.epochNum) + return i.startAtEpoch(epochChange.validators, epochChange.epoch) +} + +// isValidatorForLatestEpoch returns if this instance is a validator for the highest validator set +func (i *Instance) isValidatorForLatestEpoch(epoch uint64, newValidatorSet common.Nodes) bool { + if i.nv != nil { + highestEpoch, highestValidatorSet := i.nv.HighestValidatedEpoch() + return highestValidatorSet.Contains(i.Config.ID) && highestEpoch == epoch + } + + // TODO: this assumes newValidatorSet & epoch are the highest, which may not hold. + // Validators should collect a threshold of votes for the highest epoch, like non-validators do when starting. + return newValidatorSet.Contains(i.Config.ID) } +// startAtEpoch starts either a validator or non-validator at epoch. func (i *Instance) startAtEpoch(validators common.Nodes, epoch uint64) error { - if i.iAmValidator(validators) { - if err := i.startValidator(); err != nil { + if i.isValidatorForLatestEpoch(epoch, validators) { + if err := i.startValidator(epoch, validators); err != nil { i.Config.Logger.Error("Error starting validator on epoch change", zap.Error(err)) return err } return nil } - if err := i.startNonValidator(epoch, validators); err != nil { + if err := i.startNonValidator(); err != nil { i.Config.Logger.Error("Error starting non-validator on epoch change", zap.Error(err)) return err } @@ -582,72 +597,15 @@ func (i *Instance) transitionEpochValidator(epochChange epochChange) error { i.Config.Logger.Error("Error garbage collecting epoch config on epoch change", zap.Error(err)) } - return i.startAtEpoch(epochChange.validators, epochChange.epochNum) + return i.startAtEpoch(epochChange.validators, epochChange.epoch) } -func constructEpochAndValidatorSet(logger common.Logger, lastNonSimplexInnerBlockHeight uint64, genesisValidatorSet metadata.NodeBLSMappings, numBlocks uint64, lastBlock *ParsedBlock, storage Storage) (common.Nodes, uint64, error) { - epochNum := lastBlock.BlockHeader().Epoch - - var validatorSet metadata.NodeBLSMappings - var nodes common.Nodes - - switch { - // If all we have in the ledger is non-Simplex blocks, load the validator set from genesis - case lastNonSimplexInnerBlockHeight+1 == numBlocks: - validatorSet = genesisValidatorSet - nodes = validatorSetToNodes(genesisValidatorSet) - epochNum = lastNonSimplexInnerBlockHeight + 1 - logger.Debug("Determined epoch and validator set from genesis (ledger holds only non-Simplex blocks)", - zap.Uint64("epoch", epochNum)) - // If the last block persisted is a sealing block, then we are in the next epoch. - case lastBlock.SealingBlockInfo() != nil: - epochNum = lastBlock.BlockHeader().Seq - validatorSet = constructValidatorSetFromSealingBlock(lastBlock) - nodes = lastBlock.SealingBlockInfo().ValidatorSet - logger.Debug("Determined epoch and validator set from sealing block at tip", - zap.Uint64("epoch", epochNum)) - // Else, we have at least one Simplex block in the ledger, and it's not a sealing block. - default: - // Therefore, the sequence of the sealing block is the epoch number. - sealingBlockSeq := lastBlock.BlockHeader().Epoch - sealingBlock, _, err := storage.GetBlock(sealingBlockSeq) - if err != nil { - return nil, 0, fmt.Errorf("error retrieving sealing block from storage: %w", err) - } - if sealingBlock.Metadata.SimplexEpochInfo.BlockValidationDescriptor == nil { - return nil, 0, fmt.Errorf("expected sealing block at seq %d, but got a non-sealing block", sealingBlockSeq) - } - validatorSet = constructValidatorSetFromSealingBlock(&ParsedBlock{StateMachineBlock: sealingBlock}) - nodes = validatorSetToNodes(validatorSet) - logger.Debug("Determined epoch and validator set from sealing block in storage", - zap.Uint64("epoch", epochNum), zap.Uint64("sealingBlockSeq", sealingBlockSeq)) +func GetHighestValidatorSet(platform PlatformChain) (common.Nodes, error) { + height := platform.GetCurrentHeight() + mappings, err := platform.GetValidatorSet(height) + if err != nil { + return nil, err } - return nodes, epochNum, nil -} - -func validatorSetToNodes(validatorSet metadata.NodeBLSMappings) common.Nodes { - var nodes common.Nodes - for i := range validatorSet { - vdr := &validatorSet[i] - nodes = append(nodes, common.Node{ - Id: vdr.NodeID[:], - Weight: vdr.Weight, - PK: vdr.BLSKey, - }) - } - return nodes -} -func constructValidatorSetFromSealingBlock(lastBlock *ParsedBlock) metadata.NodeBLSMappings { - var validatorSet metadata.NodeBLSMappings - vdrs := lastBlock.Metadata.SimplexEpochInfo.BlockValidationDescriptor.AggregatedMembership.Members - for i := range vdrs { - vdr := &vdrs[i] - validatorSet = append(validatorSet, metadata.NodeBLSMapping{ - NodeID: vdr.NodeID, - BLSKey: vdr.BLSKey, - Weight: vdr.Weight, - }) - } - return validatorSet + return mappings.Nodes(), nil } diff --git a/instance_test.go b/instance_test.go index 5f85dd84..b3639444 100644 --- a/instance_test.go +++ b/instance_test.go @@ -4,1147 +4,198 @@ package simplex import ( - "bytes" - "context" - "crypto/rand" - "crypto/sha256" - "encoding/asn1" - "encoding/binary" - "fmt" - "sort" - "strings" "sync" - "sync/atomic" "testing" "time" - "github.com/ava-labs/simplex/avalanchego" - "github.com/ava-labs/simplex/common" metadata "github.com/ava-labs/simplex/msm" - "github.com/ava-labs/simplex/testutil" - "github.com/ava-labs/simplex/wal" - "github.com/stretchr/testify/require" - "go.uber.org/zap/zapcore" ) -func TestInstanceMixedNodeType(t *testing.T) { - // One node is a validator at genesis, the other is a non-validator. - // After some blocks, the second (non-validator) node also becomes a validator. - // The test ensures that the second node tracks the chain while the first node expands the chain - // in the first epoch, and that both nodes move to the second epoch and then both are used for consensus together. - const ( - basePChainHeight = uint64(1) - epochChangePChainHeight = uint64(100) - ) - - var id [20]byte - rand.Read(id[:]) - firstNodeID := common.NodeID(id[:]) - - // The peer that joins the validator set in the last epoch. Its ID is chosen - // to differ from the (random) node under test. - var peerID [20]byte - rand.Read(peerID[:]) - secondNodeID := common.NodeID(peerID[:]) - - // Epoch 1 is single-validator - // The last epoch is expanded to two validators. - validatorSetsAtHeight := map[uint64]metadata.NodeBLSMappings{ - basePChainHeight: { - {NodeID: id, BLSKey: []byte{0xaa}, Weight: 1}, - }, - epochChangePChainHeight: { - {NodeID: id, BLSKey: []byte{0xaa}, Weight: 2}, - {NodeID: peerID, BLSKey: []byte{0xbb}, Weight: 2}, - }, - } - - pChain := newTestPlatformChain(basePChainHeight, validatorSetsAtHeight) - cops := &testCryptoOps{} - - genesisBlock := &testInnerBlock{Height_: 0, TS: time.Now(), Payload: []byte("genesis")} - - net := newInMemNetwork(t) - t.Cleanup(net.stop) - - // Create the storage for the instances and append the genesis block to each - storage := newStorageWithGenesis(t, genesisBlock) - storage2 := newStorageWithGenesis(t, genesisBlock) - - // Create the instances and register them to the network - firstInstance := newInstance(t, firstNodeID, storage, net, pChain, cops, genesisBlock) - secondInstance := newInstance(t, secondNodeID, storage2, net, pChain, cops, genesisBlock) - net.register(firstNodeID, firstInstance) - net.register(secondNodeID, secondInstance) - - /// Start the instances - require.NoError(t, firstInstance.Start(t.Context())) - require.NoError(t, secondInstance.Start(t.Context())) - t.Cleanup(firstInstance.Stop) - t.Cleanup(secondInstance.Stop) - - // Epoch 1: wait until the node has committed a series of normal blocks on its own. - const epoch1Target = uint64(5) // genesis(0) + zero block(1) + 3 normal blocks - waitForNumBlocks(t, storage, epoch1Target) - waitForNumBlocks(t, storage2, epoch1Target) +// TestValidatorIndexes tests that a validator indexes and accepts a new block sent by the network +// It is the only validator, so it will build and finalize its own block. +func TestValidatorIndexes(t *testing.T) { + validator := newBLSMapping(1) - // The validator set in force is the one introduced by the most recent block - // that carries a BlockValidationDescriptor (the zero block in epoch 1). - require.Equal(t, firstInstance.Config.ID, latestValidatorID(t, storage)) - require.Equal(t, firstInstance.Config.ID, latestValidatorID(t, storage2)) + genesisSet := []metadata.NodeBLSMapping{validator} - // Trigger the epoch change: the validator set changes at epochChangePChainHeight, - // growing from one validator to two. - pChain.advanceTo(epochChangePChainHeight) - approval := &common.ValidatorSetApproval{ - NodeID: peerID, - PChainHeight: epochChangePChainHeight, - AuxInfoDigest: sha256.Sum256(nil), - Signature: []byte{1, 2, 3}, - } + pChain := newTestPChain(genesisSet) + chain := newNetwork(t, pChain) + chain.addNode(validator.NodeID[:]) - // The node seals the epoch once it has a quorum of approvals of the new - // (two-validator) set. With two validators the node's self-approval is no longer - // a quorum and the peer is not running yet, so waitForSealingBlock injects the - // peer's approval on each poll until the sealing block is committed. - // TODO: Implement this capability in production so we won't need to inject approvals in tests. - sealingBlockSeq := waitForSealingBlock(t, firstInstance, approval, storage.NumBlocks()) - waitForNumBlocks(t, storage2, sealingBlockSeq) // Ensure the new validator has replicated the sealing block. - - // With both validators live, the two-validator epoch commits more blocks. - const epoch2Extra = uint64(3) - waitForNumBlocks(t, storage, sealingBlockSeq+epoch2Extra) - - // Confirm the second epoch has the second validator in the sealing block - require.Equal(t, secondInstance.Config.ID, latestValidatorID(t, storage)) + chain.acceptNewBlock() } -func TestInstanceNonValidatorBootstraps(t *testing.T) { - // One node is a validator and progresses the chain by building blocks, - // and its weight changes while the chain progresses in 3 different P-chain epoch heights. - // Then, we add another node which is a non-validator. - // The node should bootstrap the chain but without shutting down the non-validator instance, - // and the test should detect the log entry "I am still a non-validator at the tip of the P-chain, skipping role change" - // being printed several times until the non-validator node bootstraps. - // Later on, the non-validator becomes a validator. - const ( - basePChainHeight = uint64(1) - secondEpochP = uint64(100) - thirdEpochP = uint64(200) - joinEpochP = uint64(300) - ) - - var id [20]byte - rand.Read(id[:]) - validatorNodeID := common.NodeID(id[:]) - - // The node that joins later, first as a non-validator and eventually as a validator. - var nv [20]byte - rand.Read(nv[:]) - nonValidatorNodeID := common.NodeID(nv[:]) - - // The lone validator's weight changes at three different P-chain heights, sealing an - // epoch on each change. Because it remains the sole validator throughout, its own - // approval is a quorum and every epoch seals without any other node's participation. - // The last checkpoint (joinEpochP) grows the set to two validators, admitting the peer. - validatorSetsAtHeight := map[uint64]metadata.NodeBLSMappings{ - basePChainHeight: { - {NodeID: id, BLSKey: []byte{0xaa}, Weight: 1}, - }, - secondEpochP: { - {NodeID: id, BLSKey: []byte{0xaa}, Weight: 2}, - }, - thirdEpochP: { - {NodeID: id, BLSKey: []byte{0xaa}, Weight: 3}, - }, - joinEpochP: { - {NodeID: id, BLSKey: []byte{0xaa}, Weight: 3}, - {NodeID: nv, BLSKey: []byte{0xbb}, Weight: 1}, - }, - } +// TestNonValidatorSyncs that a non-validator syncs the chain when added to the network. +func TestNonValidatorSyncs(t *testing.T) { + validator := newBLSMapping(1) + genesisSet := []metadata.NodeBLSMapping{validator} - pChain := newTestPlatformChain(basePChainHeight, validatorSetsAtHeight) - cops := &testCryptoOps{} + pChain := newTestPChain(genesisSet) + chain := newNetwork(t, pChain) + chain.addNode(validator.NodeID[:]) - genesisBlock := &testInnerBlock{Height_: 0, TS: time.Now(), Payload: []byte("genesis")} - - net := newInMemNetwork(t) - t.Cleanup(net.stop) - - // Both storages start with only the genesis block. - storage := newStorageWithGenesis(t, genesisBlock) - storage2 := newStorageWithGenesis(t, genesisBlock) - - validatorInstance := newInstance(t, validatorNodeID, storage, net, pChain, cops, genesisBlock) - nonValidatorInstance := newInstance(t, nonValidatorNodeID, storage2, net, pChain, cops, genesisBlock) - - // Count how many times the non-validator reports that it is still not a validator at the - // tip of the P-chain while it replicates across the sealed epochs. - var stillNonValidatorLogs atomic.Uint64 - // transitioned is closed when the node starts a Simplex epoch, i.e. becomes a validator. - // The node only ever starts an epoch here as part of its non-validator -> validator - // transition. - transitioned := make(chan struct{}) - nonValidatorInstance.Config.Logger.(*testutil.TestLogger).Intercept(func(entry zapcore.Entry) error { - if strings.Contains(entry.Message, "I am still a non-validator at the tip of the P-chain, skipping role change") { - stillNonValidatorLogs.Add(1) - } - if strings.Contains(entry.Message, "Starting Simplex Epoch") { - select { - case <-transitioned: - default: - close(transitioned) - } - } - return nil - }) + chain.acceptNewBlock() - // Only the validator is running at first; it builds and seals the chain on its own. - net.register(validatorNodeID, validatorInstance) - require.NoError(t, validatorInstance.Start(t.Context())) - t.Cleanup(validatorInstance.Stop) - - // Epoch 1: wait until the validator has committed a series of blocks on its own. - waitForNumBlocks(t, storage, 5) // genesis(0) + zero block(1) + a few normal blocks - - // Drive two more epoch transitions by changing the validator's weight. Each change seals - // an epoch (and produces a sealing block) without any other node, since the validator's - // own approval is a quorum of the single-node set. - pChain.advanceTo(secondEpochP) - waitForSealingBlockCount(t, storage, 2) - - pChain.advanceTo(thirdEpochP) - waitForSealingBlockCount(t, storage, 3) - - // Let the third epoch grow a few normal blocks before the non validator joins, so bootstrap has to - // replicate past the sealing blocks and into ordinary blocks. - waitForNumBlocks(t, storage, storage.NumBlocks()+3) - - // The new node joins as a non-validator (it is absent from the validator set at the current - // P-chain tip) and bootstraps the chain from the validator. - net.register(nonValidatorNodeID, nonValidatorInstance) - require.NoError(t, nonValidatorInstance.Start(t.Context())) - t.Cleanup(nonValidatorInstance.Stop) - - // The non-validator replicates every sealed epoch. It stays a non-validator throughout, - // so on each sealing block it logs that it is still a non-validator at the tip. - bootstrapTarget := storage.NumBlocks() - waitForNumBlocks(t, storage2, bootstrapTarget) - - // The "still a non-validator" message was printed several times (once per sealed epoch it - // replicated through) while it caught up. - require.Eventually(t, func() bool { - return stillNonValidatorLogs.Load() >= 3 - }, 20*time.Second, 100*time.Millisecond) - - // Now grow the validator set to include the peer at the P-chain tip. - pChain.advanceTo(joinEpochP) - approval := &common.ValidatorSetApproval{ - NodeID: nv, - PChainHeight: joinEpochP, - AuxInfoDigest: sha256.Sum256(nil), - Signature: []byte{1, 2, 3}, - } - - // With two validators the validator's self-approval is no longer a quorum and the peer is - // still a non-validator, so we inject the peer's approval until the sealing block commits. - // TODO: Implement this capability in production so we won't need to inject approvals in tests. - sealingBlockSeq := waitForSealingBlock(t, validatorInstance, approval, storage.NumBlocks()) - waitForNumBlocks(t, storage2, sealingBlockSeq) - - // Once the non-validator replicates the sealing block that admits it, it detects that it is - // now a validator at the tip and transitions from non-validator to validator. - select { - case <-transitioned: - case <-time.After(20 * time.Second): - t.Fatal("non-validator did not transition to validator") - } - - // The newly promoted validator now participates in extending the chain. - require.Equal(t, nonValidatorInstance.Config.ID, latestValidatorID(t, storage)) - - // With both validators live, the two-validator epoch keeps committing blocks, and both - // nodes replicate them together. This confirms the promoted node contributes to consensus - // rather than merely tracking the chain. - const twoValidatorExtra = uint64(3) - extendedTarget := sealingBlockSeq + twoValidatorExtra - waitForNumBlocks(t, storage, extendedTarget) - waitForNumBlocks(t, storage2, extendedTarget) + nonValidator := newBLSMapping(2) + chain.addNode(nonValidator.NodeID[:]) } -func TestInstanceRestartAcrossEpochs(t *testing.T) { - // Restart a single validator at three different points in its lifecycle so that, - // on each (re)start, constructEpochAndValidatorSet takes a different branch of - // its switch: - // - // - Cold boot, ledger holds only the genesis (non-Simplex) block -> "genesis" branch. - // - Restart when the tip is a sealing block -> "sealing block at tip" branch. - // - Restart mid-epoch, when the tip is an ordinary Simplex block -> "sealing block in storage" branch. - // - const basePChainHeight = uint64(1) - - var id [20]byte - rand.Read(id[:]) - nodeID := common.NodeID(id[:]) - - validatorSetsAtHeight := map[uint64]metadata.NodeBLSMappings{ - basePChainHeight: { - {NodeID: id, BLSKey: []byte{0xaa}, Weight: 1}, - }, - } - - pChain := newTestPlatformChain(basePChainHeight, validatorSetsAtHeight) - cops := &testCryptoOps{} - genesisBlock := &testInnerBlock{Height_: 0, TS: time.Now(), Payload: []byte("genesis")} - - net := newInMemNetwork(t) - t.Cleanup(net.stop) - - storage := newStorageWithGenesis(t, genesisBlock) - - vm := newTestVM() - - const ( - logEpochFromGenesis = "Determined epoch and validator set from genesis (ledger holds only non-Simplex blocks)" - logEpochFromSealingTip = "Determined epoch and validator set from sealing block at tip" - logEpochFromSealingStorage = "Determined epoch and validator set from sealing block in storage" - ) - - // lastEpochBranch holds the full debug message constructEpochAndValidatorSet - // logs, identifying which branch of its switch the latest (re)start took. It is - // written synchronously during Start, but also from the epoch-change goroutine, - // so an atomic guards it. - var lastEpochBranch atomic.Pointer[string] +// TestNonValidator_BecomesValidator tests that an upcoming validator becomes a validator +// when an epoch change they are following is sealed. It does so by checking they participated in signing +// NOTE: This test will not pass until approval dissemination happens from non-validators. +// Equivalent test as the previous TestInstanceMixedNodeType. +func TestNonValidator_BecomesValidator(t *testing.T) { + validator := newBLSMapping(1) - // start (re)creates an instance over the same storage/network/VM. The log - // interceptor, installed before Start, records which branch startup took. - start := func() *Instance { - inst := newInstanceWithVM(t, nodeID, storage, net, pChain, cops, genesisBlock, vm) - inst.Config.Logger.(*testutil.TestLogger).Intercept(func(entry zapcore.Entry) error { - switch entry.Message { - case logEpochFromGenesis, logEpochFromSealingTip, logEpochFromSealingStorage: - msg := entry.Message - lastEpochBranch.Store(&msg) - } - return nil - }) - net.register(nodeID, inst) - require.NoError(t, inst.Start(t.Context())) - return inst - } + genesisSet := []metadata.NodeBLSMapping{validator} - // Pause block production before the node even starts: only protocol blocks (the - // zero block, the epoch transition and its sealing block) get built, and the - // chain stops at the sealing block since no ordinary block can be built on top. - vm.pause() + pChain := newTestPChain(genesisSet) + chain := newNetwork(t, pChain) + chain.addNode(validator.NodeID[:]) - // --- Case 1: cold boot, ledger holds only the genesis block. --- - inst := start() - require.Equal(t, logEpochFromGenesis, *lastEpochBranch.Load()) + chain.acceptNewBlock() - // --- Case 2: restart when the tip is a sealing block. --- - // countSealingBlocks == 2: the zero block plus the sealing block of the initial - // epoch transition. With the VM paused, that sealing block stays the tip. - waitForSealingBlockCount(t, storage, 2) - requireTipIsSealing(t, storage, true) + // The non-validator node syncs the accepted blocks and then contributes to the next blocks + upcomingValidator := newBLSMapping(2) + chain.addNode(upcomingValidator.NodeID[:]) - inst.Stop() - inst = start() - require.Equal(t, logEpochFromSealingTip, *lastEpochBranch.Load()) + // initiate an epoch change + newValidatorSet := metadata.NodeBLSMappings{validator, upcomingValidator} + pChain.setValidatorSetAt(10, newValidatorSet) + pChain.advanceHeight(10) - // --- Case 3: restart mid-epoch, tip is an ordinary Simplex block. --- - // Resume production; the node extends the new epoch with ordinary blocks. - vm.resume() - waitForNumBlocks(t, storage, storage.NumBlocks()+3) - requireTipIsSealing(t, storage, false) - - inst.Stop() - inst = start() - t.Cleanup(inst.Stop) - require.Equal(t, logEpochFromSealingStorage, *lastEpochBranch.Load()) - - // The restarted node keeps extending the chain. - waitForNumBlocks(t, storage, storage.NumBlocks()+2) -} - -func TestParseBlockSizeMatchesBytes(t *testing.T) { - // Case 1: Bytes() first, Size() second, size returns the cached length. - pb := &ParsedBlock{ - StateMachineBlock: metadata.StateMachineBlock{ - Metadata: metadata.StateMachineMetadata{ - SimplexProtocolMetadata: common.ProtocolMetadata{ - Version: 1, - Prev: common.Digest{}, - Round: 1, - Epoch: 4, - Seq: 2, - }, - SimplexBlacklist: common.Blacklist{ - Updates: common.BlacklistUpdates{{NodeIndex: 1, Type: 1}}, - NodeCount: 2, - }, - PChainHeight: 6, - }, - InnerBlock: &testInnerBlock{ - Height_: 7, - TS: time.UnixMilli(8), - Payload: []byte("payload"), - }, - }, - } - bytes := pb.Bytes() - require.Equal(t, len(bytes), pb.Size()) - - // Case 2: Size() first on a non serialized block. it will - // compute the size and match a later Byte() call. - pb2 := &ParsedBlock{ - StateMachineBlock: metadata.StateMachineBlock{ - Metadata: metadata.StateMachineMetadata{ - SimplexProtocolMetadata: common.ProtocolMetadata{ - Version: 1, - Prev: common.Digest{}, - Round: 1, - Epoch: 4, - Seq: 2, - }, - SimplexBlacklist: common.Blacklist{ - Updates: common.BlacklistUpdates{{NodeIndex: 1, Type: 1}}, - NodeCount: 2, - }, - PChainHeight: 6, - }, - InnerBlock: &testInnerBlock{ - Height_: 9, - TS: time.UnixMilli(10), - Payload: []byte("other payload"), - }, - }, - } - size := pb2.Size() - require.NotZero(t, size) - bytes2 := pb2.Bytes() - require.Equal(t, len(bytes2), size) - - // case 3: cincurrent Size() calls on a block that was never serialized. - // the goroutines rase to compute the size, the lock must make this - // safe and every call must return the correct value - - pb3 := &ParsedBlock{ - StateMachineBlock: metadata.StateMachineBlock{ - Metadata: metadata.StateMachineMetadata{ - SimplexProtocolMetadata: common.ProtocolMetadata{ - Version: 1, - Prev: common.Digest{}, - Round: 1, - Epoch: 4, - Seq: 2, - }, - SimplexBlacklist: common.Blacklist{ - Updates: common.BlacklistUpdates{{NodeIndex: 1, Type: 1}}, - NodeCount: 2, - }, - PChainHeight: 6, - }, - InnerBlock: &testInnerBlock{ - Height_: 11, - TS: time.UnixMilli(12), - Payload: []byte("concurrent"), - }, - }, - } - var wg sync.WaitGroup - sizes := make([]int, 4) - for i := range sizes { - wg.Add(1) - go func() { - defer wg.Done() - sizes[i] = pb3.Size() - }() - } - wg.Wait() - bytes3 := pb3.Bytes() - for _, size := range sizes { - require.Equal(t, len(bytes3), size) - } -} - -func TestInstanceDoubleStartFails(t *testing.T) { - const basePChainHeight = uint64(1) - - var id [20]byte - rand.Read(id[:]) - nodeID := common.NodeID(id[:]) - - // Single-validator set including this node, so Start brings up a validator epoch. - validatorSetsAtHeight := map[uint64]metadata.NodeBLSMappings{ - basePChainHeight: { - {NodeID: id, BLSKey: []byte{0xaa}, Weight: 1}, - }, - } - - pChain := newTestPlatformChain(basePChainHeight, validatorSetsAtHeight) - cops := &testCryptoOps{} - genesisBlock := &testInnerBlock{Height_: 0, TS: time.Now(), Payload: []byte("genesis")} - - net := newInMemNetwork(t) - t.Cleanup(net.stop) - - storage := newStorageWithGenesis(t, genesisBlock) - - inst := newInstance(t, nodeID, storage, net, pChain, cops, genesisBlock) - - require.NoError(t, inst.Start(t.Context())) - t.Cleanup(inst.Stop) - - require.ErrorContains(t, inst.Start(t.Context()), "instance already started") -} - -// requireTipIsSealing asserts whether the last block in storage is a sealing block. -func requireTipIsSealing(t *testing.T, storage *MockStorage, want bool) { - t.Helper() - num := storage.NumBlocks() - require.Positive(t, num) - block, ok := storage.blockAt(num - 1) - require.True(t, ok) - require.Equal(t, want, block.SealingBlockInfo() != nil) -} - -// countSealingBlocks returns the number of sealing blocks (blocks carrying a -// BlockValidationDescriptor) currently in storage. -func countSealingBlocks(t *testing.T, storage *MockStorage) int { - t.Helper() - count := 0 - num := storage.NumBlocks() - for seq := uint64(0); seq < num; seq++ { - block, ok := storage.blockAt(seq) - if !ok { - continue - } - if block.SealingBlockInfo() != nil { - count++ - } - } - return count -} - -// waitForSealingBlockCount waits until storage holds at least target sealing blocks. -func waitForSealingBlockCount(t *testing.T, storage *MockStorage, target int) { - t.Helper() - require.Eventually(t, func() bool { - return countSealingBlocks(t, storage) >= target - }, 20*time.Second, 100*time.Millisecond) -} - -// newStorageWithGenesis returns storage holding only the genesis block, the ledger every node -// here starts from. -func newStorageWithGenesis(t *testing.T, genesisBlock *testInnerBlock) *MockStorage { - t.Helper() - storage := NewMockStorage(t) - genesis := &ParsedBlock{StateMachineBlock: metadata.StateMachineBlock{InnerBlock: genesisBlock}} - require.NoError(t, storage.Index(context.Background(), genesis, common.Finalization{})) - return storage -} - -// newInstance builds an Instance sharing the common test dependencies but with its own ID, -// storage and VM. -func newInstance(t *testing.T, nodeID common.NodeID, storage *MockStorage, net *inMemNetwork, pChain *testPlatformChain, cops *testCryptoOps, genesisBlock *testInnerBlock) *Instance { - return newInstanceWithVM(t, nodeID, storage, net, pChain, cops, genesisBlock, newTestVM()) -} - -// newInstanceWithVM is like newInstance but uses a caller-supplied VM, so a test -// can share one controllable VM across restarts of the same node. -func newInstanceWithVM(t *testing.T, nodeID common.NodeID, storage *MockStorage, net *inMemNetwork, pChain *testPlatformChain, cops *testCryptoOps, genesisBlock *testInnerBlock, vm VM) *Instance { - comm := &networkSender{net: net, self: nodeID} - config := Config{ - Logger: testutil.MakeLogger(t, int(nodeID[0])), - ID: nodeID, - VM: vm, - Storage: storage, - Sender: comm, - Broadcaster: comm, - PlatformChain: pChain, - CryptoOps: cops, - LastNonSimplexInnerBlock: genesisBlock, - WalCreator: storage.CreateWAL, - ParameterConfig: ParameterConfig{ - MaxNetworkDelay: 500 * time.Millisecond, - MaxRoundWindow: 100, - WALMaxEntryCount: 1024, - }, - } - return NewInstance(config) + sealingBlock := chain.waitUntilSealingBlock() + assertExpectedNodeIds(t, sealingBlock.SealingBlockInfo().ValidatorSet, newValidatorSet.NodeIDs()) } -func latestValidatorID(t *testing.T, storage *MockStorage) common.NodeID { - t.Helper() - num := storage.NumBlocks() - // Iterate backwards and find the latest sealing block (a block with a block validation descriptor) - for seq := int64(num) - 1; seq >= 0; seq-- { - block, ok := storage.blockAt(uint64(seq)) - if !ok { - continue - } - if info := block.SealingBlockInfo(); info != nil { - return info.ValidatorSet[len(info.ValidatorSet)-1].Id - } - } - t.Fatalf("no block with a BlockValidationDescriptor found in storage") - return nil -} +// TestValidator_ValidatorSetNotChanged tests that a pchain height increase +// that does not have a unique validator set, does not create a new epoch +func TestValidator_ValidatorSetNotChanged(t *testing.T) { + validator := newBLSMapping(1) -// waitForNumBlocks waits until the given storage has at least targetHeight blocks. -func waitForNumBlocks(t *testing.T, storage *MockStorage, targetHeight uint64) { - t.Helper() - require.Eventually(t, func() bool { - return storage.NumBlocks() >= targetHeight - }, 20*time.Second, 100*time.Millisecond) -} + genesisSet := []metadata.NodeBLSMapping{validator} -// waitForSealingBlock waits until a sealing block (a block carrying a BlockValidationDescriptor with the new weight) -// is committed at or after fromSeq. It periodically injects approvals into the given instance. -// Returns the seq of the sealing block. -func waitForSealingBlock(t *testing.T, inst *Instance, approval *common.ValidatorSetApproval, fromSeq uint64) uint64 { - t.Helper() - var result uint64 - storage := inst.Config.Storage.(*MockStorage) - require.Eventually(t, func() bool { - inst.lock.Lock() - msm := inst.msm - inst.lock.Unlock() - if msm != nil { - require.NoError(t, msm.HandleApproval(approval, 1)) - } + pChain := newTestPChain(genesisSet) + chain := newNetwork(t, pChain) + chain.addNode(validator.NodeID[:]) - num := storage.NumBlocks() - for seq := fromSeq; seq < num; seq++ { - block, ok := storage.blockAt(seq) - if !ok { - continue - } - if block.SealingBlockInfo() != nil { - result = seq - return true - } - } - return false - }, 20*time.Second, 100*time.Millisecond) - return result -} + firstBlock := chain.acceptNewBlock() -type testInnerBlock struct { - Height_ uint64 - TS time.Time - Payload []byte -} + // initiate an epoch change + pChain.setValidatorSetAt(10, []metadata.NodeBLSMapping{validator}) + pChain.advanceHeight(10) -func (b *testInnerBlock) Bytes() []byte { - out := make([]byte, 16, 16+len(b.Payload)) - binary.BigEndian.PutUint64(out[0:8], b.Height_) - binary.BigEndian.PutUint64(out[8:16], uint64(b.TS.UnixMilli())) - out = append(out, b.Payload...) - return out -} + // potential time to propose blocks (if any) + time.Sleep(3 * time.Second) -func (b *testInnerBlock) Digest() [32]byte { - bytes := b.Bytes() - return sha256.Sum256(bytes) + secondBlock := chain.acceptNewBlock() + require.Equal(t, uint64(1), secondBlock.BlockHeader().Epoch) + require.Equal(t, firstBlock.BlockHeader().Seq+1, secondBlock.BlockHeader().Seq) } -func (b *testInnerBlock) Height() uint64 { return b.Height_ } -func (b *testInnerBlock) Timestamp() time.Time { return b.TS } -func (b *testInnerBlock) Verify(context.Context, uint64) error { return nil } +// TestValidator_ValidatorSetDecreased tests that an epoch with two validators +// is reduced to one, when the pchain height notes a validator is leaving. +func TestValidator_ValidatorSetDecreased(t *testing.T) { + validator := newBLSMapping(1) + leavingValidator := newBLSMapping(2) -func parseTestInnerBlock(buff []byte) (*testInnerBlock, error) { - b := &testInnerBlock{} - b.Height_ = binary.BigEndian.Uint64(buff[0:8]) - b.TS = time.UnixMilli(int64(binary.BigEndian.Uint64(buff[8:16]))) - b.Payload = append([]byte(nil), buff[16:]...) - return b, nil -} + genesisSet := []metadata.NodeBLSMapping{validator, leavingValidator} -type testVM struct { - nextHeight atomic.Uint64 - // When paused, the VM behaves as a chain with no pending transactions: - // WaitForPendingBlock and BuildBlock block until their context expires, so the - // epoch stops producing ordinary blocks. The epoch-transition and sealing - // machinery, which builds its block once the inner build times out, still runs — - // so pausing before an epoch change leaves the sealing block at the tip with - // nothing built on top. Lets a test pin the chain tip without touching storage. - paused atomic.Bool -} - -func newTestVM() *testVM { - vm := &testVM{} - vm.nextHeight.Store(1) // the genesis inner block is height 0 - return vm -} + pChain := newTestPChain(genesisSet) + chain := newNetwork(t, pChain) + wg := sync.WaitGroup{} -func (vm *testVM) pause() { vm.paused.Store(true) } -func (vm *testVM) resume() { vm.paused.Store(false) } - -func (vm *testVM) BuildBlock(ctx context.Context, _ uint64) (avalanchego.VMBlock, error) { - if vm.paused.Load() { - <-ctx.Done() // let the caller's impatient build time out - return nil, ctx.Err() - } - h := vm.nextHeight.Add(1) - 1 - payload := make([]byte, 8) - binary.BigEndian.PutUint64(payload, h) - return &testInnerBlock{Height_: h, TS: time.Now(), Payload: payload}, nil -} - -func (vm *testVM) WaitForPendingBlock(ctx context.Context) { - if vm.paused.Load() { - <-ctx.Done() // no pending block while paused - return - } - select { - case <-ctx.Done(): - case <-time.After(100 * time.Millisecond): - } -} - -func (vm *testVM) ParseBlock(_ context.Context, b []byte) (avalanchego.VMBlock, error) { - return parseTestInnerBlock(b) -} - -func (vm *testVM) ComputeICMEpoch(input metadata.ICMEpochInput) metadata.ICMEpochInfo { - // ACP-181-style transition (mirrors the msm test helper). - var zero metadata.ICMEpochInfo - if input.ParentEpoch == zero { - return metadata.ICMEpochInfo{ - PChainEpochHeight: input.ParentPChainHeight, - EpochNumber: 1, - EpochStartTime: uint64(input.ParentTimestamp.Unix()), - } - } - endTime := time.Unix(int64(input.ParentEpoch.EpochStartTime), 0).Add(time.Second) - if input.ParentTimestamp.Before(endTime) { - return input.ParentEpoch - } - return metadata.ICMEpochInfo{ - PChainEpochHeight: input.ParentPChainHeight, - EpochNumber: input.ParentEpoch.EpochNumber + 1, - EpochStartTime: uint64(input.ParentTimestamp.Unix()), - } -} - -type testPlatformChain struct { - baseHeight uint64 - validatorSetAtHeight map[uint64]metadata.NodeBLSMappings // height --> validator set - lock sync.Mutex - cond *sync.Cond - height uint64 -} - -func newTestPlatformChain(baseHeight uint64, validatorSetsAtHeight map[uint64]metadata.NodeBLSMappings) *testPlatformChain { - pc := &testPlatformChain{ - baseHeight: baseHeight, - validatorSetAtHeight: validatorSetsAtHeight, - height: baseHeight, - } - pc.cond = sync.NewCond(&pc.lock) - return pc -} - -func (pc *testPlatformChain) advanceTo(h uint64) { - pc.lock.Lock() - defer pc.lock.Unlock() - pc.height = h - pc.cond.Broadcast() // wake any WaitForProgress waiters -} - -func (pc *testPlatformChain) currentHeight() uint64 { - pc.lock.Lock() - defer pc.lock.Unlock() - return pc.height -} - -func (pc *testPlatformChain) validatorSet(height uint64) metadata.NodeBLSMappings { - heights := make([]uint64, 0, len(pc.validatorSetAtHeight)) - for h := range pc.validatorSetAtHeight { - heights = append(heights, h) - } - sort.Slice(heights, func(i, j int) bool { return heights[i] < heights[j] }) - - var lastCheckpoint uint64 - for _, h := range heights { - if h > height { - break - } - lastCheckpoint = h - } - // Return a copy instead of the original slice so the reference won't be used in other goroutines concurrently. - // Since we allocate a nil slice, a new underlying array is allocated and the copy is safe to use concurrently. - src := pc.validatorSetAtHeight[lastCheckpoint] - return append(metadata.NodeBLSMappings(nil), src...) -} - -func (pc *testPlatformChain) GetValidatorSet(height uint64) (metadata.NodeBLSMappings, error) { - return pc.validatorSet(height), nil -} - -func (pc *testPlatformChain) GenesisValidatorSet() metadata.NodeBLSMappings { - return pc.validatorSet(pc.baseHeight) -} - -func (pc *testPlatformChain) GetMinimumHeight() uint64 { - return pc.currentHeight() -} - -func (pc *testPlatformChain) GetCurrentHeight() uint64 { - return pc.currentHeight() -} - -func (pc *testPlatformChain) WaitForProgress(ctx context.Context, pChainHeight uint64) error { - stop := pc.signalWhenContextFinished(ctx) - defer stop() - - pc.lock.Lock() - defer pc.lock.Unlock() - for pc.height == pChainHeight { - if err := ctx.Err(); err != nil { - return err - } - pc.cond.Wait() - } - return nil -} - -func (pc *testPlatformChain) signalWhenContextFinished(ctx context.Context) func() bool { - stop := context.AfterFunc(ctx, func() { - pc.lock.Lock() - defer pc.lock.Unlock() - pc.cond.Broadcast() + wg.Go(func() { + // add node is a synchronous call. + chain.addNode(validator.NodeID[:]) }) - return stop -} - -func (pc *testPlatformChain) LastNonSimplexBlockPChainHeight() uint64 { - return pc.baseHeight -} - -type testCryptoOps struct{} - -func (c *testCryptoOps) Sign(message []byte) ([]byte, error) { - // A deterministic, non-empty placeholder signature. - d := sha256.Sum256(message) - return d[:], nil -} - -func (c *testCryptoOps) AggregateKeys(keys ...[]byte) ([]byte, error) { - var out []byte - for _, k := range keys { - out = append(out, k...) - } - return out, nil -} - -func (c *testCryptoOps) VerifySignature(_ []byte, _ []byte, _ []byte) error { - return nil -} - -func (c *testCryptoOps) CreateSignatureAggregator(nodes []common.Node) common.SignatureAggregator { - return &testutil.TestSignatureAggregator{N: len(nodes)} -} - -func (c *testCryptoOps) DeserializeQuorumCertificate(bytes []byte) (common.QuorumCertificate, error) { - var qc []common.Signature - if _, err := asn1.Unmarshal(bytes, &qc); err != nil { - return nil, err - } - return testutil.TestQC(qc), nil -} - -type MockStorage struct { - t *testing.T - *testutil.InMemStorage - - snapLock sync.Mutex - blocks map[uint64]storedBlock -} - -type storedBlock struct { - rawBlock []byte - fin common.Finalization -} - -func NewMockStorage(t *testing.T) *MockStorage { - return &MockStorage{ - t: t, - InMemStorage: testutil.NewInMemStorage(), - blocks: make(map[uint64]storedBlock), - } -} - -func (m *MockStorage) Index(ctx context.Context, block common.VerifiedBlock, certificate common.Finalization) error { - // We serialized the block so that the original reference isn't shared with other goroutines that may concurrently mutate it. - encoded := block.Bytes() - seq := m.InMemStorage.NumBlocks() - m.snapLock.Lock() - m.blocks[seq] = storedBlock{rawBlock: encoded, fin: certificate} - m.snapLock.Unlock() - return m.InMemStorage.Index(ctx, block, certificate) -} - -func (m *MockStorage) GetBlock(seq uint64) (metadata.StateMachineBlock, *common.Finalization, error) { - _, f, err := m.InMemStorage.Retrieve(seq) - if err != nil { - return metadata.StateMachineBlock{}, nil, err - } - sb, ok := m.blockAt(seq) - if !ok { - return metadata.StateMachineBlock{}, nil, fmt.Errorf("no snapshot for seq %d", seq) - } - return sb, &f, nil -} - -// blockAt reconstructs an independent copy of the block at seq from its -// stored bytes. Test-only readers use it instead of GetBlock so they never touch -// the instance's live block objects (whose canoto digest cache the instance keeps -// mutating). -func (m *MockStorage) blockAt(seq uint64) (metadata.StateMachineBlock, bool) { - m.snapLock.Lock() - sb, ok := m.blocks[seq] - m.snapLock.Unlock() - if !ok { - return metadata.StateMachineBlock{}, false - } - return m.parseStored(sb.rawBlock), true -} - -func (m *MockStorage) parseStored(encoded []byte) metadata.StateMachineBlock { - raw := &metadata.RawBlock{} - require.NoError(m.t, raw.UnmarshalCanoto(encoded)) - var inner avalanchego.VMBlock - if len(raw.InnerBlockBytes) > 0 { - parsed, err := parseTestInnerBlock(raw.InnerBlockBytes) - require.NoError(m.t, err) - inner = parsed - } - return metadata.StateMachineBlock{InnerBlock: inner, Metadata: raw.Metadata} -} -func (m *MockStorage) CreateWAL() (wal.DeletableWAL, error) { - return testutil.NewTestWAL(m.t), nil -} - -// --------------------------------------------------------------------------- -// inMemNetwork: Routes messages between Instances. -// Delivery happens on a per-node goroutine rather than inline in Send, -// due to locking. -// --------------------------------------------------------------------------- + chain.addNode(leavingValidator.NodeID[:]) -type netMsg struct { - from common.NodeID - msg *common.Message -} + // all nodes have synced the first every simplex block + // TODO: we should initialize our node so that it already stores the first ever simplex block + // otherwise, we rely on all nodes to be connected in order to finalize it. + wg.Wait() -type netNode struct { - inst *Instance - // in is a buffered inbox drained by the delivery goroutine. The channel itself - // signals that work is available, so no separate wake signal is needed. Sends - // never block (see enqueue); on the rare chance the buffer fills, a dropped - // message costs at most an empty round the epoch recovers from. - in chan netMsg - done chan struct{} - stopped chan struct{} -} + block := chain.acceptNewBlock() + require.Equal(t, uint64(2), block.BlockHeader().Round) -type inMemNetwork struct { - t *testing.T - lock sync.Mutex - nodes map[string]*netNode -} + // initiate an epoch change + newValidatorSet := metadata.NodeBLSMappings{validator} + pChain.setValidatorSetAt(10, newValidatorSet) + pChain.advanceHeight(10) -func newInMemNetwork(t *testing.T) *inMemNetwork { - return &inMemNetwork{t: t, nodes: make(map[string]*netNode)} + sealing := chain.waitUntilSealingBlock() + assertExpectedNodeIds(t, sealing.SealingBlockInfo().ValidatorSet, newValidatorSet.NodeIDs()) } -// register wires inst into the network and starts delivering messages to it. -// Messages that arrive before the epoch exists are dropped by the instance's -// nil-epoch guard, which at worst costs a few empty rounds the epoch recovers from. -func (n *inMemNetwork) register(id common.NodeID, inst *Instance) { - node := &netNode{ - inst: inst, - in: make(chan netMsg, 1024), - done: make(chan struct{}), - stopped: make(chan struct{}), - } - n.lock.Lock() - defer n.lock.Unlock() - - // If an instance was previously registered under this id (e.g. a restart replacing - // the node), stop its delivery goroutine before swapping in the new one. - old := n.nodes[string(id)] - if old != nil { - close(old.done) - <-old.stopped - } - - n.nodes[string(id)] = node - - go n.deliver(node) +// TestNonValidator_StaysNonValidator ensures that a non-validator does not restart when it is processing +// previous epoch changes. +// Equivalent to TestInstanceNonValidatorBootstraps +func TestNonValidator_StaysNonValidator(t *testing.T) { + // case 1: epoch change is not highest and we are NOT in the validator set + // case 1: epoch change is not highest, and we are in the validator set + // case 1: epoch change is highest, and we are in the validator set + // case 1: epoch change is highest, and we are NOT in the validator set } -func (n *inMemNetwork) stop() { - n.lock.Lock() - nodes := make([]*netNode, 0, len(n.nodes)) - for _, node := range n.nodes { - nodes = append(nodes, node) - } - n.nodes = make(map[string]*netNode) - n.lock.Unlock() - for _, node := range nodes { - close(node.done) - <-node.stopped - } -} +// TestInstanceValidatorSkipsAnEpoch tests that a validator stops and starts being a validator +// It boots up as a non-validator then syncs to the highest epoch where it is a validator, +// then it is no longer a validator, and finally it is +// Equivalent to: TestInstanceValidatorSkipsAnEpoch +func TestInstanceValidatorSkipsAnEpoch(t *testing.T) { + validator := newBLSMapping(1) -// registeredIDs returns the nodes currently wired into the network. Broadcasters go through it -// rather than reading the map directly, since nodes register while others are already running. -func (n *inMemNetwork) registeredIDs() []common.NodeID { - n.lock.Lock() - defer n.lock.Unlock() - ids := make([]common.NodeID, 0, len(n.nodes)) - for _, node := range n.nodes { - ids = append(ids, node.inst.Config.ID) - } - return ids -} + genesisSet := []metadata.NodeBLSMapping{validator} -func (n *inMemNetwork) enqueue(dest common.NodeID, m netMsg) { - n.lock.Lock() - node := n.nodes[string(dest)] - n.lock.Unlock() - if node == nil { - // Destination not registered; drop. This only happens before an instance - // is registered, never mid-run. - return - } - select { - case node.in <- m: - default: - // Never block the sender (Send runs under the epoch lock). A dropped message - // costs at most an empty round the epoch recovers from. - } -} + pChain := newTestPChain(genesisSet) + chain := newNetwork(t, pChain) + chain.addNode(validator.NodeID[:]) -func (n *inMemNetwork) deliver(node *netNode) { - defer close(node.stopped) - for { - select { - case <-node.done: - return - case m := <-node.in: - n.dispatch(node.inst, m) - } - } -} + // The non-validator node syncs the accepted blocks and then contributes to the next blocks + onOffValidator := newBLSMapping(2) + chain.addNode(onOffValidator.NodeID[:]) -func (n *inMemNetwork) dispatch(inst *Instance, m netMsg) { - if err := inst.HandleMessage(m.msg, m.from); err != nil { - n.t.Logf("HandleMessage from %x failed: %v", m.from, err) - } -} + // initiate an epoch change + newValidatorSet := metadata.NodeBLSMappings{validator, onOffValidator} + pChain.setValidatorSetAt(10, newValidatorSet) + pChain.advanceHeight(10) -// toRawBlock re-encodes a verified block into the wire RawBlock the receiving -// instance parses in HandleBlockMessage. -func toRawBlock(t *testing.T, vb common.VerifiedBlock) *metadata.RawBlock { - bytes := vb.Bytes() - raw := &metadata.RawBlock{} - require.NoError(t, raw.UnmarshalCanoto(bytes)) - return raw -} + sealingBlock := chain.waitUntilSealingBlock() + assertExpectedNodeIds(t, sealingBlock.SealingBlockInfo().ValidatorSet, newValidatorSet.NodeIDs()) -// reparseBlock reconstructs an independent *ParsedBlock from a verified block's -// wire bytes. Each call yields a fresh object sharing no pointers with the -// sender's live block, so that remaining references to the sender's block don't race with the receiver. -func reparseBlock(t *testing.T, vb common.VerifiedBlock) *ParsedBlock { - raw := toRawBlock(t, vb) - var inner avalanchego.VMBlock - if len(raw.InnerBlockBytes) > 0 { - parsed, err := parseTestInnerBlock(raw.InnerBlockBytes) - require.NoError(t, err) - inner = parsed - } - return &ParsedBlock{ - StateMachineBlock: metadata.StateMachineBlock{InnerBlock: inner, Metadata: raw.Metadata}, - } -} + newValidatorSet = metadata.NodeBLSMappings{validator} + pChain.setValidatorSetAt(20, newValidatorSet) + pChain.advanceHeight(20) + sealingBlock = chain.waitUntilSealingBlock() + assertExpectedNodeIds(t, sealingBlock.SealingBlockInfo().ValidatorSet, newValidatorSet.NodeIDs()) -type networkSender struct { - net *inMemNetwork - self common.NodeID -} + // accept a new block to ensure both nodes are still syncing the chain + chain.acceptNewBlock() -func (s *networkSender) Broadcast(msg *common.Message) { - for _, dest := range s.net.registeredIDs() { - s.Send(msg, dest) - } -} + // initiate the final epoch change + newValidatorSet = metadata.NodeBLSMappings{validator, onOffValidator} + pChain.setValidatorSetAt(30, newValidatorSet) + pChain.advanceHeight(30) -func (s *networkSender) Send(msg *common.Message, dest common.NodeID) { - if bytes.Equal(s.self, dest) { - // Do not send to myself - return - } - m := s.createIngressMessage(msg) - s.net.enqueue(dest, m) + sealingBlock = chain.waitUntilSealingBlock() + assertExpectedNodeIds(t, sealingBlock.SealingBlockInfo().ValidatorSet, newValidatorSet.NodeIDs()) } -// CreateIngressMessage translates a message into the form the receiving instance expects on the wire. -// For example, a VerifiedBlockMessage is re-encoded as a BlockMessage with a RawBlock. -// A VerifiedReplicationResponse is re-encoded as a ReplicationResponse with independent copies of the carried blocks. -func (s *networkSender) createIngressMessage(msg *common.Message) netMsg { - m := netMsg{from: s.self} - switch { - case msg.VerifiedBlockMessage != nil: - m.msg = &common.Message{ - BlockMessage: &common.BlockMessage{ - Vote: msg.VerifiedBlockMessage.Vote, - Block: reparseBlock(s.net.t, msg.VerifiedBlockMessage.VerifiedBlock), - }, - } - case msg.VerifiedReplicationResponse != nil: - m.msg = &common.Message{ReplicationResponse: toReplicationResponse(s.net.t, msg.VerifiedReplicationResponse)} - default: - m.msg = msg - } - return m +// TestInstanceTransitionFromSnowman tests that when passed in a config that indicates we are transitioning from snowman, +// the instance still properly starts and indexes future blocks. +// This test, alongside the util tests cover the previous TestInstanceRestartAcrossEpochs +func TestInstanceTransitionFromSnowman(t *testing.T) { + // validator := newBLSMapping(1) } -// toReplicationResponse translates a VerifiedReplicationResponse (the sender's -// internal form) into the ReplicationResponse a receiver handles on the wire, -// mirroring testutil.TestComm. Each carried block is reconstructed as an -// independent copy so the delivery goroutine never touches the sender's live -// block object (whose canoto digest cache the sender keeps mutating). -func toReplicationResponse(t *testing.T, vrr *common.VerifiedReplicationResponse) *common.ReplicationResponse { - data := make([]common.QuorumRound, 0, len(vrr.Data)) - for _, vqr := range vrr.Data { - data = append(data, verifiedQuorumRoundToQuorumRound(t, vqr)) - } - resp := &common.ReplicationResponse{Data: data} - if vrr.LatestRound != nil { - qr := verifiedQuorumRoundToQuorumRound(t, *vrr.LatestRound) - resp.LatestRound = &qr - } - if vrr.LatestFinalizedSeq != nil { - qr := verifiedQuorumRoundToQuorumRound(t, *vrr.LatestFinalizedSeq) - resp.LatestSeq = &qr - } - return resp -} +func TestInstanceDoubleStartFails(t *testing.T) { + validator := newBLSMapping(1) + genesisSet := []metadata.NodeBLSMapping{validator} -func verifiedQuorumRoundToQuorumRound(t *testing.T, vqr common.VerifiedQuorumRound) common.QuorumRound { - qr := common.QuorumRound{ - Notarization: vqr.Notarization, - Finalization: vqr.Finalization, - EmptyNotarization: vqr.EmptyNotarization, - } - if vqr.VerifiedBlock != nil { - qr.Block = reparseBlock(t, vqr.VerifiedBlock) - } - return qr + pChain := newTestPChain(genesisSet) + chain := newNetwork(t, pChain) + node := chain.addNode(validator.NodeID[:]) + require.ErrorIs(t, node.inst.Start(t.Context()), errAlreadyStarted) } diff --git a/instance_testhelpers_test.go b/instance_testhelpers_test.go new file mode 100644 index 00000000..5f022668 --- /dev/null +++ b/instance_testhelpers_test.go @@ -0,0 +1,670 @@ +// Copyright (C) 2019-2025, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package simplex + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/asn1" + "encoding/binary" + "fmt" + "sync" + "testing" + "time" + + "github.com/ava-labs/simplex/avalanchego" + "github.com/ava-labs/simplex/common" + metadata "github.com/ava-labs/simplex/msm" + "github.com/ava-labs/simplex/simplex" + "github.com/ava-labs/simplex/testutil" + "github.com/ava-labs/simplex/wal" + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +type testCryptoOps struct{} + +func (c *testCryptoOps) Sign(message []byte) ([]byte, error) { + // A deterministic, non-empty placeholder signature. + d := sha256.Sum256(message) + return d[:], nil +} + +func (c *testCryptoOps) AggregateKeys(keys ...[]byte) ([]byte, error) { + var out []byte + for _, k := range keys { + out = append(out, k...) + } + return out, nil +} + +func (c *testCryptoOps) VerifySignature(_ []byte, _ []byte, _ []byte) error { + return nil +} + +func (c *testCryptoOps) CreateSignatureAggregator(nodes []common.Node) common.SignatureAggregator { + return &testutil.TestSignatureAggregator{N: len(nodes)} +} + +func (c *testCryptoOps) DeserializeQuorumCertificate(bytes []byte) (common.QuorumCertificate, error) { + var qc []common.Signature + if _, err := asn1.Unmarshal(bytes, &qc); err != nil { + return nil, err + } + return testutil.TestQC(qc), nil +} + +type MockStorage struct { + t *testing.T + *testutil.InMemStorage + bd *testInnerBlockDeserializer + + snapLock sync.Mutex + blocks map[uint64]storedBlock +} + +type storedBlock struct { + rawBlock []byte + fin common.Finalization +} + +func NewMockStorageWithGenesis(t *testing.T, bd *testInnerBlockDeserializer) *MockStorage { + s := &MockStorage{ + t: t, + InMemStorage: testutil.NewInMemStorage(), + blocks: make(map[uint64]storedBlock), + bd: bd, + } + + genesis := &ParsedBlock{StateMachineBlock: metadata.StateMachineBlock{InnerBlock: genesisBlock}} + require.NoError(t, s.Index(context.Background(), genesis, common.Finalization{})) + return s +} + +func (m *MockStorage) Index(ctx context.Context, block common.VerifiedBlock, certificate common.Finalization) error { + // We serialized the block so that the original reference isn't shared with other goroutines that may concurrently mutate it. + encoded := block.Bytes() + seq := m.InMemStorage.NumBlocks() + m.snapLock.Lock() + m.blocks[seq] = storedBlock{rawBlock: encoded, fin: certificate} + m.snapLock.Unlock() + return m.InMemStorage.Index(ctx, block, certificate) +} + +func (m *MockStorage) GetBlock(seq uint64) (metadata.StateMachineBlock, *common.Finalization, error) { + _, f, err := m.InMemStorage.Retrieve(seq) + if err != nil { + return metadata.StateMachineBlock{}, nil, err + } + sb, ok := m.blockAt(seq) + if !ok { + return metadata.StateMachineBlock{}, nil, fmt.Errorf("no snapshot for seq %d", seq) + } + return sb, &f, nil +} + +// blockAt reconstructs an independent copy of the block at seq from its +// stored bytes. Test-only readers use it instead of GetBlock so they never touch +// the instance's live block objects (whose canoto digest cache the instance keeps +// mutating). +func (m *MockStorage) blockAt(seq uint64) (metadata.StateMachineBlock, bool) { + m.snapLock.Lock() + sb, ok := m.blocks[seq] + m.snapLock.Unlock() + if !ok { + return metadata.StateMachineBlock{}, false + } + return m.parseStored(sb.rawBlock), true +} + +func (m *MockStorage) parseStored(encoded []byte) metadata.StateMachineBlock { + raw := &metadata.RawBlock{} + require.NoError(m.t, raw.UnmarshalCanoto(encoded)) + var inner avalanchego.VMBlock + if len(raw.InnerBlockBytes) > 0 { + parsed, err := m.bd.ParseBlock(context.Background(), raw.InnerBlockBytes) + require.NoError(m.t, err) + inner = parsed + } + return metadata.StateMachineBlock{InnerBlock: inner, Metadata: raw.Metadata} +} + +type testInnerBlockDeserializer struct{} + +func (ibd *testInnerBlockDeserializer) ParseBlock(_ context.Context, buff []byte) (avalanchego.VMBlock, error) { + b := &testInnerBlock{} + b.Height_ = binary.BigEndian.Uint64(buff[0:8]) + b.TS = time.UnixMilli(int64(binary.BigEndian.Uint64(buff[8:16]))) + b.Payload = append([]byte(nil), buff[16:]...) + return b, nil +} + +type testInnerBlock struct { + Height_ uint64 + TS time.Time + Payload []byte +} + +func (b *testInnerBlock) Bytes() []byte { + out := make([]byte, 16, 16+len(b.Payload)) + binary.BigEndian.PutUint64(out[0:8], b.Height_) + binary.BigEndian.PutUint64(out[8:16], uint64(b.TS.UnixMilli())) + out = append(out, b.Payload...) + return out +} + +func (b *testInnerBlock) Digest() [32]byte { + bytes := b.Bytes() + return sha256.Sum256(bytes) +} + +func (b *testInnerBlock) Height() uint64 { return b.Height_ } +func (b *testInnerBlock) Timestamp() time.Time { return b.TS } +func (b *testInnerBlock) Verify(context.Context, uint64) error { return nil } + +type instanceComm struct { + c *network + // id is the node this comm belongs to, reported as the sender of every message it sends. + id common.NodeID +} + +func newInstanceComm(c *network, id common.NodeID) *instanceComm { + return &instanceComm{c: c, id: id} +} + +func (c *instanceComm) Send(msg *common.Message, destination common.NodeID) { + // loop through all nodes in chain directly send to handle message directly via but do it in a separate go routine + for _, n := range c.c.nodesSnapshot() { + if !bytes.Equal(n.id, destination) { + continue + } + + go func(dst *Instance) { + require.NotNil(c.c.t, dst, "node %x was sent a message before it was created", destination) + require.NoError(c.c.t, dst.HandleMessage(translateOutgoingToIncomingMessage(c.c.t, msg), c.id)) + }(n.inst) + return + } +} + +func (c *instanceComm) Broadcast(msg *common.Message) { + // send to every node in the chain but ourselves, each on its own go routine + for _, n := range c.c.nodesSnapshot() { + if bytes.Equal(n.id, c.id) { + continue + } + + go func(dst *Instance) { + require.NotNil(c.c.t, dst, "node %x was sent a message before it was created", n.id) + require.NoError(c.c.t, dst.HandleMessage(translateOutgoingToIncomingMessage(c.c.t, msg), c.id)) + }(n.inst) + } +} + +// translateOutgoingToIncomingMessage converts the verified message types an instance +// sends into the wire types a receiver handles, like testutil.TestComm. Each carried +// block is re-parsed into a fresh ParsedBlock because HandleMessage mutates the block +// it receives, so recipients cannot share the sender's live block. +func translateOutgoingToIncomingMessage(t *testing.T, msg *common.Message) *common.Message { + switch { + case msg.VerifiedBlockMessage != nil: + return &common.Message{ + BlockMessage: &common.BlockMessage{ + Vote: msg.VerifiedBlockMessage.Vote, + Block: reparseBlock(t, msg.VerifiedBlockMessage.VerifiedBlock), + }, + } + case msg.VerifiedReplicationResponse != nil: + vrr := msg.VerifiedReplicationResponse + data := make([]common.QuorumRound, 0, len(vrr.Data)) + for _, vqr := range vrr.Data { + data = append(data, verifiedQuorumRoundToQuorumRound(t, vqr)) + } + resp := &common.ReplicationResponse{Data: data} + if vrr.LatestRound != nil { + qr := verifiedQuorumRoundToQuorumRound(t, *vrr.LatestRound) + resp.LatestRound = &qr + } + if vrr.LatestFinalizedSeq != nil { + qr := verifiedQuorumRoundToQuorumRound(t, *vrr.LatestFinalizedSeq) + resp.LatestSeq = &qr + } + return &common.Message{ReplicationResponse: resp} + default: + return msg + } +} + +func verifiedQuorumRoundToQuorumRound(t *testing.T, vqr common.VerifiedQuorumRound) common.QuorumRound { + qr := common.QuorumRound{ + Notarization: vqr.Notarization, + Finalization: vqr.Finalization, + EmptyNotarization: vqr.EmptyNotarization, + } + if vqr.VerifiedBlock != nil { + qr.Block = reparseBlock(t, vqr.VerifiedBlock) + } + return qr +} + +// reparseBlock rebuilds an independent ParsedBlock from a verified block's wire bytes. +// The zero block has no inner block, so its inner bytes stay empty. +func reparseBlock(t *testing.T, vb common.VerifiedBlock) *ParsedBlock { + var rawBlock metadata.RawBlock + require.NoError(t, rawBlock.UnmarshalCanoto(vb.Bytes())) + + var inner avalanchego.VMBlock + if len(rawBlock.InnerBlockBytes) > 0 { + bd := &testInnerBlockDeserializer{} + parsed, err := bd.ParseBlock(context.Background(), rawBlock.InnerBlockBytes) + require.NoError(t, err) + inner = parsed + } + return &ParsedBlock{ + StateMachineBlock: metadata.StateMachineBlock{InnerBlock: inner, Metadata: rawBlock.Metadata}, + } +} + +// pendingBlockSignal broadcasts to every waiter by closing the current channel and +// replacing it with a fresh one for the next generation of waiters. +type pendingBlockSignal struct { + lock sync.Mutex + ch chan struct{} +} + +func newPendingBlockSignal() *pendingBlockSignal { + return &pendingBlockSignal{ch: make(chan struct{})} +} + +// wait returns when the signal is broadcast or ctx is cancelled. +func (s *pendingBlockSignal) wait(ctx context.Context) { + s.lock.Lock() + ch := s.ch + s.lock.Unlock() + + select { + case <-ch: + case <-ctx.Done(): + } +} + +// broadcast wakes every current waiter. +func (s *pendingBlockSignal) broadcast() { + s.lock.Lock() + close(s.ch) + s.ch = make(chan struct{}) + s.lock.Unlock() +} + +// blockBuilderVM builds an inner block only when the test triggers one on the block builder, so +// the chain grows one block per index call. +type blockBuilderVM struct { + bb *testutil.TestControlledBlockBuilder + storage *MockStorage + pending *pendingBlockSignal +} + +func newBlockBuilderVM(bb *testutil.TestControlledBlockBuilder, storage *MockStorage, pending *pendingBlockSignal) *blockBuilderVM { + return &blockBuilderVM{bb: bb, storage: storage, pending: pending} +} + +func (vm *blockBuilderVM) BuildBlock(ctx context.Context, pChainHeight uint64) (avalanchego.VMBlock, error) { + // The builder gates when a block is built; the block it returns is not an inner block, so + // it is thrown away. + if _, ok := vm.bb.BuildBlock(ctx, common.ProtocolMetadata{}, common.Blacklist{}); !ok { + return nil, ctx.Err() + } + + // the inner height is the seq of the block being built, which is how many blocks the node + // has committed so far + height := vm.storage.NumBlocks() + payload := make([]byte, 8) + binary.BigEndian.PutUint64(payload, height) + return &testInnerBlock{Height_: height, TS: time.Now(), Payload: payload}, nil +} + +// WaitForPendingBlock returns when index broadcasts that a block is being created, +// or when ctx is cancelled. +func (vm *blockBuilderVM) WaitForPendingBlock(ctx context.Context) { + vm.pending.wait(ctx) +} + +type node struct { + t *testing.T + id common.NodeID + vm *blockBuilderVM + inst *Instance + storage *MockStorage + comm *instanceComm +} + +// restart stops the node, and starts it again from a fresh instance +// keeping same storage and id +func (n *node) restart() { + n.inst.Stop() + + prevConfig := n.inst.Config + instance := NewInstance(prevConfig) + n.inst = instance + require.NoError(n.t, n.inst.Start(n.t.Context())) +} + +// noopICMTransition keeps every block in the same ICM epoch, so an epoch only ever changes +// because the validator set did. +func noopICMTransition(_ metadata.ICMEpochInput) metadata.ICMEpochInfo { + return metadata.ICMEpochInfo{} +} + +const genesisPChainHeight uint64 = 0 + +var genesisBlock = &testInnerBlock{Height_: genesisPChainHeight, TS: time.Now(), Payload: []byte("genesis")} +var paramConfig = ParameterConfig{ + MaxNetworkDelay: 500 * time.Millisecond, + MaxRoundWindow: 100, + WALMaxEntryCount: 1024, +} + +type testPlatformChain struct { + genesisHeight uint64 // genesis height is the height of the pchain the genesis validator set lives + + // lock guards the sets, which the running instances read while a test installs new ones. + lock sync.Mutex + // validatorSetAtHeight maps a P-chain height to the validator set in force from it on. + validatorSetAtHeight map[uint64]metadata.NodeBLSMappings + + height uint64 + // heightChanged is closed and replaced on every advanceHeight, waking waiters + // so they re-check the height. + heightChanged chan struct{} +} + +// newTestPChain returns a P-chain holding only the genesis validator set, in force from +// genesisPChainHeight on. +func newTestPChain(genesisSet metadata.NodeBLSMappings) *testPlatformChain { + return &testPlatformChain{ + genesisHeight: genesisPChainHeight, + validatorSetAtHeight: map[uint64]metadata.NodeBLSMappings{ + genesisPChainHeight: genesisSet, + }, + height: genesisPChainHeight, + heightChanged: make(chan struct{}), + } +} + +func (pc *testPlatformChain) currentHeight() uint64 { + pc.lock.Lock() + defer pc.lock.Unlock() + return pc.height +} + +func (pc *testPlatformChain) GetValidatorSet(height uint64) (metadata.NodeBLSMappings, error) { + pc.lock.Lock() + defer pc.lock.Unlock() + + set, ok := pc.validatorSetAtHeight[height] + if !ok { + return nil, fmt.Errorf("no validator set at %d", height) + } + return set, nil +} + +func (pc *testPlatformChain) GenesisValidatorSet() metadata.NodeBLSMappings { + pc.lock.Lock() + defer pc.lock.Unlock() + + return pc.validatorSetAtHeight[pc.genesisHeight] +} + +func (pc *testPlatformChain) GetMinimumHeight() uint64 { + return pc.currentHeight() +} + +func (pc *testPlatformChain) GetCurrentHeight() uint64 { + return pc.currentHeight() +} + +// WaitForProgress blocks until the context is cancelled or the P-chain height +// has increased past pChainHeight. +func (pc *testPlatformChain) WaitForProgress(ctx context.Context, pChainHeight uint64) error { + for { + pc.lock.Lock() + if pc.height > pChainHeight { + pc.lock.Unlock() + return nil + } + ch := pc.heightChanged + pc.lock.Unlock() + + select { + case <-ch: + case <-ctx.Done(): + return ctx.Err() + } + } +} + +func (pc *testPlatformChain) setValidatorSetAt(height uint64, validatorSet metadata.NodeBLSMappings) { + pc.lock.Lock() + defer pc.lock.Unlock() + + pc.validatorSetAtHeight[height] = validatorSet +} + +// advanceHeight bumps the P-chain height and wakes every WaitForProgress waiter. +func (pc *testPlatformChain) advanceHeight(height uint64) { + pc.lock.Lock() + defer pc.lock.Unlock() + + if height <= pc.height { + panic("smaller height") + } + pc.height = height + close(pc.heightChanged) + pc.heightChanged = make(chan struct{}) +} + +func (pc *testPlatformChain) LastNonSimplexBlockPChainHeight() uint64 { + return pc.genesisHeight +} + +type network struct { + t *testing.T + + pChain *testPlatformChain + seq uint64 + epoch uint64 + + // pending wakes every VM blocked in WaitForPendingBlock when index creates a block. + pending *pendingBlockSignal + + validatorSets map[uint64]common.Nodes // epoch -> sorted validators + + // lock guards nodes, which comm goroutines read while addNode appends. + lock sync.Mutex + nodes []node +} + +func (n *network) nodesSnapshot() []node { + n.lock.Lock() + defer n.lock.Unlock() + return append([]node(nil), n.nodes...) +} + +func newNetwork(t *testing.T, pChain *testPlatformChain) *network { + validatorSets := make(map[uint64]common.Nodes) + genesisNodes := pChain.GenesisValidatorSet().Nodes() + common.SortNodes(genesisNodes) + validatorSets[1] = genesisNodes + + return &network{ + t: t, + pChain: pChain, + pending: newPendingBlockSignal(), + validatorSets: validatorSets, + + // Genesis at seq 0. Then first simplex block is built automatically + // without a build block notification + seq: 2, + epoch: 1, + } +} + +// addNode adds a node to the network and blocks until it catches up with the latest tip +func (n *network) addNode(id common.NodeID) *node { + // ensure a unique id + for _, node := range n.nodes { + require.NotEqual(n.t, node.id, id) + } + + comm := newInstanceComm(n, id) + bd := &testInnerBlockDeserializer{} + storage := NewMockStorageWithGenesis(n.t, bd) + + vm := newBlockBuilderVM(testutil.NewTestControlledBlockBuilder(n.t), storage, n.pending) + wc := &walCreator{t: n.t} + instance := NewInstance(Config{ + LastNonSimplexInnerBlock: genesisBlock, + ParameterConfig: paramConfig, + PlatformChain: n.pChain, + Broadcaster: comm, + Sender: comm, + CryptoOps: &testCryptoOps{}, + WalCreator: wc.createWAL, + Storage: storage, + // the first byte of the node id labels the node's log records + Logger: testutil.MakeLogger(n.t, int(id[0])), + WALs: nil, + VM: vm, + ICMETransition: noopICMTransition, + BlockDeserializer: bd, + ID: id, + }) + + node := node{ + t: n.t, + id: id, + storage: storage, + comm: comm, + vm: vm, + inst: instance, + } + + n.lock.Lock() + n.nodes = append(n.nodes, node) + n.lock.Unlock() + + ctx, cancel := context.WithCancel(context.Background()) + n.t.Cleanup(cancel) + + require.NoError(n.t, node.inst.Start(ctx)) + n.t.Cleanup(node.inst.Stop) + + node.storage.WaitForBlockCommit(n.seq - 1) + + instance.Config.Logger.Debug("Added a node to the test network", zap.Uint64("Seq", n.seq), zap.Uint64("num block", node.storage.NumBlocks())) + return &node +} + +// acceptNewBlock blocks until every node has accepted a newly indexed block. +func (n *network) acceptNewBlock() common.VerifiedBlock { + nodes, ok := n.validatorSets[n.epoch] + require.True(n.t, ok, fmt.Sprintf("epoch is not set epoch: %d. trying to index seq: %d", n.epoch, n.seq)) + + // no nodes have indexed this sequence yet + for _, node := range n.nodes { + node.storage.EnsureNoBlockCommit(n.t, n.seq) + } + + leaderID := simplex.LeaderForRound(nodes.NodeIDs(), n.seq) + for _, node := range n.nodes { + if bytes.Equal(node.id, leaderID) { + node.vm.bb.TriggerNewBlock() + } + } + + // wake every VM blocked in WaitForPendingBlock + n.pending.broadcast() + + var block common.VerifiedBlock + for _, node := range n.nodes { + committedBlock := node.storage.WaitForBlockCommit(n.seq) + if block == nil { + block = committedBlock + } else { + require.Equal(n.t, block.Bytes(), committedBlock.Bytes()) + } + } + + require.Equal(n.t, block.BlockHeader().Seq, n.seq) + n.seq++ + + // check if its a sealing + if block.SealingBlockInfo() != nil { + n.epoch = n.seq + newValidatorSet := block.SealingBlockInfo().ValidatorSet + common.SortNodes(newValidatorSet) + n.validatorSets[n.epoch] = newValidatorSet + } + + return block +} + +// waitUntilSealingBlock waits until every node commits the block at the current seq, +// repeating until that block is a sealing block. It then advances the network into +// the new epoch and returns the sealing block. +// This is useful for when we are transitioning epochs because blocks will be built impatiently +// without a notification from the mempool. +func (n *network) waitUntilSealingBlock() common.VerifiedBlock { + for { + var block common.VerifiedBlock + for _, node := range n.nodes { + committedBlock := node.storage.WaitForBlockCommit(n.seq) + if block == nil { + block = committedBlock + } else { + require.Equal(n.t, block.Bytes(), committedBlock.Bytes()) + } + } + + require.Equal(n.t, block.BlockHeader().Seq, n.seq) + n.seq++ + + if block.SealingBlockInfo() == nil { + continue + } + + // add the validator set to the networks memory for block building + n.epoch = n.seq + newValidatorSet := block.SealingBlockInfo().ValidatorSet + common.SortNodes(newValidatorSet) + n.validatorSets[n.epoch] = newValidatorSet + return block + } +} + +// newBLSMapping creates a mapping with a nodeID, BLSKey and Weight with a given [id]. +// id is passed as an int for consistent logs between runs. +func newBLSMapping(id int) metadata.NodeBLSMapping { + avaID := [20]byte{byte(id)} + + return metadata.NodeBLSMapping{ + NodeID: avalanchego.NodeID(avaID), + BLSKey: []byte{avaID[0], byte(id + 1)}, + Weight: 1, + } +} + +// assertExpectedNodeIds asserts the validator set contains exactly the expected node IDs. +func assertExpectedNodeIds(t *testing.T, validatorSet common.Nodes, expected []common.NodeID) { + require.ElementsMatch(t, expected, validatorSet.NodeIDs()) +} + +type walCreator struct { + t *testing.T +} + +func (w *walCreator) createWAL() (wal.DeletableWAL, error) { + return testutil.NewTestWAL(w.t), nil +} diff --git a/msm/approvals.go b/msm/approvals.go index d2a53674..877fc5ca 100644 --- a/msm/approvals.go +++ b/msm/approvals.go @@ -159,7 +159,7 @@ func (as *ApprovalStore) checkApprovalSignature(approval *common.ValidatorSetApp } func (as *ApprovalStore) approvalExistsAndUpToDate(approval *common.ValidatorSetApproval, timestamp uint64) bool { - if as.approvalsByNodes[avalanchego.NodeID(approval.NodeID)] == nil { + if as.approvalsByNodes[approval.NodeID] == nil { return false } @@ -168,7 +168,7 @@ func (as *ApprovalStore) approvalExistsAndUpToDate(approval *common.ValidatorSet auxInfoDigest: approval.AuxInfoDigest, } - existingApproval := as.approvalsByNodes[avalanchego.NodeID(approval.NodeID)][key] + existingApproval := as.approvalsByNodes[approval.NodeID][key] if existingApproval == nil { return false } diff --git a/msm/auxiliary.canoto.go b/msm/auxiliary.canoto.go new file mode 100644 index 00000000..65899377 --- /dev/null +++ b/msm/auxiliary.canoto.go @@ -0,0 +1,257 @@ +// Code generated by canoto. DO NOT EDIT. +// versions: +// canoto v0.19.0 +// source: auxiliary.go + +package metadata + +import ( + "io" + "reflect" + "sync/atomic" + + "github.com/StephenButtolph/canoto" +) + +// Ensure that the generated code is compatible with the library version. +const ( + _ uint = canoto.VersionCompatibility - 1 + _ uint = 1 - canoto.VersionCompatibility +) + +// Ensure that unused imports do not error +var _ = io.ErrUnexpectedEOF + +const ( + canotoNumber_AuxiliaryInfoBatch__data = 1 + canotoNumber_AuxiliaryInfoBatch__PrevAuxInfoSeq = 2 + + canotoTag_AuxiliaryInfoBatch__data = "\x0a" // canoto.Tag(canotoNumber_AuxiliaryInfoBatch__data, canoto.Len) + canotoTag_AuxiliaryInfoBatch__PrevAuxInfoSeq = "\x10" // canoto.Tag(canotoNumber_AuxiliaryInfoBatch__PrevAuxInfoSeq, canoto.Varint) +) + +type canotoData_AuxiliaryInfoBatch struct { + size uint64 +} + +// CanotoSpec returns the specification of this canoto message. +func (*AuxiliaryInfoBatch) CanotoSpec(types ...reflect.Type) *canoto.Spec { + types = append(types, reflect.TypeFor[AuxiliaryInfoBatch]()) + var zero AuxiliaryInfoBatch + s := &canoto.Spec{ + Name: "AuxiliaryInfoBatch", + Fields: []canoto.FieldType{ + canoto.FieldTypeFromField( + /*type inference:*/ (canoto.MakeEntryNilPointer(zero.data)), + /*FieldNumber: */ canotoNumber_AuxiliaryInfoBatch__data, + /*Name: */ "data", + /*FixedLength: */ 0, + /*Repeated: */ true, + /*OneOf: */ "", + /*Pointer: */ false, + /*types: */ types, + ), + { + FieldNumber: canotoNumber_AuxiliaryInfoBatch__PrevAuxInfoSeq, + Name: "PrevAuxInfoSeq", + OneOf: "", + TypeUint: canoto.SizeOf(zero.PrevAuxInfoSeq), + }, + }, + } + s.CalculateCanotoCache() + return s +} + +// UnmarshalCanoto unmarshals a Canoto-encoded byte slice into the struct. +// +// During parsing, the canoto cache is saved. +func (c *AuxiliaryInfoBatch) UnmarshalCanoto(bytes []byte) error { + r := canoto.Reader{ + B: bytes, + } + return c.UnmarshalCanotoFrom(r) +} + +// UnmarshalCanotoFrom populates the struct from a [canoto.Reader]. Most users +// should just use UnmarshalCanoto. +// +// During parsing, the canoto cache is saved. +// +// This function enables configuration of reader options. +func (c *AuxiliaryInfoBatch) UnmarshalCanotoFrom(r canoto.Reader) error { + // Zero the struct before unmarshaling. + *c = AuxiliaryInfoBatch{} + atomic.StoreUint64(&c.canotoData.size, uint64(len(r.B))) + + var minField uint32 + for canoto.HasNext(&r) { + field, wireType, err := canoto.ReadTag(&r) + if err != nil { + return err + } + if field < minField { + return canoto.ErrInvalidFieldOrder + } + + switch field { + case canotoNumber_AuxiliaryInfoBatch__data: + if wireType != canoto.Len { + return canoto.ErrUnexpectedWireType + } + + // Read the first entry manually because the tag is already + // stripped. + originalUnsafe := r.Unsafe + r.Unsafe = true + var msgBytes []byte + if err := canoto.ReadBytes(&r, &msgBytes); err != nil { + return err + } + r.Unsafe = originalUnsafe + + // Count the number of additional entries after the first entry. + countMinus1, err := canoto.CountBytes(r.B, canotoTag_AuxiliaryInfoBatch__data) + if err != nil { + return err + } + + c.data = canoto.MakeSlice(c.data, countMinus1+1) + field := c.data + additionalField := field[1:] + if len(msgBytes) != 0 { + remainingBytes := r.B + r.B = msgBytes + if err := (&field[0]).UnmarshalCanotoFrom(r); err != nil { + return err + } + r.B = remainingBytes + } + + // Read the rest of the entries, stripping the tag each time. + for i := range additionalField { + r.B = r.B[len(canotoTag_AuxiliaryInfoBatch__data):] + r.Unsafe = true + if err := canoto.ReadBytes(&r, &msgBytes); err != nil { + return err + } + r.Unsafe = originalUnsafe + if len(msgBytes) == 0 { + continue + } + + remainingBytes := r.B + r.B = msgBytes + if err := (&additionalField[i]).UnmarshalCanotoFrom(r); err != nil { + return err + } + r.B = remainingBytes + } + case canotoNumber_AuxiliaryInfoBatch__PrevAuxInfoSeq: + if wireType != canoto.Varint { + return canoto.ErrUnexpectedWireType + } + + if err := canoto.ReadUint(&r, &c.PrevAuxInfoSeq); err != nil { + return err + } + if canoto.IsZero(c.PrevAuxInfoSeq) { + return canoto.ErrZeroValue + } + default: + return canoto.ErrUnknownField + } + + minField = field + 1 + } + return nil +} + +// ValidCanoto validates that the struct can be correctly marshaled into the +// Canoto format. +// +// Specifically, ValidCanoto ensures: +// 1. All OneOfs are specified at most once. +// 2. All strings are valid utf-8. +// 3. All custom fields are ValidCanoto. +func (c *AuxiliaryInfoBatch) ValidCanoto() bool { + { + field := c.data + for i := range field { + if !(&field[i]).ValidCanoto() { + return false + } + } + } + return true +} + +// CalculateCanotoCache populates size and OneOf caches based on the current +// values in the struct. +// +// It is not safe to copy this struct concurrently. +func (c *AuxiliaryInfoBatch) CalculateCanotoCache() { + var size uint64 + { + field := c.data + for i := range field { + (&field[i]).CalculateCanotoCache() + fieldSize := (&field[i]).CachedCanotoSize() + size += uint64(len(canotoTag_AuxiliaryInfoBatch__data)) + canoto.SizeUint(fieldSize) + fieldSize + } + } + if !canoto.IsZero(c.PrevAuxInfoSeq) { + size += uint64(len(canotoTag_AuxiliaryInfoBatch__PrevAuxInfoSeq)) + canoto.SizeUint(c.PrevAuxInfoSeq) + } + atomic.StoreUint64(&c.canotoData.size, size) +} + +// CachedCanotoSize returns the previously calculated size of the Canoto +// representation from CalculateCanotoCache. +// +// If CalculateCanotoCache has not yet been called, it will return 0. +// +// If the struct has been modified since the last call to CalculateCanotoCache, +// the returned size may be incorrect. +func (c *AuxiliaryInfoBatch) CachedCanotoSize() uint64 { + return atomic.LoadUint64(&c.canotoData.size) +} + +// MarshalCanoto returns the Canoto representation of this struct. +// +// It is assumed that this struct is ValidCanoto. +// +// It is not safe to copy this struct concurrently. +func (c *AuxiliaryInfoBatch) MarshalCanoto() []byte { + c.CalculateCanotoCache() + w := canoto.Writer{ + B: make([]byte, 0, c.CachedCanotoSize()), + } + w = c.MarshalCanotoInto(w) + return w.B +} + +// MarshalCanotoInto writes the struct into a [canoto.Writer] and returns the +// resulting [canoto.Writer]. Most users should just use MarshalCanoto. +// +// It is assumed that CalculateCanotoCache has been called since the last +// modification to this struct. +// +// It is assumed that this struct is ValidCanoto. +// +// It is not safe to copy this struct concurrently. +func (c *AuxiliaryInfoBatch) MarshalCanotoInto(w canoto.Writer) canoto.Writer { + { + field := c.data + for i := range field { + canoto.Append(&w, canotoTag_AuxiliaryInfoBatch__data) + canoto.AppendUint(&w, (&field[i]).CachedCanotoSize()) + w = (&field[i]).MarshalCanotoInto(w) + } + } + if !canoto.IsZero(c.PrevAuxInfoSeq) { + canoto.Append(&w, canotoTag_AuxiliaryInfoBatch__PrevAuxInfoSeq) + canoto.AppendUint(&w, c.PrevAuxInfoSeq) + } + return w +} diff --git a/msm/auxiliary.go b/msm/auxiliary.go new file mode 100644 index 00000000..607469c2 --- /dev/null +++ b/msm/auxiliary.go @@ -0,0 +1,177 @@ +package metadata + +import ( + "bytes" + "crypto/sha256" + "fmt" + "slices" + "sync" + + "github.com/ava-labs/simplex/avalanchego" + "github.com/ava-labs/simplex/common" +) + +//go:generate go run github.com/StephenButtolph/canoto/canoto auxiliary.go + +// AuxiliaryInfoBatch is a batch of AuxiliaryInfos to be included in a block +type AuxiliaryInfoBatch struct { + // data is how we expect the order being appended. 0 index is appended first, then data[len()-1] is last + data []common.AuxiliaryInfo `canoto:"repeated value,1"` + // PrevAuxInfoSeq is a sequence number that applications can use to find previous AuxiliaryInfo in the chain. + // It is zero if this is the first AuxiliaryInfoBatch for this epoch. + PrevAuxInfoSeq uint64 `canoto:"uint,2"` + + canotoData canotoData_AuxiliaryInfoBatch +} + +func (ai *AuxiliaryInfoBatch) IsZero() bool { + var zero AuxiliaryInfoBatch + return ai.Equal(&zero) +} + +func (ai *AuxiliaryInfoBatch) Equal(a *AuxiliaryInfoBatch) bool { + if ai == nil { + return a == nil + } + if a == nil { + return false + } + if ai.PrevAuxInfoSeq != a.PrevAuxInfoSeq || len(ai.data) != len(a.data) { + return false + } + for i := range ai.data { + if ai.data[i].Version != a.data[i].Version || !bytes.Equal(ai.data[i].Data, a.data[i].Data) { + return false + } + } + return true +} + +type AuxInfoHistory struct { + Data [][]byte + LastSeq uint64 + OldestVersionID common.VersionID // oldest version id in the histories data, or DefaultVersionID if no history +} + +func (aih *AuxInfoHistory) LastHistoryDigest() [32]byte { + if len(aih.Data) == 0 { + return [32]byte{} + } + last := aih.Data[len(aih.Data)-1] + return sha256.Sum256(last) +} + +// GetAuxiliaryHistory traverses backwards starting from the given block and returns the AuxInfoHistory of all blocks in the chain. +// It returns the collected auxiliary info ordered from oldest to newest, the sequence of the newest block it was collected from, +// and the version ID of the oldest non-empty auxiliary info entry (or defaultVersionID if there was none). +// blockSeq must be the sequence of the given block. +func GetAuxiliaryHistory(block *StateMachineBlock, blockSeq uint64, getBlock BlockRetriever, defaultVersionID common.VersionID) (AuxInfoHistory, error) { + var lastSeq *uint64 + var history [][]byte + var versionID = defaultVersionID + + // We traverse the chain of blocks backwards in the following manner: + // (1) Every block that doesn't have an AuxiliaryInfoBatch, its parents also do not have one. + // (2) Every block that has an AuxiliaryInfoBatch, its descendants also have one. + // (3) A block's AuxiliaryInfoBatch may have no entries, but its PrevAuxInfoSeq field must point + // to a block whose AuxiliaryInfoBatch isn't nil and has non-empty entries. + // (4) When a block with an empty batch is built on a parent block that has an AuxiliaryInfoBatch, + // if its parent block's batch has non-empty entries, then the block's PrevAuxInfoSeq points to its parent block. + // Else, its parent block's batch is also empty, then the block's PrevAuxInfoSeq is inherited from its parent block's PrevAuxInfoSeq. + + batch := block.Metadata.AuxiliaryInfoBatch + currentSeq := blockSeq + for batch != nil { + // Entries within a batch are ordered oldest to newest, so iterate newest-first: + // the full history is reversed once traversal completes. + for i := len(batch.data) - 1; i >= 0; i-- { + entry := batch.data[i] + if len(entry.Data) == 0 { + continue + } + history = append(history, entry.Data) + if lastSeq == nil { + lastSeq = new(uint64) + *lastSeq = currentSeq + } + versionID = entry.Version + } + if batch.PrevAuxInfoSeq == 0 { + // This is the first auxiliary info of the epoch, we can stop traversing back. + break + } + currentSeq = batch.PrevAuxInfoSeq + prevBlock, _, err := getBlock(batch.PrevAuxInfoSeq, [32]byte{}) + if err != nil { + return AuxInfoHistory{}, fmt.Errorf("%w: at sequence %d: %w", errAuxInfoBlockRetrieval, batch.PrevAuxInfoSeq, err) + } + batch = prevBlock.Metadata.AuxiliaryInfoBatch + } + + if lastSeq == nil { + lastSeq = new(uint64) + *lastSeq = 0 + } + + // Reverse so the history is ordered from oldest to newest. + slices.Reverse(history) + return AuxInfoHistory{Data: history, LastSeq: *lastSeq, OldestVersionID: versionID}, nil +} + +// auxInfoStore stores auxiliary info that has been received but not yet included in blocks +type auxInfoStore struct { + app AuxiliaryInfoGenVerifier + + lock sync.Mutex + sentInfo map[avalanchego.NodeID]common.AuxiliaryInfo +} + +func newAuxInfoStore(app AuxiliaryInfoGenVerifier) *auxInfoStore { + return &auxInfoStore{ + app: app, + sentInfo: make(map[avalanchego.NodeID]common.AuxiliaryInfo), + } +} + +func (a *auxInfoStore) HandleAuxiliaryMessage(info common.AuxiliaryInfo, from avalanchego.NodeID) { + a.lock.Lock() + defer a.lock.Unlock() + + // just set the nodes Auxiliary info to the most recent one they sent + a.sentInfo[from] = info +} + +// collectAuxInfo returns the stored entries that are legal appends to the given history. +func (a *auxInfoStore) collectAuxInfo(history AuxInfoHistory, validators NodeBLSMappings) []common.AuxiliaryInfo { + a.lock.Lock() + defer a.lock.Unlock() + + // Iterate in node ID order so the returned entries are deterministic. + nodeIDs := make([]avalanchego.NodeID, 0, len(a.sentInfo)) + for nodeID := range a.sentInfo { + nodeIDs = append(nodeIDs, nodeID) + } + slices.SortFunc(nodeIDs, func(x, y avalanchego.NodeID) int { + return bytes.Compare(x[:], y[:]) + }) + + var legalAppends []common.AuxiliaryInfo + legalHistory := append([][]byte{}, history.Data...) + + for _, nodeID := range nodeIDs { + info := a.sentInfo[nodeID] + if history.OldestVersionID != info.Version { + continue // keep consistent versions throughout epoch transition + } + + if err := a.app.IsLegalAppend(info.Version, validators, legalHistory, info.Data); err != nil { + // we don't remove this info from the mempool. maybe it can be added in a different block + continue + } + + legalAppends = append(legalAppends, info) + legalHistory = append(legalHistory, info.Data) + } + + return legalAppends +} diff --git a/msm/auxiliary_test.go b/msm/auxiliary_test.go new file mode 100644 index 00000000..49860c49 --- /dev/null +++ b/msm/auxiliary_test.go @@ -0,0 +1,425 @@ +package metadata + +import ( + "fmt" + "testing" + + "github.com/ava-labs/simplex/avalanchego" + "github.com/ava-labs/simplex/common" + + "github.com/stretchr/testify/require" +) + +func TestAuxiliaryInfoBatchEqual(t *testing.T) { + for _, tt := range []struct { + name string + a *AuxiliaryInfoBatch + b *AuxiliaryInfoBatch + expected bool + }{ + { + name: "both nil", + a: nil, + b: nil, + expected: true, + }, + { + name: "nil vs non-nil", + a: nil, + b: &AuxiliaryInfoBatch{}, + expected: false, + }, + { + name: "both zero", + a: &AuxiliaryInfoBatch{}, + b: &AuxiliaryInfoBatch{}, + expected: true, + }, + { + name: "equal with data", + a: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{ + {Version: 1, Data: []byte{1, 2, 3}}, + {Version: 2, Data: []byte{4, 5}}, + }, + PrevAuxInfoSeq: 7, + }, + b: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{ + {Version: 1, Data: []byte{1, 2, 3}}, + {Version: 2, Data: []byte{4, 5}}, + }, + PrevAuxInfoSeq: 7, + }, + expected: true, + }, + { + name: "nil data vs empty data", + a: &AuxiliaryInfoBatch{}, + b: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{}, + }, + expected: true, + }, + { + name: "different PrevAuxInfoSeq", + a: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{{Version: 1, Data: []byte{1}}}, + PrevAuxInfoSeq: 1, + }, + b: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{{Version: 1, Data: []byte{1}}}, + PrevAuxInfoSeq: 2, + }, + expected: false, + }, + { + name: "different number of entries", + a: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{{Version: 1, Data: []byte{1}}}, + }, + b: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{ + {Version: 1, Data: []byte{1}}, + {Version: 1, Data: []byte{2}}, + }, + }, + expected: false, + }, + { + name: "different entry version", + a: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{{Version: 1, Data: []byte{1}}}, + }, + b: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{{Version: 2, Data: []byte{1}}}, + }, + expected: false, + }, + { + name: "different entry data", + a: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{{Version: 1, Data: []byte{1}}}, + }, + b: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{{Version: 1, Data: []byte{2}}}, + }, + expected: false, + }, + { + name: "same entries in different order", + a: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{ + {Version: 1, Data: []byte{1}}, + {Version: 2, Data: []byte{2}}, + }, + }, + b: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{ + {Version: 2, Data: []byte{2}}, + {Version: 1, Data: []byte{1}}, + }, + }, + expected: false, + }, + } { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.expected, tt.a.Equal(tt.b)) + require.Equal(t, tt.expected, tt.b.Equal(tt.a)) + }) + } +} + +func TestAuxiliaryInfoBatchIsZero(t *testing.T) { + for _, tt := range []struct { + name string + batch *AuxiliaryInfoBatch + expected bool + }{ + { + name: "zero value", + batch: &AuxiliaryInfoBatch{}, + expected: true, + }, + { + name: "empty data slice", + batch: &AuxiliaryInfoBatch{data: []common.AuxiliaryInfo{}}, + expected: true, + }, + { + name: "non-zero PrevAuxInfoSeq", + batch: &AuxiliaryInfoBatch{PrevAuxInfoSeq: 1}, + expected: false, + }, + { + name: "non-empty data", + batch: &AuxiliaryInfoBatch{data: []common.AuxiliaryInfo{{Version: 1, Data: []byte{1}}}}, + expected: false, + }, + } { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.expected, tt.batch.IsZero()) + }) + } +} + +// batchBlock returns a StateMachineBlock whose metadata carries the given AuxiliaryInfoBatch. +func batchBlock(batch *AuxiliaryInfoBatch) StateMachineBlock { + return StateMachineBlock{ + Metadata: StateMachineMetadata{ + AuxiliaryInfoBatch: batch, + }, + } +} + +// blockRetrieverFromMap returns a BlockRetriever backed by the given seq -> block mapping, +// failing the test if a sequence outside the mapping is requested. +func blockRetrieverFromMap(t *testing.T, blocks map[uint64]StateMachineBlock) BlockRetriever { + return func(seq uint64, _ common.Digest) (StateMachineBlock, *common.Finalization, error) { + block, ok := blocks[seq] + require.True(t, ok, "requested unexpected block at seq %d", seq) + return block, nil, nil + } +} + +func TestGetAuxiliaryHistory(t *testing.T) { + const ( + defaultVersionID = common.VersionID(42) + startSeq = uint64(10) + ) + + for _, tt := range []struct { + name string + // batch of the block traversal starts from + startBatch *AuxiliaryInfoBatch + // batches of ancestor blocks by seq, reachable via PrevAuxInfoSeq links + prevBatches map[uint64]*AuxiliaryInfoBatch + expected AuxInfoHistory + }{ + { + name: "no batch", + startBatch: nil, + expected: AuxInfoHistory{ + LastSeq: 0, + OldestVersionID: defaultVersionID, + }, + }, + { + name: "batch with no entries", + startBatch: &AuxiliaryInfoBatch{}, + expected: AuxInfoHistory{ + LastSeq: 0, + OldestVersionID: defaultVersionID, + }, + }, + { + name: "single batch preserves entry order", + startBatch: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{ + {Version: 1, Data: []byte("a")}, + {Version: 2, Data: []byte("b")}, + }, + }, + expected: AuxInfoHistory{ + Data: [][]byte{[]byte("a"), []byte("b")}, + LastSeq: startSeq, + OldestVersionID: 1, + }, + }, + { + name: "entries with empty data are skipped", + startBatch: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{ + {Version: 1, Data: []byte("a")}, + {Version: 2, Data: nil}, + {Version: 3, Data: []byte("c")}, + }, + }, + expected: AuxInfoHistory{ + Data: [][]byte{[]byte("a"), []byte("c")}, + LastSeq: startSeq, + OldestVersionID: 1, + }, + }, + { + name: "chain of batches ordered oldest to newest", + startBatch: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{{Version: 3, Data: []byte("d")}}, + PrevAuxInfoSeq: 5, + }, + prevBatches: map[uint64]*AuxiliaryInfoBatch{ + 5: { + data: []common.AuxiliaryInfo{ + {Version: 2, Data: []byte("b")}, + {Version: 2, Data: []byte("c")}, + }, + PrevAuxInfoSeq: 3, + }, + 3: { + data: []common.AuxiliaryInfo{{Version: 1, Data: []byte("a")}}, + }, + }, + expected: AuxInfoHistory{ + Data: [][]byte{[]byte("a"), []byte("b"), []byte("c"), []byte("d")}, + LastSeq: startSeq, + OldestVersionID: 1, + }, + }, + { + name: "empty starting batch inherits from ancestors", + startBatch: &AuxiliaryInfoBatch{ + PrevAuxInfoSeq: 4, + }, + prevBatches: map[uint64]*AuxiliaryInfoBatch{ + 4: { + data: []common.AuxiliaryInfo{{Version: 7, Data: []byte("a")}}, + }, + }, + expected: AuxInfoHistory{ + Data: [][]byte{[]byte("a")}, + LastSeq: 4, + OldestVersionID: 7, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + blocks := make(map[uint64]StateMachineBlock, len(tt.prevBatches)) + for seq, batch := range tt.prevBatches { + blocks[seq] = batchBlock(batch) + } + + startBlock := batchBlock(tt.startBatch) + history, err := GetAuxiliaryHistory(&startBlock, startSeq, blockRetrieverFromMap(t, blocks), defaultVersionID) + require.NoError(t, err) + require.Equal(t, tt.expected, history) + }) + } +} + +func TestGetAuxiliaryHistoryRetrievalError(t *testing.T) { + startBlock := batchBlock(&AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{{Version: 1, Data: []byte("a")}}, + PrevAuxInfoSeq: 5, + }) + + getBlock := func(seq uint64, _ common.Digest) (StateMachineBlock, *common.Finalization, error) { + return StateMachineBlock{}, nil, fmt.Errorf("no block at seq %d", seq) + } + + _, err := GetAuxiliaryHistory(&startBlock, 10, getBlock, 0) + require.ErrorIs(t, err, errAuxInfoBlockRetrieval) +} + +type sentAuxInfo struct { + from avalanchego.NodeID + info common.AuxiliaryInfo +} + +func TestCollectAuxInfo(t *testing.T) { + node1 := avalanchego.NodeID{1} + node2 := avalanchego.NodeID{2} + node3 := avalanchego.NodeID{3} + + // voteCountingAuxInfoApp rejects appends whose data is already in the history. + history := AuxInfoHistory{ + Data: [][]byte{[]byte("a")}, + OldestVersionID: 1, + } + + for _, tt := range []struct { + name string + sends []sentAuxInfo + expected []common.AuxiliaryInfo + }{ + { + name: "empty store", + sends: nil, + expected: nil, + }, + { + name: "legal entries returned sorted by node id", + sends: []sentAuxInfo{ + {from: node3, info: common.AuxiliaryInfo{Version: 1, Data: []byte("d")}}, + {from: node1, info: common.AuxiliaryInfo{Version: 1, Data: []byte("b")}}, + {from: node2, info: common.AuxiliaryInfo{Version: 1, Data: []byte("c")}}, + }, + expected: []common.AuxiliaryInfo{ + {Version: 1, Data: []byte("b")}, + {Version: 1, Data: []byte("c")}, + {Version: 1, Data: []byte("d")}, + }, + }, + { + name: "version mismatch filtered", + sends: []sentAuxInfo{ + {from: node1, info: common.AuxiliaryInfo{Version: 2, Data: []byte("b")}}, + }, + expected: nil, + }, + { + name: "inconsistent versions only keep entries matching the history version", + sends: []sentAuxInfo{ + {from: node1, info: common.AuxiliaryInfo{Version: 2, Data: []byte("b")}}, + {from: node2, info: common.AuxiliaryInfo{Version: 1, Data: []byte("c")}}, + {from: node3, info: common.AuxiliaryInfo{Version: 3, Data: []byte("d")}}, + }, + expected: []common.AuxiliaryInfo{ + {Version: 1, Data: []byte("c")}, + }, + }, + { + name: "entries already in history filtered", + sends: []sentAuxInfo{ + {from: node1, info: common.AuxiliaryInfo{Version: 1, Data: []byte("a")}}, + }, + expected: nil, + }, + { + name: "accepted entries extend the history for later entries", + sends: []sentAuxInfo{ + {from: node1, info: common.AuxiliaryInfo{Version: 1, Data: []byte("b")}}, + {from: node2, info: common.AuxiliaryInfo{Version: 1, Data: []byte("b")}}, + {from: node3, info: common.AuxiliaryInfo{Version: 1, Data: []byte("c")}}, + }, + expected: []common.AuxiliaryInfo{ + {Version: 1, Data: []byte("b")}, + {Version: 1, Data: []byte("c")}, + }, + }, + { + name: "latest info from a node wins", + sends: []sentAuxInfo{ + {from: node1, info: common.AuxiliaryInfo{Version: 1, Data: []byte("b")}}, + {from: node1, info: common.AuxiliaryInfo{Version: 1, Data: []byte("c")}}, + }, + expected: []common.AuxiliaryInfo{ + {Version: 1, Data: []byte("c")}, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + store := newAuxInfoStore(&voteCountingAuxInfoApp{}) + for _, send := range tt.sends { + store.HandleAuxiliaryMessage(send.info, send.from) + } + + require.Equal(t, tt.expected, store.collectAuxInfo(history, nil)) + }) + } +} + +func TestCollectAuxInfoKeepsRejectedEntries(t *testing.T) { + store := newAuxInfoStore(&voteCountingAuxInfoApp{}) + info := common.AuxiliaryInfo{Version: 1, Data: []byte("a")} + store.HandleAuxiliaryMessage(info, avalanchego.NodeID{1}) + + // the entry duplicates the history, so it is rejected but kept in the store + history := AuxInfoHistory{ + Data: [][]byte{[]byte("a")}, + OldestVersionID: 1, + } + require.Empty(t, store.collectAuxInfo(history, nil)) + + // with a history that no longer contains the entry, it becomes legal + require.Equal(t, []common.AuxiliaryInfo{info}, store.collectAuxInfo(AuxInfoHistory{OldestVersionID: 1}, nil)) +} diff --git a/msm/encoding.canoto.go b/msm/encoding.canoto.go index a196d7d2..16664850 100644 --- a/msm/encoding.canoto.go +++ b/msm/encoding.canoto.go @@ -29,7 +29,7 @@ const ( canotoNumber_StateMachineMetadata__PChainHeight = 4 canotoNumber_StateMachineMetadata__Timestamp = 5 canotoNumber_StateMachineMetadata__ICMEpochInfo = 6 - canotoNumber_StateMachineMetadata__AuxiliaryInfo = 7 + canotoNumber_StateMachineMetadata__AuxiliaryInfoBatch = 7 canotoTag_StateMachineMetadata__SimplexEpochInfo = "\x0a" // canoto.Tag(canotoNumber_StateMachineMetadata__SimplexEpochInfo, canoto.Len) canotoTag_StateMachineMetadata__SimplexProtocolMetadata = "\x12" // canoto.Tag(canotoNumber_StateMachineMetadata__SimplexProtocolMetadata, canoto.Len) @@ -37,7 +37,7 @@ const ( canotoTag_StateMachineMetadata__PChainHeight = "\x20" // canoto.Tag(canotoNumber_StateMachineMetadata__PChainHeight, canoto.Varint) canotoTag_StateMachineMetadata__Timestamp = "\x28" // canoto.Tag(canotoNumber_StateMachineMetadata__Timestamp, canoto.Varint) canotoTag_StateMachineMetadata__ICMEpochInfo = "\x32" // canoto.Tag(canotoNumber_StateMachineMetadata__ICMEpochInfo, canoto.Len) - canotoTag_StateMachineMetadata__AuxiliaryInfo = "\x3a" // canoto.Tag(canotoNumber_StateMachineMetadata__AuxiliaryInfo, canoto.Len) + canotoTag_StateMachineMetadata__AuxiliaryInfoBatch = "\x3a" // canoto.Tag(canotoNumber_StateMachineMetadata__AuxiliaryInfoBatch, canoto.Len) ) type canotoData_StateMachineMetadata struct { @@ -104,9 +104,9 @@ func (*StateMachineMetadata) CanotoSpec(types ...reflect.Type) *canoto.Spec { /*types: */ types, ), canoto.FieldTypeFromField( - /*type inference:*/ (zero.AuxiliaryInfo), - /*FieldNumber: */ canotoNumber_StateMachineMetadata__AuxiliaryInfo, - /*Name: */ "AuxiliaryInfo", + /*type inference:*/ (zero.AuxiliaryInfoBatch), + /*FieldNumber: */ canotoNumber_StateMachineMetadata__AuxiliaryInfoBatch, + /*Name: */ "AuxiliaryInfoBatch", /*FixedLength: */ 0, /*Repeated: */ false, /*OneOf: */ "", @@ -269,7 +269,7 @@ func (c *StateMachineMetadata) UnmarshalCanotoFrom(r canoto.Reader) error { return err } r.B = remainingBytes - case canotoNumber_StateMachineMetadata__AuxiliaryInfo: + case canotoNumber_StateMachineMetadata__AuxiliaryInfoBatch: if wireType != canoto.Len { return canoto.ErrUnexpectedWireType } @@ -286,8 +286,8 @@ func (c *StateMachineMetadata) UnmarshalCanotoFrom(r canoto.Reader) error { // Unmarshal the field from the bytes. remainingBytes := r.B r.B = msgBytes - c.AuxiliaryInfo = canoto.MakePointer(c.AuxiliaryInfo) - if err := (c.AuxiliaryInfo).UnmarshalCanotoFrom(r); err != nil { + c.AuxiliaryInfoBatch = canoto.MakePointer(c.AuxiliaryInfoBatch) + if err := (c.AuxiliaryInfoBatch).UnmarshalCanotoFrom(r); err != nil { return err } r.B = remainingBytes @@ -320,7 +320,7 @@ func (c *StateMachineMetadata) ValidCanoto() bool { if !(&c.ICMEpochInfo).ValidCanoto() { return false } - if c.AuxiliaryInfo != nil && !(c.AuxiliaryInfo).ValidCanoto() { + if c.AuxiliaryInfoBatch != nil && !(c.AuxiliaryInfoBatch).ValidCanoto() { return false } return true @@ -354,10 +354,10 @@ func (c *StateMachineMetadata) CalculateCanotoCache() { if fieldSize := (&c.ICMEpochInfo).CachedCanotoSize(); fieldSize != 0 { size += uint64(len(canotoTag_StateMachineMetadata__ICMEpochInfo)) + canoto.SizeUint(fieldSize) + fieldSize } - if c.AuxiliaryInfo != nil { - (c.AuxiliaryInfo).CalculateCanotoCache() - fieldSize := (c.AuxiliaryInfo).CachedCanotoSize() - size += uint64(len(canotoTag_StateMachineMetadata__AuxiliaryInfo)) + canoto.SizeUint(fieldSize) + fieldSize + if c.AuxiliaryInfoBatch != nil { + (c.AuxiliaryInfoBatch).CalculateCanotoCache() + fieldSize := (c.AuxiliaryInfoBatch).CachedCanotoSize() + size += uint64(len(canotoTag_StateMachineMetadata__AuxiliaryInfoBatch)) + canoto.SizeUint(fieldSize) + fieldSize } atomic.StoreUint64(&c.canotoData.size, size) } @@ -425,11 +425,11 @@ func (c *StateMachineMetadata) MarshalCanotoInto(w canoto.Writer) canoto.Writer canoto.AppendUint(&w, fieldSize) w = (&c.ICMEpochInfo).MarshalCanotoInto(w) } - if c.AuxiliaryInfo != nil { - fieldSize := (c.AuxiliaryInfo).CachedCanotoSize() - canoto.Append(&w, canotoTag_StateMachineMetadata__AuxiliaryInfo) + if c.AuxiliaryInfoBatch != nil { + fieldSize := (c.AuxiliaryInfoBatch).CachedCanotoSize() + canoto.Append(&w, canotoTag_StateMachineMetadata__AuxiliaryInfoBatch) canoto.AppendUint(&w, fieldSize) - w = (c.AuxiliaryInfo).MarshalCanotoInto(w) + w = (c.AuxiliaryInfoBatch).MarshalCanotoInto(w) } return w } @@ -631,203 +631,6 @@ func (c *ICMEpochInfo) MarshalCanotoInto(w canoto.Writer) canoto.Writer { return w } -const ( - canotoNumber_AuxiliaryInfo__Info = 1 - canotoNumber_AuxiliaryInfo__PrevAuxInfoSeq = 2 - canotoNumber_AuxiliaryInfo__VersionID = 3 - - canotoTag_AuxiliaryInfo__Info = "\x0a" // canoto.Tag(canotoNumber_AuxiliaryInfo__Info, canoto.Len) - canotoTag_AuxiliaryInfo__PrevAuxInfoSeq = "\x10" // canoto.Tag(canotoNumber_AuxiliaryInfo__PrevAuxInfoSeq, canoto.Varint) - canotoTag_AuxiliaryInfo__VersionID = "\x18" // canoto.Tag(canotoNumber_AuxiliaryInfo__VersionID, canoto.Varint) -) - -type canotoData_AuxiliaryInfo struct { - size uint64 -} - -// CanotoSpec returns the specification of this canoto message. -func (*AuxiliaryInfo) CanotoSpec(...reflect.Type) *canoto.Spec { - var zero AuxiliaryInfo - s := &canoto.Spec{ - Name: "AuxiliaryInfo", - Fields: []canoto.FieldType{ - { - FieldNumber: canotoNumber_AuxiliaryInfo__Info, - Name: "Info", - OneOf: "", - TypeBytes: true, - }, - { - FieldNumber: canotoNumber_AuxiliaryInfo__PrevAuxInfoSeq, - Name: "PrevAuxInfoSeq", - OneOf: "", - TypeUint: canoto.SizeOf(zero.PrevAuxInfoSeq), - }, - { - FieldNumber: canotoNumber_AuxiliaryInfo__VersionID, - Name: "VersionID", - OneOf: "", - TypeUint: canoto.SizeOf(zero.VersionID), - }, - }, - } - s.CalculateCanotoCache() - return s -} - -// UnmarshalCanoto unmarshals a Canoto-encoded byte slice into the struct. -// -// During parsing, the canoto cache is saved. -func (c *AuxiliaryInfo) UnmarshalCanoto(bytes []byte) error { - r := canoto.Reader{ - B: bytes, - } - return c.UnmarshalCanotoFrom(r) -} - -// UnmarshalCanotoFrom populates the struct from a [canoto.Reader]. Most users -// should just use UnmarshalCanoto. -// -// During parsing, the canoto cache is saved. -// -// This function enables configuration of reader options. -func (c *AuxiliaryInfo) UnmarshalCanotoFrom(r canoto.Reader) error { - // Zero the struct before unmarshaling. - *c = AuxiliaryInfo{} - atomic.StoreUint64(&c.canotoData.size, uint64(len(r.B))) - - var minField uint32 - for canoto.HasNext(&r) { - field, wireType, err := canoto.ReadTag(&r) - if err != nil { - return err - } - if field < minField { - return canoto.ErrInvalidFieldOrder - } - - switch field { - case canotoNumber_AuxiliaryInfo__Info: - if wireType != canoto.Len { - return canoto.ErrUnexpectedWireType - } - - if err := canoto.ReadBytes(&r, &c.Info); err != nil { - return err - } - if len(c.Info) == 0 { - return canoto.ErrZeroValue - } - case canotoNumber_AuxiliaryInfo__PrevAuxInfoSeq: - if wireType != canoto.Varint { - return canoto.ErrUnexpectedWireType - } - - if err := canoto.ReadUint(&r, &c.PrevAuxInfoSeq); err != nil { - return err - } - if canoto.IsZero(c.PrevAuxInfoSeq) { - return canoto.ErrZeroValue - } - case canotoNumber_AuxiliaryInfo__VersionID: - if wireType != canoto.Varint { - return canoto.ErrUnexpectedWireType - } - - if err := canoto.ReadUint(&r, &c.VersionID); err != nil { - return err - } - if canoto.IsZero(c.VersionID) { - return canoto.ErrZeroValue - } - default: - return canoto.ErrUnknownField - } - - minField = field + 1 - } - return nil -} - -// ValidCanoto validates that the struct can be correctly marshaled into the -// Canoto format. -// -// Specifically, ValidCanoto ensures: -// 1. All OneOfs are specified at most once. -// 2. All strings are valid utf-8. -// 3. All custom fields are ValidCanoto. -func (c *AuxiliaryInfo) ValidCanoto() bool { - return true -} - -// CalculateCanotoCache populates size and OneOf caches based on the current -// values in the struct. -// -// It is not safe to copy this struct concurrently. -func (c *AuxiliaryInfo) CalculateCanotoCache() { - var size uint64 - if len(c.Info) != 0 { - size += uint64(len(canotoTag_AuxiliaryInfo__Info)) + canoto.SizeBytes(c.Info) - } - if !canoto.IsZero(c.PrevAuxInfoSeq) { - size += uint64(len(canotoTag_AuxiliaryInfo__PrevAuxInfoSeq)) + canoto.SizeUint(c.PrevAuxInfoSeq) - } - if !canoto.IsZero(c.VersionID) { - size += uint64(len(canotoTag_AuxiliaryInfo__VersionID)) + canoto.SizeUint(c.VersionID) - } - atomic.StoreUint64(&c.canotoData.size, size) -} - -// CachedCanotoSize returns the previously calculated size of the Canoto -// representation from CalculateCanotoCache. -// -// If CalculateCanotoCache has not yet been called, it will return 0. -// -// If the struct has been modified since the last call to CalculateCanotoCache, -// the returned size may be incorrect. -func (c *AuxiliaryInfo) CachedCanotoSize() uint64 { - return atomic.LoadUint64(&c.canotoData.size) -} - -// MarshalCanoto returns the Canoto representation of this struct. -// -// It is assumed that this struct is ValidCanoto. -// -// It is not safe to copy this struct concurrently. -func (c *AuxiliaryInfo) MarshalCanoto() []byte { - c.CalculateCanotoCache() - w := canoto.Writer{ - B: make([]byte, 0, c.CachedCanotoSize()), - } - w = c.MarshalCanotoInto(w) - return w.B -} - -// MarshalCanotoInto writes the struct into a [canoto.Writer] and returns the -// resulting [canoto.Writer]. Most users should just use MarshalCanoto. -// -// It is assumed that CalculateCanotoCache has been called since the last -// modification to this struct. -// -// It is assumed that this struct is ValidCanoto. -// -// It is not safe to copy this struct concurrently. -func (c *AuxiliaryInfo) MarshalCanotoInto(w canoto.Writer) canoto.Writer { - if len(c.Info) != 0 { - canoto.Append(&w, canotoTag_AuxiliaryInfo__Info) - canoto.AppendBytes(&w, c.Info) - } - if !canoto.IsZero(c.PrevAuxInfoSeq) { - canoto.Append(&w, canotoTag_AuxiliaryInfo__PrevAuxInfoSeq) - canoto.AppendUint(&w, c.PrevAuxInfoSeq) - } - if !canoto.IsZero(c.VersionID) { - canoto.Append(&w, canotoTag_AuxiliaryInfo__VersionID) - canoto.AppendUint(&w, c.VersionID) - } - return w -} - const ( canotoNumber_SimplexEpochInfo__PChainReferenceHeight = 1 canotoNumber_SimplexEpochInfo__EpochNumber = 2 diff --git a/msm/encoding.go b/msm/encoding.go index 856105c6..29da324b 100644 --- a/msm/encoding.go +++ b/msm/encoding.go @@ -33,9 +33,9 @@ type StateMachineMetadata struct { Timestamp uint64 `canoto:"uint,5"` // ICMEpochInfo is the metadata that the StateMachine uses for ICM epoching. ICMEpochInfo ICMEpochInfo `canoto:"value,6"` - // AuxiliaryInfo is application-specific information that the StateMachine doesn't need to understand, + // AuxiliaryInfoBatch is application-specific information that the StateMachine doesn't need to understand, // but can be used by applications that care about epoch changes, such as threshold distributed public key generation. - AuxiliaryInfo *AuxiliaryInfo `canoto:"pointer,7"` + AuxiliaryInfoBatch *AuxiliaryInfoBatch `canoto:"pointer,7"` canotoData canotoData_StateMachineMetadata } @@ -50,7 +50,7 @@ func (smm *StateMachineMetadata) Clone() StateMachineMetadata { PChainHeight: smm.PChainHeight, Timestamp: smm.Timestamp, ICMEpochInfo: smm.ICMEpochInfo.Clone(), - AuxiliaryInfo: smm.AuxiliaryInfo.Clone(), + AuxiliaryInfoBatch: smm.AuxiliaryInfoBatch, } } @@ -88,48 +88,6 @@ func (ei *ICMEpochInfo) Equal(other *ICMEpochInfo) bool { return ei.EpochStartTime == other.EpochStartTime && ei.EpochNumber == other.EpochNumber && ei.PChainEpochHeight == other.PChainEpochHeight } -// AuxiliaryInfo defines application-specific information for applications that might care about epoch change, -// such as threshold distributed public key generation. -type AuxiliaryInfo struct { - // Info is opaque bytes that can be used by applications to encode any information that describes - // the current state for the application. - Info []byte `canoto:"bytes,1"` - // PrevAuxInfoSeq is a sequence number that applications can use to find previous AuxiliaryInfo in the chain. - // It is zero if this is the first AuxiliaryInfo for this epoch. - PrevAuxInfoSeq uint64 `canoto:"uint,2"` - // VersionID is an identifier that identifies the application. - // Can be used for backward-compatibility and upgrade purposes. - VersionID common.VersionID `canoto:"uint,3"` - - canotoData canotoData_AuxiliaryInfo -} - -func (ai *AuxiliaryInfo) Clone() *AuxiliaryInfo { - if ai == nil { - return nil - } - return &AuxiliaryInfo{ - Info: ai.Info, - PrevAuxInfoSeq: ai.PrevAuxInfoSeq, - VersionID: ai.VersionID, - } -} - -func (ai *AuxiliaryInfo) IsZero() bool { - var zero AuxiliaryInfo - return ai.Equal(&zero) -} - -func (ai *AuxiliaryInfo) Equal(a *AuxiliaryInfo) bool { - if ai == nil { - return a == nil - } - if a == nil { - return ai == nil - } - return bytes.Equal(ai.Info, a.Info) && ai.PrevAuxInfoSeq == a.PrevAuxInfoSeq && ai.VersionID == a.VersionID -} - // SimplexEpochInfo is metadata used by the StateMachine. type SimplexEpochInfo struct { // PChainReferenceHeight is the P-Chain height that the StateMachine uses as a reference for the current epoch. @@ -381,6 +339,15 @@ func (nbms NodeBLSMappings) Nodes() common.Nodes { return nodeWeights } +// NodeIDs returns the NodeIDs of the mappings. +func (nbms NodeBLSMappings) NodeIDs() []common.NodeID { + nodeIDs := make([]common.NodeID, len(nbms)) + for i := range nbms { + nodeIDs[i] = nbms[i].NodeID[:] + } + return nodeIDs +} + // IndexByNodeID returns a mapping from NodeID to the validator's index in the set, // which is the position used by approval bitmasks. func (nbms NodeBLSMappings) IndexByNodeID() map[avalanchego.NodeID]int { diff --git a/msm/fake_node_test.go b/msm/fake_node_test.go index 6f7f292d..5d1db90c 100644 --- a/msm/fake_node_test.go +++ b/msm/fake_node_test.go @@ -6,7 +6,6 @@ package metadata import ( "context" "crypto/rand" - "crypto/sha256" "fmt" "sync/atomic" "testing" @@ -18,7 +17,9 @@ import ( "github.com/stretchr/testify/require" ) -var emptyAuxInfoDigest = sha256.Sum256(nil) +// emptyAuxInfoDigest is the candidate digest approvals commit to when the auxiliary info +// history is empty: LastHistoryDigest returns the zero digest in that case. +var emptyAuxInfoDigest [32]byte func TestFakeNodeEpochChangesDespiteEmptyMempool(t *testing.T) { validatorSetRetriever := validatorSetRetriever{ diff --git a/msm/fuzz_test.go b/msm/fuzz_test.go index 65e474c4..f9a6bd09 100644 --- a/msm/fuzz_test.go +++ b/msm/fuzz_test.go @@ -6,7 +6,6 @@ package metadata import ( "bytes" "context" - "crypto/sha256" "testing" "time" @@ -72,7 +71,7 @@ const numBuiltBlocks = 8 // inputs (selected by index). For each input, a freshly instantiated verifier MSM first // verifies the unfuzzed block (which must succeed), then verifies a copy whose // consensus-authoritative metadata has been mutated (which must fail). -// + // The mutation is applied at the field level (rather than by flipping serialized bytes) // so the fuzzed block is always well-formed: byte-level mutations of the Canoto encoding // overwhelmingly corrupt the structure and merely exercise the decoder. Each fuzzed field @@ -116,8 +115,10 @@ func FuzzVerifyBlock(f *testing.F) { fuzzedMD := block.Metadata field.set(&fuzzedMD, value) - if fieldIdx%2 == 1 && block.Metadata.AuxiliaryInfo == nil { - fuzzedMD.AuxiliaryInfo = &AuxiliaryInfo{PrevAuxInfoSeq: value} + if fieldIdx%2 == 1 && block.Metadata.AuxiliaryInfoBatch == nil { + // value|1 forces a non-zero PrevAuxInfoSeq: collecting-approvals blocks reconstruct it + // as 0 (parent has no aux info), so value 0 would match and slip through unrejected. + fuzzedMD.AuxiliaryInfoBatch = &AuxiliaryInfoBatch{PrevAuxInfoSeq: value | 1} } if bytes.Equal(fuzzedMD.MarshalCanoto(), block.Metadata.MarshalCanoto()) { @@ -251,10 +252,11 @@ func buildEpochChain(tb testing.TB, logger common.Logger) ([]*StateMachineBlock, block3 := build(3, 2, 1, block2) addBlock(3, block3, nil) - // The noopTestAuxInfoApp is always "ready" with an empty aux info history, so the candidate - // aux info digest the builder signs over is sha256 of the empty history. Peer approvals must - // carry the same digest to survive sanitizeApprovals' digest filter. - auxInfoDigest := sha256.Sum256(nil) + // The noopTestAuxInfoApp is always "ready" with an empty aux info history, and + // LastHistoryDigest returns the zero digest for an empty history. That zero value is the + // candidate digest the builder signs over, so peer approvals must carry it to survive + // sanitizeApprovals' digest filter. + var auxInfoDigest [32]byte // block4 & block5: collecting-approvals blocks (1/3 then 2/3, not enough to seal). require.NoError(tb, sm.HandleApproval(&common.ValidatorSetApproval{NodeID: node1, PChainHeight: pChainHeight2, AuxInfoDigest: auxInfoDigest, Signature: signApproval(pChainHeight2, auxInfoDigest)}, 1)) diff --git a/msm/msm.go b/msm/msm.go index ef2f9469..3b67bda1 100644 --- a/msm/msm.go +++ b/msm/msm.go @@ -5,13 +5,11 @@ package metadata import ( "context" - "crypto/sha256" "encoding/asn1" "encoding/binary" "errors" "fmt" "math" - "slices" "sync" "time" @@ -152,7 +150,7 @@ type AuxiliaryInfoGenVerifier interface { // Generate generates an auxiliary information encoded as a byte slice based on the history of auxiliary information // for the given versionID in the current epoch so far. - // If this is the first invocation in the epoch, DefaultVersionID() should be passed as the VersionID. + // If this is the first invocation in the epoch, DefaultversionID() should be passed as the VersionID. // Otherwise, the versionID from previous blocks in the epoch should be used. // If the application deems the given history to be sufficient for the epoch change, it can return a nil byte slice, // in which case it will not be appended to the history. @@ -168,6 +166,8 @@ type StateMachine struct { lock sync.RWMutex approvalStore *ApprovalStore approvalStoreValidatorSet NodeBLSMappings + + auxInfoStore *auxInfoStore } // Config contains the dependencies and configuration parameters needed to initialize the StateMachine. @@ -238,10 +238,18 @@ func NewStateMachine(config *Config) (*StateMachine, error) { if config.TimeSkewLimit == 0 { config.TimeSkewLimit = maxSkew } - sm := StateMachine{Config: config} + sm := StateMachine{Config: config, auxInfoStore: newAuxInfoStore(config.AuxiliaryInfoApp)} return &sm, nil } +// HandleAuxiliaryMessage processes +func (sm *StateMachine) HandleAuxiliaryInfo(info common.AuxiliaryInfo, from avalanchego.NodeID) { + sm.auxInfoStore.HandleAuxiliaryMessage(info, from) +} + +// HandleApproval processes a validator set approval from a node. +// timestamp is the time the approval was received, in milliseconds +// elapsed since January 1, 1970 UTC. func (sm *StateMachine) HandleApproval(approval *common.ValidatorSetApproval, timestamp uint64) error { sm.lock.Lock() approvalStore := sm.approvalStore @@ -257,6 +265,24 @@ func (sm *StateMachine) HandleApproval(approval *common.ValidatorSetApproval, ti return approvalStore.HandleApproval(approval, timestamp) } +// InitializeApprovalStore initializes the approval store for the given validator set +// if it is not already initialized for it. +func (sm *StateMachine) InitializeApprovalStore(validatorSet NodeBLSMappings) { + sm.maybeInitializeApprovalStore(validatorSet) +} + +// Approvals returns the approvals accumulated in the approval store, +// or nil if the store has not been initialized. +func (sm *StateMachine) Approvals() ValidatorSetApprovals { + sm.lock.RLock() + defer sm.lock.RUnlock() + + if sm.approvalStore == nil { + return nil + } + return sm.approvalStore.Approvals() +} + func (sm *StateMachine) maybeInitializeApprovalStore(validatorSet NodeBLSMappings) *ApprovalStore { sm.lock.Lock() defer sm.lock.Unlock() @@ -292,6 +318,7 @@ func (sm *StateMachine) BuildBlock(ctx context.Context, metadata common.Protocol sm.Logger.Debug("Building block", zap.Uint64("seq", metadata.Seq), + zap.Uint64("round", metadata.Round), zap.Uint64("epoch", metadata.Epoch), zap.Stringer("prevHash", metadata.Prev)) @@ -299,6 +326,7 @@ func (sm *StateMachine) BuildBlock(ctx context.Context, metadata common.Protocol elapsed := time.Since(start) sm.Logger.Debug("Built block", zap.Uint64("seq", metadata.Seq), + zap.Uint64("round", metadata.Round), zap.Uint64("epoch", metadata.Epoch), zap.Stringer("prevHash", metadata.Prev), zap.Duration("elapsed", elapsed), @@ -513,7 +541,7 @@ func verifyAgainstExpected( nextBlock *StateMachineBlock, timestamp time.Time, expectedIcmEpochInfo ICMEpochInfo, - auxInfo *AuxiliaryInfo, + auxInfo *AuxiliaryInfoBatch, ) error { if innerBlock != nil { if err := innerBlock.Verify(ctx, expectedIcmEpochInfo.PChainEpochHeight); err != nil { @@ -888,14 +916,19 @@ func (sm *StateMachine) buildBlockCollectingApprovals(ctx context.Context, paren return nil, err } - auxInfo, isAuxInfoReadyForEpochTransition, auxInfoDigest, err := sm.computeAuxInfo(parentBlock, prevBlockSeq, validators) + auxInfoHistory, err := GetAuxiliaryHistory(parentBlock, prevBlockSeq, sm.GetBlock, sm.AuxiliaryInfoApp.DefaultVersionID()) if err != nil { - return nil, fmt.Errorf("failed to compute auxiliary info: %w", err) + return nil, err + } + + isAuxInfoReadyForEpochTransition, err := sm.AuxiliaryInfoApp.IsSufficient(auxInfoHistory.OldestVersionID, validators, auxInfoHistory.Data) + if err != nil { + return nil, fmt.Errorf("failed to check if auxiliary info history is final: %w", err) } var newApprovals *approvals if isAuxInfoReadyForEpochTransition { - newApprovals, err = sm.computeNewApprovals(parentBlock, validators, auxInfoDigest) + newApprovals, err = sm.computeNewApprovals(parentBlock, validators, auxInfoHistory.LastHistoryDigest()) if err != nil { return nil, err } @@ -911,7 +944,10 @@ func (sm *StateMachine) buildBlockCollectingApprovals(ctx context.Context, paren now := sm.GetTime() icmEpochInfo := computeICMEpochInfo(parentBlock, sm.ComputeICMEpoch, now) - + auxInfo, err := sm.buildAuxInfoBatch(auxInfoHistory, parentBlock, validators, !isAuxInfoReadyForEpochTransition) + if err != nil { + return nil, fmt.Errorf("failed to build the auxiliary info batch: %w", err) + } // We might not have enough approvals to seal the current epoch, // in which case we just carry over the approvals we have so far to the next block, // so that eventually we'll have enough approvals to seal the epoch. @@ -1035,6 +1071,19 @@ func assembleApprovalToBeSigned(pChainHeight uint64, auxInfoDigest [32]byte) ([] return asn1.Marshal(signedMsg) } +func SignApproval(signer common.Signer, nextPChainReferenceHeight uint64, auxInfoDigest [32]byte) ([]byte, error) { + toBeSigned, err := assembleApprovalToBeSigned(nextPChainReferenceHeight, auxInfoDigest) + if err != nil { + return nil, err + } + + sig, err := signer.Sign(toBeSigned) + if err != nil { + return nil, fmt.Errorf("failed to sign approval: %w", err) + } + return sig, nil +} + func (sm *StateMachine) aggregatePubKeysForBitmask(nodeIDsBitmask []byte, validators NodeBLSMappings) ([]byte, error) { approvingNodes := avalanchego.BitmaskFromBytes(nodeIDsBitmask) publicKeys := make([][]byte, 0, len(validators)) @@ -1086,8 +1135,7 @@ func (sm *StateMachine) computeNewApprovals(parentBlock *StateMachineBlock, vali // Optimistically sign the epoch transition even if we have already did so in a previous round. // We'll just deduplicate this approval later on. - - sig, err := sm.createSelfApproval(prevBlockNextPChainReferenceHeight, auxInfoDigest) + sig, err := SignApproval(sm.Signer, prevBlockNextPChainReferenceHeight, auxInfoDigest) if err != nil { return nil, err } @@ -1099,6 +1147,8 @@ func (sm *StateMachine) computeNewApprovals(parentBlock *StateMachineBlock, vali Signature: sig, }) + sm.Logger.Debug("Retrieved approvals from peers", zap.Int("numApprovals", len(approvalsFromPeers))) + nextPChainHeight := prevBlockNextPChainReferenceHeight prevNextEpochApprovals := parentBlock.Metadata.SimplexEpochInfo.NextEpochApprovals @@ -1109,81 +1159,6 @@ func (sm *StateMachine) computeNewApprovals(parentBlock *StateMachineBlock, vali return newApprovals, nil } -func (sm *StateMachine) createSelfApproval(nextPChainReferenceHeight uint64, auxInfoDigest [32]byte) ([]byte, error) { - toBeSigned, err := assembleApprovalToBeSigned(nextPChainReferenceHeight, auxInfoDigest) - if err != nil { - return nil, err - } - - sig, err := sm.Signer.Sign(toBeSigned) - if err != nil { - return nil, fmt.Errorf("failed to sign approval: %w", err) - } - return sig, nil -} - -type auxInfoHistory struct { - data [][]byte - lastSeq uint64 -} - -func (aih *auxInfoHistory) lastHistory() []byte { - if len(aih.data) == 0 { - return nil - } - return aih.data[len(aih.data)-1] -} - -// collectAuxiliaryInfo traverses backwards starting from the given block and collects the AuxiliaryInfo of all blocks in the chain. -// returns the collected AuxiliaryInfo, the corresponding sequences of the blocks they were collected from, -// and the application ID of the oldest block that contains a non empty Info (or defaultVersionID if there was none). -func collectAuxiliaryInfo(block *StateMachineBlock, startSeq uint64, getBlock BlockRetriever, defaultVersionID common.VersionID) (auxInfoHistory, common.VersionID, error) { - var lastSeq *uint64 - var history [][]byte - var versionID = defaultVersionID - - // We traverse the chain of blocks backwards in the following manner: - // (1) Every block that doesn't have AuxiliaryInfo, its parents also do not have AuxiliaryInfo. - // (2) Every block that has AuxiliaryInfo, its descendants also have AuxiliaryInfo. - // (3) A block that has AuxiliaryInfo may have an empty Info field, but its PrevAuxInfoSeq field must point - // to a block that its AuxiliaryInfo isn't nil, and its Info field is also non-nil. - // (4) When a block with an empty Info field is built on a parent block that has AuxiliaryInfo, - // if its parent block has a non-empty Info field, then the block's PrevAuxInfoSeq points to its parent block. - // Else, its parent block has an empty Info field, then the block's PrevAuxInfoSeq is inherited from its parent block's PrevAuxInfoSeq. - - auxInfo := block.Metadata.AuxiliaryInfo - currentSeq := startSeq - for auxInfo != nil { - if len(auxInfo.Info) > 0 { - history = append(history, auxInfo.Info) - if lastSeq == nil { - lastSeq = new(uint64) - *lastSeq = currentSeq - } - versionID = auxInfo.VersionID - } - if auxInfo.PrevAuxInfoSeq == 0 { - // This is the first auxiliary info of the epoch, we can stop traversing back. - break - } - currentSeq = auxInfo.PrevAuxInfoSeq - prevBlock, _, err := getBlock(auxInfo.PrevAuxInfoSeq, [32]byte{}) - if err != nil { - return auxInfoHistory{}, 0, fmt.Errorf("%w: at sequence %d: %w", errAuxInfoBlockRetrieval, auxInfo.PrevAuxInfoSeq, err) - } - auxInfo = prevBlock.Metadata.AuxiliaryInfo - } - - if lastSeq == nil { - lastSeq = new(uint64) - *lastSeq = 0 - } - - // Reverse so the history (and the matching seqs) are ordered from oldest to newest. - slices.Reverse(history) - return auxInfoHistory{data: history, lastSeq: *lastSeq}, versionID, nil -} - // buildBlockImpatiently builds a block by waiting for the VM to build a block until MaxBlockBuildingWaitTime. // If the VM fails to build a block within that time, we build a block without an inner block, // so that we can continue making progress and not get stuck waiting for the VM. @@ -1194,7 +1169,7 @@ func (sm *StateMachine) buildBlockImpatiently(ctx context.Context, simplexEpochInfo SimplexEpochInfo, pChainHeight uint64, icmEpochInfo ICMEpochInfo, - auxInfo *AuxiliaryInfo) (*StateMachineBlock, error) { + auxInfo *AuxiliaryInfoBatch) (*StateMachineBlock, error) { impatientContext, cancel := context.WithTimeout(ctx, sm.MaxBlockBuildingWaitTime) defer cancel() @@ -1221,7 +1196,7 @@ func (sm *StateMachine) createSealingBlock(ctx context.Context, simplexEpochInfo SimplexEpochInfo, pChainHeight uint64, icmEpochInfo ICMEpochInfo, - auxInfo *AuxiliaryInfo) (*StateMachineBlock, error) { + auxInfo *AuxiliaryInfoBatch) (*StateMachineBlock, error) { simplexEpochInfo, err := sm.computeSimplexEpochInfoForSealingBlock(simplexEpochInfo) if err != nil { return nil, fmt.Errorf("failed to compute simplex epoch info for sealing block: %w", err) @@ -1262,7 +1237,7 @@ func wrapBlock( simplexBlacklist common.Blacklist, timestamp time.Time, icmEpochInfo ICMEpochInfo, - auxiliaryInfo *AuxiliaryInfo) *StateMachineBlock { + auxiliaryInfo *AuxiliaryInfoBatch) *StateMachineBlock { return &StateMachineBlock{ InnerBlock: childBlock, @@ -1273,7 +1248,7 @@ func wrapBlock( SimplexEpochInfo: newSimplexEpochInfo, PChainHeight: pChainHeight, ICMEpochInfo: icmEpochInfo, - AuxiliaryInfo: auxiliaryInfo, + AuxiliaryInfoBatch: auxiliaryInfo, }, } } @@ -1398,19 +1373,19 @@ func (sm *StateMachine) verifyBlockEpochSealed(ctx context.Context, parentBlock // computeExpectedAuxInfoForApprovalCollection computes the expected AuxiliaryInfo that should be included in the proposed block // for approval collection, and returns the auxiliary info digest, and whether the auxiliary info history is ready for epoch transition. -func (sm *StateMachine) computeExpectedAuxInfoForApprovalCollection(parentBlock *StateMachineBlock, nextBlock *StateMachineBlock, prevBlockSeq uint64, validators NodeBLSMappings) (*AuxiliaryInfo, [32]byte, bool, error) { +func (sm *StateMachine) computeExpectedAuxInfoForApprovalCollection(parentBlock *StateMachineBlock, nextBlock *StateMachineBlock, prevBlockSeq uint64, validators NodeBLSMappings) (*AuxiliaryInfoBatch, [32]byte, bool, error) { nextMD := nextBlock.Metadata prevMD := parentBlock.Metadata - auxInfoHistory, versionID, err := collectAuxiliaryInfo(parentBlock, prevBlockSeq, sm.GetBlock, sm.AuxiliaryInfoApp.DefaultVersionID()) + auxInfoHistory, err := GetAuxiliaryHistory(parentBlock, prevBlockSeq, sm.GetBlock, sm.AuxiliaryInfoApp.DefaultVersionID()) if err != nil { return nil, [32]byte{}, false, err } - if len(auxInfoHistory.data) > 0 && nextMD.AuxiliaryInfo == nil { + if len(auxInfoHistory.Data) > 0 && nextMD.AuxiliaryInfoBatch == nil { // If we have auxiliary info history but the proposed block doesn't include any auxiliary info, // it means the block builder has dropped the auxiliary info, which is not allowed. - return nil, [32]byte{}, false, fmt.Errorf("expected auxiliary info for application %d with history length %d, but got nil", versionID, len(auxInfoHistory.data)) + return nil, [32]byte{}, false, fmt.Errorf("expected auxiliary info for application %d with history length %d, but got nil", auxInfoHistory.OldestVersionID, len(auxInfoHistory.Data)) } // Else, either len(auxInfoHistory) == 0, @@ -1418,86 +1393,66 @@ func (sm *StateMachine) computeExpectedAuxInfoForApprovalCollection(parentBlock // Both of these cases are fine, because a node doesn't have to include Auxiliary information. // We will verify the legality of the proposed auxiliary info (if any) in the next step. - var expectedAuxInfo *AuxiliaryInfo - var proposedAuxInf []byte + var expectedAuxInfo *AuxiliaryInfoBatch + var proposedAuxInfos []common.AuxiliaryInfo - if nextMD.AuxiliaryInfo != nil { - proposedAuxInf = nextMD.AuxiliaryInfo.Info - expectedAuxInfo = &AuxiliaryInfo{ - VersionID: versionID, - Info: proposedAuxInf, + if nextMD.AuxiliaryInfoBatch != nil { + proposedAuxInfos = nextMD.AuxiliaryInfoBatch.data + expectedAuxInfo = &AuxiliaryInfoBatch{ + data: proposedAuxInfos, } - if prevMD.AuxiliaryInfo != nil { - expectedAuxInfo.PrevAuxInfoSeq = auxInfoHistory.lastSeq + if prevMD.AuxiliaryInfoBatch != nil { + expectedAuxInfo.PrevAuxInfoSeq = auxInfoHistory.LastSeq } } - if err := sm.AuxiliaryInfoApp.IsLegalAppend(versionID, validators, auxInfoHistory.data, proposedAuxInf); err != nil { - return nil, [32]byte{}, false, fmt.Errorf("proposed auxiliary info is not a legal append to the history for application %d: %w", versionID, err) + // go through all the collected data and return whether proposed datum are legal + + for _, info := range proposedAuxInfos { + if auxInfoHistory.OldestVersionID != info.Version { + return nil, [32]byte{}, false, fmt.Errorf("proposed auxiliary info does not have the proper version %d: %w", auxInfoHistory.OldestVersionID, err) + } + if err := sm.AuxiliaryInfoApp.IsLegalAppend(auxInfoHistory.OldestVersionID, validators, auxInfoHistory.Data, info.Data); err != nil { + return nil, [32]byte{}, false, fmt.Errorf("proposed auxiliary info is not a legal append to the history for application %d: %w", auxInfoHistory.OldestVersionID, err) + } } - auxInfoReady, err := sm.AuxiliaryInfoApp.IsSufficient(versionID, validators, auxInfoHistory.data) + auxInfoReady, err := sm.AuxiliaryInfoApp.IsSufficient(auxInfoHistory.OldestVersionID, validators, auxInfoHistory.Data) if err != nil { - return nil, [32]byte{}, false, fmt.Errorf("failed to check if auxiliary info history is final for application %d: %w", versionID, err) + return nil, [32]byte{}, false, fmt.Errorf("failed to check if auxiliary info history is final for application %d: %w", auxInfoHistory.OldestVersionID, err) } var digest [32]byte if auxInfoReady { - digest = sha256.Sum256(auxInfoHistory.lastHistory()) + digest = auxInfoHistory.LastHistoryDigest() } return expectedAuxInfo, digest, auxInfoReady, nil } -// computeAuxInfo computes the AuxiliaryInfo that should be included in the block being built, and whether the auxiliary info history is ready for epoch transition, -func (sm *StateMachine) computeAuxInfo(parentBlock *StateMachineBlock, prevBlockSeq uint64, validators NodeBLSMappings) (*AuxiliaryInfo, bool, common.Digest, error) { - auxInfoHistory, versionID, err := collectAuxiliaryInfo(parentBlock, prevBlockSeq, sm.GetBlock, sm.AuxiliaryInfoApp.DefaultVersionID()) - if err != nil { - return nil, false, common.Digest{}, err - } - - isAuxInfoReadyForEpochTransition, err := sm.AuxiliaryInfoApp.IsSufficient(versionID, validators, auxInfoHistory.data) - if err != nil { - return nil, false, common.Digest{}, fmt.Errorf("failed to check if auxiliary info history is final: %w", err) - } - - var auxInfo *AuxiliaryInfo - parentAuxInfo := parentBlock.Metadata.AuxiliaryInfo - if parentAuxInfo != nil { - auxInfo = &AuxiliaryInfo{ - VersionID: parentAuxInfo.VersionID, - PrevAuxInfoSeq: auxInfoHistory.lastSeq, - } +// buildAuxInfoBatch builds the AuxiliaryInfoBatch that should be included in the block being built. +func (sm *StateMachine) buildAuxInfoBatch(history AuxInfoHistory, parentBlock *StateMachineBlock, validators NodeBLSMappings, shouldGenerate bool) (*AuxiliaryInfoBatch, error) { + var prevAuxInfoSeq uint64 + if parentBlock.Metadata.AuxiliaryInfoBatch != nil { + prevAuxInfoSeq = history.LastSeq } - if !isAuxInfoReadyForEpochTransition { - // If the auxiliary info isn't ready for epoch transition, - // we should focus on contributing to finalizing it before collecting approvals for the epoch transition, - // as without it being ready, we won't be able to transition epochs anyway. - auxInf, err := sm.AuxiliaryInfoApp.Generate(versionID, validators, auxInfoHistory.data) - if err != nil { - return nil, false, common.Digest{}, fmt.Errorf("failed to generate auxiliary info: %w", err) - } - if auxInfo == nil { - // This is the first auxiliary info we're generating for this epoch, - // so we need to initialize it. - auxInfo = &AuxiliaryInfo{ - VersionID: versionID, - Info: auxInf, - } - } else { - // Otherwise, we already have auxiliary info from the parent block, - // so we just update the Info field and carry over the VersionID and PrevAuxInfoSeq. - auxInfo.Info = auxInf - } + var info []common.AuxiliaryInfo + if shouldGenerate { + info = sm.auxInfoStore.collectAuxInfo(history, validators) } - var auxInfoDigest common.Digest - if isAuxInfoReadyForEpochTransition { - auxInfoDigest = sha256.Sum256(auxInfoHistory.lastHistory()) + // Only emit a batch when there's new info to record, or a prior batch in the chain + // to link back to. An empty batch with PrevAuxInfoSeq == 0 would violate the invariant + // that an empty batch points to an ancestor with non-empty entries. + if len(info) == 0 && prevAuxInfoSeq == 0 { + return nil, nil } - return auxInfo, isAuxInfoReadyForEpochTransition, auxInfoDigest, nil + return &AuxiliaryInfoBatch{ + data: info, + PrevAuxInfoSeq: prevAuxInfoSeq, + }, nil } // constructSimplexZeroBlockSimplexEpochInfo constructs the SimplexEpochInfo for the zero block, which is the first ever block built by Simplex. diff --git a/msm/msm_test.go b/msm/msm_test.go index 83ae9d6c..3c316b94 100644 --- a/msm/msm_test.go +++ b/msm/msm_test.go @@ -7,7 +7,6 @@ import ( "context" "crypto/rand" "crypto/sha256" - "errors" "fmt" "math" "testing" @@ -1596,8 +1595,9 @@ func TestVerifyCollectingApprovalsNotReady(t *testing.T) { sm, tc, parent := newSM(t) block := build(t, sm, tc, parent) - // The builder generated auxiliary info but collected no approvals. - require.NotNil(t, block.Metadata.AuxiliaryInfo) + // No auxiliary info was received and the history isn't ready, so the builder collects + // neither auxiliary info (nil batch) nor approvals. + require.Nil(t, block.Metadata.AuxiliaryInfoBatch) require.Empty(t, block.Metadata.SimplexEpochInfo.NextEpochApprovals.NodeIDs) require.Empty(t, block.Metadata.SimplexEpochInfo.NextEpochApprovals.Signature) @@ -1650,7 +1650,7 @@ func TestCollectingApprovalsAuxInfoGating(t *testing.T) { vote1 := []byte("vote-1") vote2 := []byte("vote-2") votes := [][]byte{vote1, vote2} - sm.AuxiliaryInfoApp = &voteCountingAuxInfoApp{ + auxiliaryApp := &voteCountingAuxInfoApp{ threshold: 2, randomTape: func() []byte { next := votes[0] @@ -1658,10 +1658,9 @@ func TestCollectingApprovalsAuxInfoGating(t *testing.T) { return next }, } + sm.AuxiliaryInfoApp = auxiliaryApp - // A 3-node validator set including MyNodeID at index 0, so the optimistic self-approval - // is retained once approvals are collected, but a single approval is below quorum (the - // block stays in the collecting state rather than sealing). + // A 3-node validator set including MyNodeID at index 0 validators := NodeBLSMappings{ {NodeID: avalanchego.NodeID(sm.MyNodeID), BLSKey: []byte{1}, Weight: 1}, {NodeID: avalanchego.NodeID{0xBB}, BLSKey: []byte{2}, Weight: 1}, @@ -1686,9 +1685,9 @@ func TestCollectingApprovalsAuxInfoGating(t *testing.T) { } tc.blockStore[parentSeq] = &outerBlock{block: parent} - // build constructs the next collecting block on top of prev, stores it so it can serve + // buildAndVerify constructs the next collecting block on top of prev, stores it so it can serve // as a parent (and as a back-pointer target for the aux info history), and verifies it. - build := func(seq uint64, prev StateMachineBlock) *StateMachineBlock { + buildAndVerify := func(seq uint64, prev StateMachineBlock) *StateMachineBlock { tc.blockBuilder.Block = &testutil.InnerBlock{TS: time.Now(), BlockHeight: seq, Content: []byte{byte(seq)}} md := common.ProtocolMetadata{Seq: seq, Round: seq, Epoch: 1, Prev: prev.Digest()} block, err := sm.BuildBlock(context.Background(), md, emptyBlacklist) @@ -1702,29 +1701,44 @@ func TestCollectingApprovalsAuxInfoGating(t *testing.T) { return b.Metadata.SimplexEpochInfo.NextEpochApprovals } // requireAuxInfo compares the meaningful fields, ignoring the cached canoto size. - requireAuxInfo := func(want, got *AuxiliaryInfo) { + requireAuxInfo := func(want, got *AuxiliaryInfoBatch) { require.True(t, want.Equal(got), "expected aux info %+v, got %+v", want, got) } + auxVersionId := auxiliaryApp.DefaultVersionID() + firstAuxInfoBytes, err := auxiliaryApp.Generate(auxVersionId, validators, [][]byte{}) + require.NoError(t, err) + firstAuxInfo := common.AuxiliaryInfo{ + Version: auxVersionId, + Data: firstAuxInfoBytes, + } + sm.HandleAuxiliaryInfo(firstAuxInfo, validators[0].NodeID) + // block1: history empty, not final -> generates vote1, collects no approvals. - block1 := build(parentSeq+1, parent) - requireAuxInfo(&AuxiliaryInfo{Info: vote1, VersionID: 1}, block1.Metadata.AuxiliaryInfo) + block1 := buildAndVerify(parentSeq+1, parent) + requireAuxInfo(&AuxiliaryInfoBatch{data: []common.AuxiliaryInfo{firstAuxInfo}}, block1.Metadata.AuxiliaryInfoBatch) require.Empty(t, approvals(block1).NodeIDs) + // we get another auxiliary info sent + auxInfoHistory, err := GetAuxiliaryHistory(block1, parentSeq+1, sm.GetBlock, auxVersionId) + secondAuxInfoBytes, err := auxiliaryApp.Generate(auxVersionId, validators, auxInfoHistory.Data) + require.NoError(t, err) + secondAuxInfo := common.AuxiliaryInfo{ + Version: auxVersionId, + Data: secondAuxInfoBytes, + } + sm.HandleAuxiliaryInfo(secondAuxInfo, validators[1].NodeID) + // block2: history [vote1], still not final -> generates vote2, collects no approvals. - block2 := build(parentSeq+2, *block1) - requireAuxInfo(&AuxiliaryInfo{Info: vote2, PrevAuxInfoSeq: parentSeq + 1, VersionID: 1}, block2.Metadata.AuxiliaryInfo) + block2 := buildAndVerify(parentSeq+2, *block1) + requireAuxInfo(&AuxiliaryInfoBatch{data: []common.AuxiliaryInfo{secondAuxInfo}, PrevAuxInfoSeq: parentSeq + 1}, block2.Metadata.AuxiliaryInfoBatch) require.Empty(t, approvals(block2).NodeIDs) - // block3: history [vote1, vote2] is now final -> no new vote, and approvals are - // collected (the optimistic self-approval sets MyNodeID's bit). block3 is the first - // empty-Info block; it points at block2, the last non-empty Info block. - block3 := build(parentSeq+3, *block2) - requireAuxInfo(&AuxiliaryInfo{PrevAuxInfoSeq: parentSeq + 2, VersionID: 1}, block3.Metadata.AuxiliaryInfo) - require.Equal(t, []byte{1}, approvals(block3).NodeIDs, "self-approval bit should be set once aux info is ready") + block3 := buildAndVerify(parentSeq+3, *block2) + requireAuxInfo(&AuxiliaryInfoBatch{PrevAuxInfoSeq: parentSeq + 2}, block3.Metadata.AuxiliaryInfoBatch) // The collected approval must be signed over the epoch-transition payload for the - //mnext epoch's P-chain reference height (200) and the digest + //next epoch's P-chain reference height (200) and the digest // of the final auxiliary info history, which is sha256 of the last vote (vote2). wantSigned, err := assembleApprovalToBeSigned(nextPChainRefHeight, sha256.Sum256(vote2)) require.NoError(t, err) @@ -1735,16 +1749,16 @@ func TestCollectingApprovalsAuxInfoGating(t *testing.T) { // quorum). Its PrevAuxInfoSeq must SKIP the empty block3 and point at block2 (parentSeq+2), // the most recent non-empty Info block -- not at its immediate parent block3 (parentSeq+3). // This is the case the rest of the chain never reaches and where "skip" differs from "successive". - block4 := build(parentSeq+4, *block3) - require.NotEqual(t, parentSeq+3, block4.Metadata.AuxiliaryInfo.PrevAuxInfoSeq, + block4 := buildAndVerify(parentSeq+4, *block3) + require.NotEqual(t, parentSeq+3, block4.Metadata.AuxiliaryInfoBatch.PrevAuxInfoSeq, "PrevAuxInfoSeq must not point at the empty-Info parent block3") - requireAuxInfo(&AuxiliaryInfo{PrevAuxInfoSeq: parentSeq + 2, VersionID: 1}, block4.Metadata.AuxiliaryInfo) + requireAuxInfo(&AuxiliaryInfoBatch{PrevAuxInfoSeq: parentSeq + 2}, block4.Metadata.AuxiliaryInfoBatch) // block5: another empty-Info block on top of the empty block4. The back-pointer still skips // the whole empty run and points at block2, confirming the skip persists across consecutive // empty-Info blocks (collectAuxiliaryInfo finds the same most-recent non-empty block each time). - block5 := build(parentSeq+5, *block4) - requireAuxInfo(&AuxiliaryInfo{PrevAuxInfoSeq: parentSeq + 2, VersionID: 1}, block5.Metadata.AuxiliaryInfo) + block5 := buildAndVerify(parentSeq+5, *block4) + requireAuxInfo(&AuxiliaryInfoBatch{PrevAuxInfoSeq: parentSeq + 2}, block5.Metadata.AuxiliaryInfoBatch) } func TestCollectingApprovalsAuxInfoVersionIDIsBackwardCompatible(t *testing.T) { @@ -1752,14 +1766,16 @@ func TestCollectingApprovalsAuxInfoVersionIDIsBackwardCompatible(t *testing.T) { // VersionID must be reused for the rest of the epoch -- for both building AND verifying // subsequent blocks -- even if the application's DefaultVersionID() later changes. // - // collectAuxiliaryInfo only consults DefaultVersionID() when the auxiliary info history is - // empty; once a block carries a VersionID, every later buildAndVerify and verify reads that VersionID - // back from the chain instead. So we seed the epoch's parent with auxiliary info stamped with - // VersionID 1, then flip DefaultVersionID() to 2 right after the first Generate(). Because the - // epoch already has a VersionID on-chain, every Generate()/IsLegalAppend()/IsSufficient() - // invocation -- on the buildAndVerify path and the verify path -- must keep using VersionID 1, never 2. - // The app asserts that internally: it requires the VersionID it receives to equal - // expectedVersionID, which we hold at 1 throughout. + // GetAuxiliaryHistory only consults DefaultVersionID() when the auxiliary info history is + // empty; once a block carries a VersionID, every later build and verify reads that VersionID + // back from the chain instead. Auxiliary info is no longer generated inside the block: it + // arrives from peers via HandleAuxiliaryInfo and is collected into the block being built. So the + // first collecting block establishes the epoch's VersionID (1) from the received vote while the + // default is still 1, then we flip DefaultVersionID() to 2. Because the epoch already carries a + // VersionID on-chain, every Generate()/IsLegalAppend()/IsSufficient() invocation -- on both the + // build and verify paths -- must keep using VersionID 1, never 2. The app asserts that + // internally: it requires the VersionID it receives to equal expectedVersionID, which we hold at + // 1 throughout. const ( pChainRefHeight = uint64(100) @@ -1771,10 +1787,11 @@ func TestCollectingApprovalsAuxInfoVersionIDIsBackwardCompatible(t *testing.T) { sm.GetPChainHeightForVerifying = func() uint64 { return nextPChainRefHeight } sm.GetPChainHeightForProposing = func() uint64 { return nextPChainRefHeight } - // threshold 4 so Generate() runs for the first three collecting blocks built on top of the - // pre-seeded parent (history not yet sufficient), giving us one "first" and several "later" - // Generate() invocations. defaultVersionID starts at 1 (the original default); expectedVersionID - // stays 1 for the whole test -- the app asserts every invocation uses it. + // threshold 4 so the history never becomes sufficient across the three collecting blocks we + // build: every block collects a freshly received auxiliary vote (never approvals), giving one + // "first" build under the original default and two "later" builds after the default changes. + // defaultVersionID starts at 1 (the original default); expectedVersionID stays 1 for the whole + // test -- the app asserts every invocation uses it. app := &versionRecordingAuxInfoApp{ t: t, threshold: 4, @@ -1791,8 +1808,8 @@ func TestCollectingApprovalsAuxInfoVersionIDIsBackwardCompatible(t *testing.T) { } tc.validatorSetRetriever.result = validators - // The parent already carries auxiliary info for this epoch, stamped with VersionID 1. - // This is the backward-compatibility precondition: the epoch's VersionID is already set. + // A plain parent with no auxiliary info yet: the epoch's VersionID is established by the first + // received auxiliary vote rather than pre-seeded into the block. parent := StateMachineBlock{ InnerBlock: &testutil.InnerBlock{TS: time.Now(), BlockHeight: 1, Content: []byte{0xAA}}, Metadata: StateMachineMetadata{ @@ -1806,11 +1823,6 @@ func TestCollectingApprovalsAuxInfoVersionIDIsBackwardCompatible(t *testing.T) { NextPChainReferenceHeight: nextPChainRefHeight, PrevVMBlockSeq: parentSeq - 1, }, - AuxiliaryInfo: &AuxiliaryInfo{ - VersionID: 1, - Info: []byte("vote-0"), - PrevAuxInfoSeq: 0, - }, }, } tc.blockStore[parentSeq] = &outerBlock{block: parent} @@ -1827,133 +1839,109 @@ func TestCollectingApprovalsAuxInfoVersionIDIsBackwardCompatible(t *testing.T) { return block } - // block1: the epoch already has VersionID 1 (from the parent), so the buildAndVerify reads 1 from the - // chain and generates vote-1 under VersionID 1. Being the first Generate(), we now flip the - // application's default to 2. Verifying block1 also reads VersionID 1 from the parent's aux - // info, so it passes despite the changed default. + // receiveAuxVote generates the next vote under VersionID 1 and delivers it as if received from + // the given validator, so the next built block collects it into its auxiliary info. + receiveAuxVote := func(from avalanchego.NodeID) { + data, err := app.Generate(app.expectedVersionID, validators, nil) + require.NoError(t, err) + sm.HandleAuxiliaryInfo(common.AuxiliaryInfo{Version: app.expectedVersionID, Data: data}, from) + } + + // block1: default is still 1 and the history is empty, so the received vote (VersionID 1) sets + // the epoch's VersionID. Building and verifying block1 both read 1 from the default. We then flip + // the default to 2; every later build/verify must keep reading 1 back from the chain. + receiveAuxVote(validators[0].NodeID) block1 := buildAndVerify(parentSeq+1, parent) - require.Equal(t, common.VersionID(1), block1.Metadata.AuxiliaryInfo.VersionID) + require.Equal(t, common.VersionID(1), block1.Metadata.AuxiliaryInfoBatch.data[0].Version) app.defaultVersionID = 2 - // block2, block3: the default is now 2, but each block's buildAndVerify and verify still read VersionID - // 1 back from the chain and ignore the changed default. + // block2, block3: the default is now 2, but each block's build and verify still read VersionID 1 + // back from the chain and ignore the changed default. + receiveAuxVote(validators[1].NodeID) block2 := buildAndVerify(parentSeq+2, *block1) - require.Equal(t, common.VersionID(1), block2.Metadata.AuxiliaryInfo.VersionID) + require.Equal(t, common.VersionID(1), block2.Metadata.AuxiliaryInfoBatch.data[0].Version) + receiveAuxVote(validators[2].NodeID) block3 := buildAndVerify(parentSeq+3, *block2) - require.Equal(t, common.VersionID(1), block3.Metadata.AuxiliaryInfo.VersionID) - - // block4: history [vote-0, vote-1, vote-2, vote-3] is now sufficient, so no further vote is - // generated and approvals are collected -- still under VersionID 1. - block4 := buildAndVerify(parentSeq+4, *block3) - require.Equal(t, common.VersionID(1), block4.Metadata.AuxiliaryInfo.VersionID) + require.Equal(t, common.VersionID(1), block3.Metadata.AuxiliaryInfoBatch.data[0].Version) } -func TestCollectAuxiliaryInfo(t *testing.T) { - const versionID = common.VersionID(7) +func TestCollectingApprovalsIncludesMultipleAuxInfoMessages(t *testing.T) { + // Multiple auxiliary info messages received from distinct validators are all collected into a + // single built block's AuxiliaryInfoBatch. - blockWithAuxInfo := func(info []byte, prevAuxInfoSeq uint64) StateMachineBlock { - return StateMachineBlock{ - Metadata: StateMachineMetadata{ - AuxiliaryInfo: &AuxiliaryInfo{ - Info: info, - PrevAuxInfoSeq: prevAuxInfoSeq, - VersionID: versionID, - }, - }, - } - } + const ( + pChainRefHeight = uint64(100) + nextPChainRefHeight = uint64(200) + parentSeq = uint64(10) + ) - errRetrieval := errors.New("retrieval failed") + sm, tc := newStateMachine(t) + sm.GetPChainHeightForProposing = func() uint64 { return nextPChainRefHeight } + sm.GetPChainHeightForVerifying = func() uint64 { return nextPChainRefHeight } - // startSeq is the sequence of tt.block itself (the block collectAuxiliaryInfo starts from). - const startSeq = uint64(10) + // A high threshold keeps the history from ever becoming sufficient, so the block stays in the + // collecting-approvals state and carries the received auxiliary info instead of sealing. + sm.AuxiliaryInfoApp = &voteCountingAuxInfoApp{threshold: 10} - tests := []struct { - name string - block StateMachineBlock - blocks map[uint64]StateMachineBlock - getBlockErr error - expectedHistory [][]byte - expectedLastSeq uint64 - expectedversionID common.VersionID - expectedErr error - }{ - { - name: "block without auxiliary info", - block: StateMachineBlock{}, - }, - { - name: "empty info, first of epoch", - block: blockWithAuxInfo(nil, 0), - }, - { - name: "non-empty info, first of epoch", - block: blockWithAuxInfo([]byte{1}, 0), - expectedHistory: [][]byte{{1}}, - expectedLastSeq: startSeq, - expectedversionID: versionID, - }, - { - name: "empty info pointing back to non-empty info", - block: blockWithAuxInfo(nil, 3), - blocks: map[uint64]StateMachineBlock{ - 3: blockWithAuxInfo([]byte{1}, 0), - }, - expectedHistory: [][]byte{{1}}, - expectedLastSeq: 3, - expectedversionID: versionID, - }, - { - name: "history is ordered from oldest to newest", - block: blockWithAuxInfo([]byte{3}, 5), - blocks: map[uint64]StateMachineBlock{ - 5: blockWithAuxInfo([]byte{2}, 2), - 2: blockWithAuxInfo([]byte{1}, 0), + validators := NodeBLSMappings{ + {NodeID: avalanchego.NodeID(sm.MyNodeID), BLSKey: []byte{1}, Weight: 1}, + {NodeID: avalanchego.NodeID{0xBB}, BLSKey: []byte{2}, Weight: 1}, + {NodeID: avalanchego.NodeID{0xCC}, BLSKey: []byte{3}, Weight: 1}, + } + tc.validatorSetRetriever.result = validators + + parent := StateMachineBlock{ + InnerBlock: &testutil.InnerBlock{TS: time.Now(), BlockHeight: 1, Content: []byte{0xAA}}, + Metadata: StateMachineMetadata{ + PChainHeight: nextPChainRefHeight, + SimplexProtocolMetadata: common.ProtocolMetadata{ + Seq: parentSeq, Round: 5, Epoch: 1, }, - expectedHistory: [][]byte{{1}, {2}, {3}}, - expectedLastSeq: startSeq, - expectedversionID: versionID, - }, - { - name: "traversal stops at a block without auxiliary info", - block: blockWithAuxInfo([]byte{2}, 4), - blocks: map[uint64]StateMachineBlock{ - 4: {}, + SimplexEpochInfo: SimplexEpochInfo{ + PChainReferenceHeight: pChainRefHeight, + EpochNumber: 1, + NextPChainReferenceHeight: nextPChainRefHeight, + PrevVMBlockSeq: parentSeq - 1, }, - expectedHistory: [][]byte{{2}}, - expectedLastSeq: startSeq, - expectedversionID: versionID, - }, - { - name: "block retrieval failure", - block: blockWithAuxInfo([]byte{2}, 4), - getBlockErr: errRetrieval, - expectedErr: errRetrieval, }, } + tc.blockStore[parentSeq] = &outerBlock{block: parent} - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - getBlock := func(seq uint64, _ common.Digest) (StateMachineBlock, *common.Finalization, error) { - if tt.getBlockErr != nil { - return StateMachineBlock{}, nil, tt.getBlockErr - } - block, ok := tt.blocks[seq] - require.True(t, ok, "unexpected retrieval of block at sequence %d", seq) - return block, nil, nil - } + version := sm.AuxiliaryInfoApp.DefaultVersionID() - history, gotversionID, err := collectAuxiliaryInfo(&tt.block, startSeq, getBlock, 0) - if tt.expectedErr != nil { - require.ErrorIs(t, err, tt.expectedErr) - require.ErrorIs(t, err, errAuxInfoBlockRetrieval) - return - } - require.NoError(t, err) - require.Equal(t, tt.expectedHistory, history.data) - require.Equal(t, tt.expectedLastSeq, history.lastSeq) - require.Equal(t, tt.expectedversionID, gotversionID) - }) + // buildWithAuxMessages delivers one distinct auxiliary message per validator, builds a block on + // top of prev, verifies and stores it, and asserts the block collected exactly those messages. + // collectAuxInfo orders entries by NodeID, so the payloads are compared as a set. + buildWithAuxMessages := func(seq uint64, prev StateMachineBlock, payloads [][]byte) *StateMachineBlock { + require.Len(t, payloads, len(validators)) + for i, payload := range payloads { + sm.HandleAuxiliaryInfo(common.AuxiliaryInfo{Version: version, Data: payload}, validators[i].NodeID) + } + + tc.blockBuilder.Block = &testutil.InnerBlock{TS: time.Now(), BlockHeight: seq, Content: []byte{byte(seq)}} + md := common.ProtocolMetadata{Seq: seq, Round: seq, Epoch: 1, Prev: prev.Digest()} + block, err := sm.BuildBlock(context.Background(), md, emptyBlacklist) + require.NoError(t, err) + require.NoError(t, sm.VerifyBlock(context.Background(), block)) + tc.blockStore[seq] = &outerBlock{block: *block} + + require.NotNil(t, block.Metadata.AuxiliaryInfoBatch) + gotPayloads := make([][]byte, 0, len(payloads)) + for _, info := range block.Metadata.AuxiliaryInfoBatch.data { + require.Equal(t, version, info.Version) + gotPayloads = append(gotPayloads, info.Data) + } + require.ElementsMatch(t, payloads, gotPayloads) + return block } + + // First batch of three messages lands in block1. + block1 := buildWithAuxMessages(parentSeq+1, parent, [][]byte{[]byte("aux-a"), []byte("aux-b"), []byte("aux-c")}) + + // A second batch of three messages lands in block2, built on top of block1. + block2 := buildWithAuxMessages(parentSeq+2, *block1, [][]byte{[]byte("aux-d"), []byte("aux-e"), []byte("aux-f")}) + + // block2 links back to block1, the most recent block carrying non-empty auxiliary info. + require.Equal(t, parentSeq+1, block2.Metadata.AuxiliaryInfoBatch.PrevAuxInfoSeq) } diff --git a/nonvalidator/non_validator.go b/nonvalidator/non_validator.go index ae1243b2..ae54cf4f 100644 --- a/nonvalidator/non_validator.go +++ b/nonvalidator/non_validator.go @@ -559,6 +559,13 @@ func (n *NonValidator) broadcastLatestEpoch() { }) } +func (n *NonValidator) HighestValidatedEpoch() (uint64, common.Nodes) { + n.lock.Lock() + defer n.lock.Unlock() + + return n.epochs.highestEpoch() +} + // sendRequest sends a common.ReplicationRequest for a given sequence to a node. func (n *NonValidator) sendRequest(seq uint64, to common.NodeID) { request := common.ReplicationRequest{ diff --git a/simplex/epoch.go b/simplex/epoch.go index 903d694c..4bed453d 100644 --- a/simplex/epoch.go +++ b/simplex/epoch.go @@ -171,6 +171,7 @@ func (e *Epoch) HandleMessage(msg *common.Message, from common.NodeID) error { switch { case msg.ReplicationRequest != nil && e.ReplicationEnabled: return e.handleReplicationRequest(msg.ReplicationRequest, from) + // TODO: process finalizations from non-validators default: e.Logger.Debug("Invalid message type", zap.Stringer("from", from)) return nil @@ -788,6 +789,7 @@ func (e *Epoch) Stop() { e.buildBlockScheduler.Close() e.timeoutHandler.Close() e.replicationState.Close() + e.Logger.Info("Node shutdown complete") } func (e *Epoch) isEpochSealed() bool { @@ -804,6 +806,8 @@ func (e *Epoch) handleFinalizationMessage(message *common.Finalization, from com return nil } + // TODO: check if this finalization message is from a different epoch. If so, we need to request the sealing block. + // https://github.com/ava-labs/Simplex/issues/442 if err := VerifyQC(message.QC, e.signatureAggregator.IsQuorum, e.validatorsToPKs, message, e.validators); err != nil { e.Logger.Debug(fmt.Sprintf("Finalization %s", err), zap.Int("round", int(message.Finalization.Round)), @@ -1466,13 +1470,17 @@ func (e *Epoch) indexFinalizations(startRound uint64) error { } func (e *Epoch) indexFinalization(block common.VerifiedBlock, finalization common.Finalization) error { - if err := e.Storage.Index(e.finishCtx, block, finalization); err != nil { - return err + // index only if the epoch is not sealed + if !e.epochSealed.Load() { + if err := e.Storage.Index(e.finishCtx, block, finalization); err != nil { + return err + } + e.Logger.Info("Committed block", + zap.Uint64("round", finalization.Finalization.Round), + zap.Uint64("sequence", finalization.Finalization.Seq), + zap.Stringer("digest", finalization.Finalization.BlockHeader.Digest)) } - e.Logger.Info("Committed block", - zap.Uint64("round", finalization.Finalization.Round), - zap.Uint64("sequence", finalization.Finalization.Seq), - zap.Stringer("digest", finalization.Finalization.BlockHeader.Digest)) + e.lastBlock = &common.VerifiedFinalizedBlock{ VerifiedBlock: block, Finalization: finalization, @@ -2570,7 +2578,6 @@ func (e *Epoch) createBlockBuildingTask(metadata common.ProtocolMetadata, blackl e.blockBuilderCtx, e.blockBuilderCancelFunc = context.WithCancel(e.finishCtx) context := e.blockBuilderCtx cancel := e.blockBuilderCancelFunc - return func() common.Digest { e.lock.Lock() if e.isEpochSealed() { @@ -2595,6 +2602,7 @@ func (e *Epoch) createBlockBuildingTask(metadata common.ProtocolMetadata, blackl return common.Digest{} } + e.Logger.Info("block is proposed") e.proposeBlock(block) return block.BlockHeader().Digest @@ -3187,7 +3195,7 @@ func (e *Epoch) handleReplicationRequest(req *common.ReplicationRequest, from co remainingBytes := e.MaxReplicationResponseSize if req.LatestFinalizedSeq > 0 { - if e.lastBlock != nil && e.lastBlock.Finalization.Finalization.Seq > req.LatestFinalizedSeq { + if e.lastBlock != nil && e.lastBlock.Finalization.Finalization.Seq >= req.LatestFinalizedSeq { latestFinalizedSeq := &common.VerifiedQuorumRound{ VerifiedBlock: e.lastBlock.VerifiedBlock, Finalization: &e.lastBlock.Finalization, diff --git a/testutil/controlled.go b/testutil/controlled.go index 05fece1e..84eed38a 100644 --- a/testutil/controlled.go +++ b/testutil/controlled.go @@ -91,7 +91,7 @@ func (n *ControlledInMemoryNetwork) AdvanceWithoutLeader(round uint64, laggingNo type ControlledNode struct { *BasicNode - bb *testControlledBlockBuilder + bb *TestControlledBlockBuilder WAL *TestWAL Storage *InMemStorage } @@ -185,9 +185,9 @@ func (t *ControlledNode) TickUntilRoundAdvanced(round uint64, tick time.Duration } } -// testControlledBlockBuilder is a BlockBuilder that only builds a block when +// TestControlledBlockBuilder is a BlockBuilder that only builds a block when // a control signal is received. -type testControlledBlockBuilder struct { +type TestControlledBlockBuilder struct { t *testing.T control chan struct{} TestBlockBuilder @@ -195,22 +195,22 @@ type testControlledBlockBuilder struct { // NewTestControlledBlockBuilder returns a BlockBuilder that only builds a block // when triggerNewBlock is called. -func NewTestControlledBlockBuilder(t *testing.T) *testControlledBlockBuilder { - return &testControlledBlockBuilder{ +func NewTestControlledBlockBuilder(t *testing.T) *TestControlledBlockBuilder { + return &TestControlledBlockBuilder{ t: t, control: make(chan struct{}, 1), TestBlockBuilder: *NewTestBlockBuilder(), } } -func (t *testControlledBlockBuilder) TriggerNewBlock() { +func (t *TestControlledBlockBuilder) TriggerNewBlock() { select { case t.control <- struct{}{}: default: } } -func (t *testControlledBlockBuilder) BuildBlock(ctx context.Context, metadata common.ProtocolMetadata, blacklist common.Blacklist) (common.VerifiedBlock, bool) { +func (t *TestControlledBlockBuilder) BuildBlock(ctx context.Context, metadata common.ProtocolMetadata, blacklist common.Blacklist) (common.VerifiedBlock, bool) { select { case <-t.control: case <-ctx.Done(): diff --git a/testutil/node.go b/testutil/node.go index 214fcdb0..c00c524c 100644 --- a/testutil/node.go +++ b/testutil/node.go @@ -239,7 +239,7 @@ type TestNodeConfig struct { Comm common.Communication SigAggregatorCreator common.SignatureAggregatorCreator ReplicationEnabled bool - BlockBuilder *testControlledBlockBuilder + BlockBuilder *TestControlledBlockBuilder // Long Running Tests MaxRoundWindow uint64 diff --git a/transition_listener.go b/transition_listener.go new file mode 100644 index 00000000..222a1e0d --- /dev/null +++ b/transition_listener.go @@ -0,0 +1,162 @@ +package simplex + +import ( + "errors" + "time" + + "github.com/ava-labs/simplex/avalanchego" + "github.com/ava-labs/simplex/common" + + metadata "github.com/ava-labs/simplex/msm" +) + +// epochTransitionListener reacts to blocks committed to storage. When a +// transition block is indexed, it performs any tasks required of this node to +// complete the epoch transition, such as sending out approval messages. +// Non-validators should also use this listener, since they may become +// validators after the transition. +type epochTransitionListener struct { + // broadcaster is used for broadcasting potential approvals and auxiliary information. + // It should be broadcast to the validators of the current epoch. + broadcaster Broadcaster + + myNodeID avalanchego.NodeID + + // getValidatorSet returns the validator set at a given P-chain height. + getValidatorSet metadata.ValidatorSetRetriever + // getBlock retrieves a previously finalized block, used to traverse the auxiliary info history. + getBlock metadata.BlockRetriever + // signer signs epoch transition approvals. + signer common.Signer + // auxInfoApp decides whether the auxiliary info history is sufficient and generates new entries. + auxInfoApp metadata.AuxiliaryInfoGenVerifier + // handleApproval records our own broadcast approval in the local approval store. + // It is set for validators (whose MSM builds the next blocks and must include the + // approval) and nil for non-validators, which have no block builder to feed. + handleApproval func(approval *common.ValidatorSetApproval, timestamp uint64) error + + // onEpochChange is a callback the listener invokes once a sealing block for `epoch` has been indexed. + onEpochChange func(epoch uint64, validators common.Nodes) error + + logger common.Logger +} + +func newEpochTransitionListener( + logger common.Logger, + broadcaster Broadcaster, + myNodeID avalanchego.NodeID, + getValidatorSet metadata.ValidatorSetRetriever, + getBlock metadata.BlockRetriever, + signer common.Signer, + auxInfoApp metadata.AuxiliaryInfoGenVerifier, + handleApproval func(approval *common.ValidatorSetApproval, timestamp uint64) error, + onEpochChange func(epoch uint64, validators common.Nodes) error, +) *epochTransitionListener { + return &epochTransitionListener{ + broadcaster: broadcaster, + myNodeID: myNodeID, + getValidatorSet: getValidatorSet, + getBlock: getBlock, + signer: signer, + auxInfoApp: auxInfoApp, + handleApproval: handleApproval, + onEpochChange: onEpochChange, + logger: logger, + } +} + +func (a *epochTransitionListener) onIndex(block *ParsedBlock) error { + switch block.Type() { + case metadata.BlockTypeSealing: + if block.SealingBlockInfo() == nil { + return errors.New("sealing block has empty SealingBlockInfo") + } + return a.onEpochChange(block.BlockHeader().Seq, block.SealingBlockInfo().ValidatorSet) + case metadata.BlockTypeTransitioning: + return a.handleTransitionBlock(block) + } + + return nil +} + +func (a *epochTransitionListener) handleTransitionBlock(block *ParsedBlock) error { + nextEpochPChainReference := block.Metadata.SimplexEpochInfo.NextPChainReferenceHeight + + // if our node is not in the next validator set, no need to send anything. + nextEpochValidatorSet, err := a.getValidatorSet(nextEpochPChainReference) + if err != nil { + return err + } + + indexes := nextEpochValidatorSet.IndexByNodeID() + if _, ok := indexes[a.myNodeID]; !ok { + return nil // we are not in the next validator set + } + + auxInfoHistory, err := metadata.GetAuxiliaryHistory(&block.StateMachineBlock, block.BlockHeader().Seq, a.getBlock, a.auxInfoApp.DefaultVersionID()) + if err != nil { + return err + } + + isSufficient, err := a.auxInfoApp.IsSufficient(auxInfoHistory.OldestVersionID, nextEpochValidatorSet, auxInfoHistory.Data) + if err != nil { + return err + } + + if isSufficient { + // no more auxiliary info to send, maybe send our approval + lastAuxInfoDigest := auxInfoHistory.LastHistoryDigest() + return a.maybeSendApprovals(block, lastAuxInfoDigest) + } + + // we need more auxiliary information, attempt to generate + generatedAuxInfo, err := a.auxInfoApp.Generate(auxInfoHistory.OldestVersionID, nextEpochValidatorSet, auxInfoHistory.Data) + if err != nil { + return err + } + + if generatedAuxInfo == nil { + return nil + } + + auxInfoMessage := &common.Message{ + AuxiliaryInfo: &common.AuxiliaryInfo{ + Version: auxInfoHistory.OldestVersionID, + Data: generatedAuxInfo, + }, + } + + a.broadcaster.Broadcast(auxInfoMessage) + return nil +} + +// TODO: use common.Digest +func (a *epochTransitionListener) maybeSendApprovals(block *ParsedBlock, auxInfoDigest [32]byte) error { + nextEpochPChainReference := block.Metadata.SimplexEpochInfo.NextPChainReferenceHeight + + sig, err := metadata.SignApproval(a.signer, nextEpochPChainReference, auxInfoDigest) + if err != nil { + return err + } + + approval := common.ValidatorSetApproval{ + NodeID: a.myNodeID, + PChainHeight: nextEpochPChainReference, + AuxInfoDigest: auxInfoDigest, + Signature: sig, + } + + approvalMessage := common.Message{ + EpochTransitionApproval: &approval, + } + + a.broadcaster.Broadcast(&approvalMessage) + + // Validators also record their own approval locally so the next block they build + // includes it. Non-validators have no block builder, so handleApproval is nil. + if a.handleApproval == nil { + return nil + } + timestamp := uint64(time.Now().UnixMilli()) + return a.handleApproval(&approval, timestamp) +} diff --git a/transition_listener_test.go b/transition_listener_test.go new file mode 100644 index 00000000..77a5858d --- /dev/null +++ b/transition_listener_test.go @@ -0,0 +1,265 @@ +package simplex + +import ( + "testing" + + "github.com/ava-labs/simplex/avalanchego" + "github.com/ava-labs/simplex/common" + metadata "github.com/ava-labs/simplex/msm" + "github.com/ava-labs/simplex/testutil" + + "github.com/stretchr/testify/require" +) + +var testNodeID = avalanchego.NodeID{1} + +type recordingBroadcaster struct { + messages []*common.Message +} + +func (rb *recordingBroadcaster) Broadcast(msg *common.Message) { + rb.messages = append(rb.messages, msg) +} + +type stubSigner struct { + sig []byte +} + +func (s stubSigner) Sign([]byte) ([]byte, error) { + return s.sig, nil +} + +type stubAuxInfoApp struct { + sufficient bool + generated []byte +} + +func (s *stubAuxInfoApp) IsLegalAppend(common.VersionID, metadata.NodeBLSMappings, [][]byte, []byte) error { + return nil +} + +func (s *stubAuxInfoApp) IsSufficient(common.VersionID, metadata.NodeBLSMappings, [][]byte) (bool, error) { + return s.sufficient, nil +} + +func (s *stubAuxInfoApp) Generate(common.VersionID, metadata.NodeBLSMappings, [][]byte) ([]byte, error) { + return s.generated, nil +} + +func (s *stubAuxInfoApp) DefaultVersionID() common.VersionID { + return 7 +} + +type listenerTestEnv struct { + broadcaster *recordingBroadcaster + epochs []epochChange + // approvals records the approvals fed back to the local store via the handleApproval + // callback. It stays empty for a non-validator listener (nil handleApproval). + approvals []common.ValidatorSetApproval + listener *epochTransitionListener +} + +// newListenerTestEnv builds a listener wired to the given next-epoch validator set and +// auxiliary info app. When isValidator is true, the listener is given a handleApproval +// callback (recording into env.approvals) as a real validator MSM would; otherwise it is +// nil, matching a non-validator that has no block builder to record its own approval. +func newListenerTestEnv(t *testing.T, validatorSet metadata.NodeBLSMappings, auxApp metadata.AuxiliaryInfoGenVerifier, isValidator bool) *listenerTestEnv { + env := &listenerTestEnv{broadcaster: &recordingBroadcaster{}} + + getValidatorSet := func(uint64) (metadata.NodeBLSMappings, error) { + return validatorSet, nil + } + getBlock := func(seq uint64, _ common.Digest) (metadata.StateMachineBlock, *common.Finalization, error) { + require.Fail(t, "unexpected getBlock call", "seq %d", seq) + return metadata.StateMachineBlock{}, nil, nil + } + + var handleApproval func(approval *common.ValidatorSetApproval, timestamp uint64) error + if isValidator { + handleApproval = func(approval *common.ValidatorSetApproval, _ uint64) error { + env.approvals = append(env.approvals, *approval) + return nil + } + } + + env.listener = newEpochTransitionListener( + testutil.MakeLogger(t, 1), + env.broadcaster, + testNodeID, + getValidatorSet, + getBlock, + stubSigner{sig: []byte("signature")}, + auxApp, + handleApproval, + func(epoch uint64, validators common.Nodes) error { + env.epochs = append(env.epochs, epochChange{ + epoch: epoch, + validators: validators, + }) + return nil + }, + ) + return env +} + +// newTransitionBlock returns a ParsedBlock of type BlockTypeTransitioning carrying the +// given next-epoch P-chain reference height. The listener supplies the validator set and +// auxiliary info app, so the block itself needs no MSM. +func newTransitionBlock(t *testing.T, nextPChainRef uint64) *ParsedBlock { + block := &ParsedBlock{ + StateMachineBlock: metadata.StateMachineBlock{ + Metadata: metadata.StateMachineMetadata{ + SimplexProtocolMetadata: common.ProtocolMetadata{Seq: 10}, + SimplexEpochInfo: metadata.SimplexEpochInfo{ + NextPChainReferenceHeight: nextPChainRef, + }, + }, + }, + } + require.Equal(t, metadata.BlockTypeTransitioning, block.Type()) + return block +} + +func TestSealingBlockCallback(t *testing.T) { + const sealingSeq = uint64(42) + + env := newListenerTestEnv(t, nil, nil, true) + + validatorSet := metadata.NodeBLSMappings{{NodeID: testNodeID, BLSKey: []byte("bls-key"), Weight: 5}} + block := &ParsedBlock{ + StateMachineBlock: metadata.StateMachineBlock{ + Metadata: metadata.StateMachineMetadata{ + SimplexProtocolMetadata: (common.ProtocolMetadata{Seq: sealingSeq}), + SimplexEpochInfo: metadata.SimplexEpochInfo{ + // a non-empty PrevSealingBlockHash distinguishes a sealing block from the zero block + PrevSealingBlockHash: [32]byte{1}, + BlockValidationDescriptor: &metadata.BlockValidationDescriptor{ + AggregatedMembership: metadata.AggregatedMembership{Members: validatorSet}, + }, + }, + }, + }, + } + require.Equal(t, metadata.BlockTypeSealing, block.Type()) + + require.NoError(t, env.listener.onIndex(block)) + require.Empty(t, env.broadcaster.messages) + + // the callback should have been invoked with the sealing block's epoch and validator set + expectedValidators := common.Nodes{{Id: testNodeID[:], Weight: 5, PK: []byte("bls-key")}} + require.Equal(t, []epochChange{{epoch: sealingSeq, validators: expectedValidators}}, env.epochs) +} + +func TestTransitionNotInValidatorSet(t *testing.T) { + // the next validator set does not contain our node + otherValidator := metadata.NodeBLSMappings{{NodeID: avalanchego.NodeID{2}, Weight: 1}} + auxApp := &stubAuxInfoApp{sufficient: false, generated: []byte("more aux info")} + env := newListenerTestEnv(t, otherValidator, auxApp, true) + + block := newTransitionBlock(t, 100) + + require.NoError(t, env.listener.onIndex(block)) + require.Empty(t, env.broadcaster.messages) + require.Empty(t, env.epochs) + require.Empty(t, env.approvals) +} + +func TestTransitionNotEnoughAuxiliary(t *testing.T) { + validatorSet := metadata.NodeBLSMappings{{NodeID: testNodeID, Weight: 1}, {NodeID: avalanchego.NodeID{2}, Weight: 1}} + auxApp := &stubAuxInfoApp{sufficient: false, generated: []byte("more aux info")} + env := newListenerTestEnv(t, validatorSet, auxApp, true) + + block := newTransitionBlock(t, 100) + + require.NoError(t, env.listener.onIndex(block)) + require.Empty(t, env.epochs) + + // the generated auxiliary info should be broadcast instead of an approval + require.Len(t, env.broadcaster.messages, 1) + msg := env.broadcaster.messages[0] + require.Nil(t, msg.EpochTransitionApproval) + require.NotNil(t, msg.AuxiliaryInfo) + require.Equal(t, auxApp.DefaultVersionID(), msg.AuxiliaryInfo.Version) + require.Equal(t, auxApp.generated, msg.AuxiliaryInfo.Data) + require.Empty(t, env.approvals) +} + +func TestTransitionBroadcastsApproval(t *testing.T) { + const nextPChainRef = uint64(100) + + validatorSet := metadata.NodeBLSMappings{{NodeID: testNodeID, Weight: 1}} + env := newListenerTestEnv(t, validatorSet, &stubAuxInfoApp{sufficient: true}, true) + + block := newTransitionBlock(t, nextPChainRef) + + require.NoError(t, env.listener.onIndex(block)) + require.Empty(t, env.epochs) + + require.Len(t, env.broadcaster.messages, 1) + msg := env.broadcaster.messages[0] + require.Nil(t, msg.AuxiliaryInfo) + require.NotNil(t, msg.EpochTransitionApproval) + + approval := msg.EpochTransitionApproval + require.Equal(t, testNodeID, approval.NodeID) + require.Equal(t, nextPChainRef, approval.PChainHeight) + require.Equal(t, [32]byte{}, approval.AuxInfoDigest) // no auxiliary info was collected + require.Equal(t, []byte("signature"), approval.Signature) + + // a validator also records its own approval locally so its next block includes it + require.Equal(t, []common.ValidatorSetApproval{*approval}, env.approvals) +} + +// TestNonValidatorContributesAuxiliaryInfo asserts that a node still on the outside of the +// current validator set but present in the NEXT one contributes auxiliary info during the +// transition, exactly like a validator does. Non-validators pass a nil handleApproval, so +// nothing is recorded locally, but the auxiliary info is still broadcast. +func TestNonValidatorContributesAuxiliaryInfo(t *testing.T) { + validatorSet := metadata.NodeBLSMappings{{NodeID: testNodeID, Weight: 1}, {NodeID: avalanchego.NodeID{2}, Weight: 1}} + auxApp := &stubAuxInfoApp{sufficient: false, generated: []byte("non-validator aux")} + env := newListenerTestEnv(t, validatorSet, auxApp, false /* non-validator */) + + block := newTransitionBlock(t, 100) + + require.NoError(t, env.listener.onIndex(block)) + + require.Len(t, env.broadcaster.messages, 1) + msg := env.broadcaster.messages[0] + require.Nil(t, msg.EpochTransitionApproval) + require.NotNil(t, msg.AuxiliaryInfo) + require.Equal(t, auxApp.DefaultVersionID(), msg.AuxiliaryInfo.Version) + require.Equal(t, auxApp.generated, msg.AuxiliaryInfo.Data) + + // non-validators do not record approvals locally + require.Empty(t, env.approvals) + require.Empty(t, env.epochs) +} + +// TestNonValidatorContributesApproval asserts that once the auxiliary info history is +// sufficient, a non-validator that belongs to the next validator set broadcasts its +// epoch transition approval. It does not record the approval locally (nil handleApproval), +// since it has no block builder to include it. +func TestNonValidatorContributesApproval(t *testing.T) { + const nextPChainRef = uint64(100) + + validatorSet := metadata.NodeBLSMappings{{NodeID: testNodeID, Weight: 1}} + env := newListenerTestEnv(t, validatorSet, &stubAuxInfoApp{sufficient: true}, false /* non-validator */) + + block := newTransitionBlock(t, nextPChainRef) + + require.NoError(t, env.listener.onIndex(block)) + + require.Len(t, env.broadcaster.messages, 1) + msg := env.broadcaster.messages[0] + require.Nil(t, msg.AuxiliaryInfo) + require.NotNil(t, msg.EpochTransitionApproval) + + approval := msg.EpochTransitionApproval + require.Equal(t, testNodeID, approval.NodeID) + require.Equal(t, nextPChainRef, approval.PChainHeight) + require.Equal(t, []byte("signature"), approval.Signature) + + // non-validators broadcast but do not record their own approval locally + require.Empty(t, env.approvals) + require.Empty(t, env.epochs) +} diff --git a/util.go b/util.go new file mode 100644 index 00000000..3874cb26 --- /dev/null +++ b/util.go @@ -0,0 +1,113 @@ +// Copyright (C) 2019-2025, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package simplex + +import ( + "errors" + "fmt" + + "github.com/ava-labs/simplex/common" + metadata "github.com/ava-labs/simplex/msm" + "go.uber.org/zap" +) + +var ( + errNoGenesisBlock = errors.New("no genesis block found in storage") + errNonSealingBlock = errors.New("expected sealing block, got a non-sealing block") +) + +// LastBlock returns the last block in storage along with the total number of blocks. +func LastBlock(storage Storage) (metadata.StateMachineBlock, uint64, error) { + numBlocks := storage.NumBlocks() + if numBlocks == 0 { + return metadata.StateMachineBlock{}, 0, errNoGenesisBlock + } + + lastBlock, _, err := storage.GetBlock(numBlocks - 1) + if err != nil { + return metadata.StateMachineBlock{}, 0, fmt.Errorf("error retrieving last block from storage: %w", err) + } + + return lastBlock, numBlocks, nil +} + +// getLastAcceptedEpoch determines the epoch the instance should start at based on +// the last block in storage. If the ledger only contains non-Simplex blocks, the +// epoch is the first Simplex height. If the last block is a sealing block, the +// epoch it seals has ended, so the next epoch is returned. Otherwise, the epoch +// of the last block is returned. +func getLastAcceptedEpochAndValidatorSet(config *Config) (common.Nodes, uint64, error) { + lastBlock, numBlocks, err := LastBlock(config.Storage) + if err != nil { + return nil, 0, fmt.Errorf("error retrieving last block: %w", err) + } + + lastNonSimplexHeight := config.LastNonSimplexInnerBlock.Height() + parsedLastBlock := ParsedBlock{StateMachineBlock: lastBlock} + epochNum := parsedLastBlock.BlockHeader().Epoch + genesisValidatorSet := config.PlatformChain.GenesisValidatorSet() + + var validatorSet metadata.NodeBLSMappings + var nodes common.Nodes + + switch { + // If all we have in the ledger is non-Simplex blocks, load the validator set from genesis + case lastNonSimplexHeight+1 == numBlocks: + validatorSet = genesisValidatorSet + nodes = validatorSetToNodes(genesisValidatorSet) + epochNum = lastNonSimplexHeight + 1 + config.Logger.Debug("Determined epoch and validator set from genesis (ledger holds only non-Simplex blocks)", + zap.Uint64("epoch", epochNum)) + // If the last block persisted is a sealing block, then we are in the next epoch. + case lastBlock.SealingBlockInfo() != nil: + epochNum = parsedLastBlock.BlockHeader().Seq + validatorSet = constructValidatorSetFromSealingBlock(&parsedLastBlock) + nodes = lastBlock.SealingBlockInfo().ValidatorSet + config.Logger.Debug("Determined epoch and validator set from sealing block at tip", + zap.Uint64("epoch", epochNum)) + // Else, we have at least one Simplex block in the ledger, and it's not a sealing block. + default: + // Therefore, the sequence of the sealing block is the epoch number. + sealingBlockSeq := parsedLastBlock.BlockHeader().Epoch + sealingBlock, _, err := config.Storage.GetBlock(sealingBlockSeq) + if err != nil { + return nil, 0, fmt.Errorf("error retrieving sealing block from storage: %w", err) + } + if sealingBlock.Metadata.SimplexEpochInfo.BlockValidationDescriptor == nil { + return nil, 0, fmt.Errorf("%w at seq %d", errNonSealingBlock, sealingBlockSeq) + } + validatorSet = constructValidatorSetFromSealingBlock(&ParsedBlock{StateMachineBlock: sealingBlock}) + nodes = validatorSetToNodes(validatorSet) + config.Logger.Debug("Determined epoch and validator set from sealing block in storage", + zap.Uint64("epoch", epochNum), zap.Uint64("sealingBlockSeq", sealingBlockSeq)) + } + return nodes, epochNum, nil +} + +func validatorSetToNodes(validatorSet metadata.NodeBLSMappings) common.Nodes { + var nodes common.Nodes + for i := range validatorSet { + vdr := &validatorSet[i] + nodes = append(nodes, common.Node{ + Id: vdr.NodeID[:], + Weight: vdr.Weight, + PK: vdr.BLSKey, + }) + } + return nodes +} + +func constructValidatorSetFromSealingBlock(lastBlock *ParsedBlock) metadata.NodeBLSMappings { + var validatorSet metadata.NodeBLSMappings + vdrs := lastBlock.Metadata.SimplexEpochInfo.BlockValidationDescriptor.AggregatedMembership.Members + for i := range vdrs { + vdr := &vdrs[i] + validatorSet = append(validatorSet, metadata.NodeBLSMapping{ + NodeID: vdr.NodeID, + BLSKey: vdr.BLSKey, + Weight: vdr.Weight, + }) + } + return validatorSet +} diff --git a/util_test.go b/util_test.go new file mode 100644 index 00000000..6d63f086 --- /dev/null +++ b/util_test.go @@ -0,0 +1,198 @@ +package simplex + +import ( + "context" + "errors" + "testing" + + "github.com/ava-labs/simplex/avalanchego" + "github.com/ava-labs/simplex/common" + metadata "github.com/ava-labs/simplex/msm" + "github.com/ava-labs/simplex/testutil" + "github.com/stretchr/testify/require" +) + +// stubStorage is a minimal Storage for exercising util functions. +// When err is set, GetBlock fails at seq errSeq. +type stubStorage struct { + blocks []metadata.StateMachineBlock + errSeq uint64 + err error +} + +func (s *stubStorage) NumBlocks() uint64 { + return uint64(len(s.blocks)) +} + +func (s *stubStorage) GetBlock(seq uint64) (metadata.StateMachineBlock, *common.Finalization, error) { + if s.err != nil && seq == s.errSeq { + return metadata.StateMachineBlock{}, nil, s.err + } + return s.blocks[seq], &common.Finalization{}, nil +} + +func (s *stubStorage) Index(context.Context, common.VerifiedBlock, common.Finalization) error { + return nil +} + +// nonSimplexBlock returns a pre-fork block holding only an inner block at the given height. +func nonSimplexBlock(height uint64) metadata.StateMachineBlock { + return metadata.StateMachineBlock{InnerBlock: &testInnerBlock{Height_: height}} +} + +// simplexBlock returns a non-sealing simplex block at the given epoch and seq. +func simplexBlock(epoch, seq uint64) metadata.StateMachineBlock { + return metadata.StateMachineBlock{ + Metadata: metadata.StateMachineMetadata{ + SimplexProtocolMetadata: common.ProtocolMetadata{Epoch: epoch, Seq: seq}, + }, + } +} + +// sealingBlock returns a sealing block at the given epoch and seq whose +// descriptor holds the given validator set. +func sealingBlock(epoch, seq uint64, members []metadata.NodeBLSMapping) metadata.StateMachineBlock { + block := simplexBlock(epoch, seq) + block.Metadata.SimplexEpochInfo.BlockValidationDescriptor = &metadata.BlockValidationDescriptor{ + AggregatedMembership: metadata.AggregatedMembership{Members: members}, + } + return block +} + +func testValidatorSet() metadata.NodeBLSMappings { + return metadata.NodeBLSMappings{ + {NodeID: avalanchego.NodeID{1}, BLSKey: []byte{1, 2}, Weight: 1}, + {NodeID: avalanchego.NodeID{2}, BLSKey: []byte{3, 4}, Weight: 2}, + } +} + +// epochTestConfig derives the last non-Simplex height from the leading +// blocks in storage that carry no Simplex metadata. +func epochTestConfig(t *testing.T, storage *stubStorage, genesisSet metadata.NodeBLSMappings) *Config { + var lastNonSimplexHeight uint64 + for seq, block := range storage.blocks { + if block.Metadata.SimplexProtocolMetadata.Epoch != 0 { + break + } + lastNonSimplexHeight = uint64(seq) + } + return &Config{ + Storage: storage, + PlatformChain: newTestPChain(genesisSet), + LastNonSimplexInnerBlock: &testInnerBlock{Height_: lastNonSimplexHeight}, + Logger: testutil.MakeLogger(t, 1), + } +} + +// LastBlock errors on empty storage. +func TestLastBlockEmptyStorage(t *testing.T) { + _, _, err := LastBlock(&stubStorage{}) + require.ErrorIs(t, err, errNoGenesisBlock) +} + +// LastBlock wraps GetBlock errors. +func TestLastBlockGetBlockError(t *testing.T) { + sentinel := errors.New("disk corrupted") + storage := &stubStorage{ + blocks: make([]metadata.StateMachineBlock, 3), + errSeq: 2, + err: sentinel, + } + _, _, err := LastBlock(storage) + require.ErrorIs(t, err, sentinel) +} + +// LastBlock returns the block at seq numBlocks-1 and the block count. +func TestLastBlockSuccess(t *testing.T) { + storage := &stubStorage{ + blocks: []metadata.StateMachineBlock{ + nonSimplexBlock(0), + simplexBlock(1, 1), + }, + } + got, numBlocks, err := LastBlock(storage) + require.NoError(t, err) + require.Equal(t, uint64(2), numBlocks) + require.Equal(t, storage.blocks[1], got) +} + +// Covers each branch of getLastAcceptedEpochAndValidatorSet: genesis, +// sealing block at tip, sealing block in storage, and the error paths. +func TestGetLastAcceptedEpochAndValidatorSet(t *testing.T) { + vdrSet := testValidatorSet() + + tests := []struct { + name string + blocks []metadata.StateMachineBlock + expectedEpoch uint64 + expectedNodes common.Nodes + expectedErr error + }{ + { + name: "only non-Simplex blocks starts at first Simplex height with genesis set", + blocks: []metadata.StateMachineBlock{nonSimplexBlock(0)}, + expectedEpoch: 1, + expectedNodes: vdrSet.Nodes(), + }, + { + name: "multiple non-Simplex blocks start at first Simplex height with genesis set", + blocks: []metadata.StateMachineBlock{ + nonSimplexBlock(0), + nonSimplexBlock(1), + nonSimplexBlock(2), + }, + expectedEpoch: 3, + expectedNodes: vdrSet.Nodes(), + }, + { + name: "sealing block at tip starts next epoch with its descriptor set", + blocks: []metadata.StateMachineBlock{ + simplexBlock(1, 1), + sealingBlock(1, 2, vdrSet), + }, + expectedEpoch: 2, + expectedNodes: vdrSet.Nodes(), + }, + { + name: "non-sealing tip keeps its epoch, set loaded from sealing block at seq==epoch", + blocks: []metadata.StateMachineBlock{ + nonSimplexBlock(0), + simplexBlock(1, 1), + sealingBlock(1, 2, vdrSet), + simplexBlock(2, 3), + }, + expectedEpoch: 2, + expectedNodes: vdrSet.Nodes(), + }, + { + name: "empty storage errors", + expectedErr: errNoGenesisBlock, + }, + { + name: "non-sealing block at the sealing seq errors", + blocks: []metadata.StateMachineBlock{ + nonSimplexBlock(0), + simplexBlock(1, 1), + simplexBlock(1, 2), + simplexBlock(2, 3), + }, + expectedErr: errNonSealingBlock, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + storage := &stubStorage{blocks: tt.blocks} + config := epochTestConfig(t, storage, vdrSet) + + nodes, epoch, err := getLastAcceptedEpochAndValidatorSet(config) + if tt.expectedErr != nil { + require.ErrorIs(t, err, tt.expectedErr) + return + } + require.NoError(t, err) + require.Equal(t, tt.expectedEpoch, epoch) + require.Equal(t, tt.expectedNodes, nodes) + }) + } +}