From 3de7f279d93b850cdaf15c2af81c45e21cc01c5d Mon Sep 17 00:00:00 2001 From: hacheigriega Date: Wed, 12 Nov 2025 08:08:09 -0500 Subject: [PATCH 1/8] feat(x/batching): batch prune batches and data results Prune batches and their associated data at every block based on two new module parameters NumBatchesToKeep and MaxBatchPrunePerBlock. For pruning data results and their batch assignment data, we resort to naive implementation because there is no mapping to data result objects from batch number or data result ID. In this implementation we go through `MaxDataResultsToCheckForPrune` items in the store starting from a random point and delete those whose associated batches have been pruned. --- proto/sedachain/batching/v1/batching.proto | 3 + proto/sedachain/batching/v1/genesis.proto | 1 - testutil/integration.go | 14 + x/batching/keeper/benchmark_endblock_test.go | 42 +-- x/batching/keeper/data_result.go | 9 + x/batching/keeper/endblock.go | 96 ++----- x/batching/keeper/endblock_pruning.go | 150 ++++++++++ x/batching/keeper/endblock_test.go | 276 +++++++++++-------- x/batching/keeper/genesis.go | 9 +- x/batching/keeper/keeper.go | 11 - x/batching/types/batching.pb.go | 150 ++++++---- x/batching/types/genesis.go | 8 +- x/batching/types/genesis.pb.go | 108 +++----- x/batching/types/keys.go | 1 - x/batching/types/params.go | 10 +- x/batching/types/telemetry.go | 5 + 16 files changed, 531 insertions(+), 362 deletions(-) create mode 100644 x/batching/keeper/endblock_pruning.go create mode 100644 x/batching/types/telemetry.go diff --git a/proto/sedachain/batching/v1/batching.proto b/proto/sedachain/batching/v1/batching.proto index e31906e1..1023872e 100644 --- a/proto/sedachain/batching/v1/batching.proto +++ b/proto/sedachain/batching/v1/batching.proto @@ -94,4 +94,7 @@ message Params { // MaxBatchPrunePerBlock is the maximum number of batches to prune per // block. uint64 max_batch_prune_per_block = 2; + // MaxDataResultsToCheckForPrune is the maximum number of data results to + // check for pruning per block. + uint64 max_data_results_to_check_for_prune = 3; } diff --git a/proto/sedachain/batching/v1/genesis.proto b/proto/sedachain/batching/v1/genesis.proto index f6343cd6..9f077f3a 100644 --- a/proto/sedachain/batching/v1/genesis.proto +++ b/proto/sedachain/batching/v1/genesis.proto @@ -17,7 +17,6 @@ message GenesisState { repeated BatchAssignment batch_assignments = 5 [ (gogoproto.nullable) = false ]; Params params = 6 [ (gogoproto.nullable) = false ]; - uint64 first_batch_number = 7; } // BatchAssignment represents a batch assignment for genesis export diff --git a/testutil/integration.go b/testutil/integration.go index f1334f57..27c2243f 100644 --- a/testutil/integration.go +++ b/testutil/integration.go @@ -1,6 +1,7 @@ package testutil import ( + "crypto/rand" "fmt" "time" @@ -206,10 +207,23 @@ func (app *IntegationApp) AddTime(seconds int64) { } // AddBlock increments the block number of the application context. +// It also sets the last commit hash to a random value. func (app *IntegationApp) AddBlock() { app.ctx = app.ctx.WithBlockHeight(app.ctx.BlockHeader().Height + 1) } +func (app *IntegationApp) SetRandomLastCommitHash() { + randomBytes := make([]byte, 32) + _, err := rand.Read(randomBytes) + if err != nil { + panic(err) + } + + newHeader := app.ctx.BlockHeader() + newHeader.LastCommitHash = randomBytes + app.ctx = app.ctx.WithBlockHeader(newHeader) +} + // QueryHelper returns the application query helper. // It can be used when registering query services. func (app *IntegationApp) QueryHelper() *baseapp.QueryServiceTestHelper { diff --git a/x/batching/keeper/benchmark_endblock_test.go b/x/batching/keeper/benchmark_endblock_test.go index 6ffc5953..8a28566c 100644 --- a/x/batching/keeper/benchmark_endblock_test.go +++ b/x/batching/keeper/benchmark_endblock_test.go @@ -1,11 +1,8 @@ package keeper_test import ( - "os" - "runtime/pprof" "testing" - "github.com/sedaprotocol/seda-chain/x/batching/types" "github.com/stretchr/testify/require" ) @@ -17,12 +14,6 @@ func BenchmarkBatchPruning(b *testing.B) { numBatchesToKeep := uint64(1000) maxBatchPrunePerBlock := uint64(100) - err := f.batchingKeeper.SetParams(f.Context(), types.Params{ - NumBatchesToKeep: numBatchesToKeep, - MaxBatchPrunePerBlock: maxBatchPrunePerBlock, - }) - require.NoError(b, err) - for range numBatches { f.AddBlock() @@ -34,16 +25,35 @@ func BenchmarkBatchPruning(b *testing.B) { require.NoError(b, err) } - cpuFile, err := os.Create("cpu_refactor_new.out") - require.NoError(b, err) - defer cpuFile.Close() + for b.Loop() { + _, err := f.batchingKeeper.PruneBatches(f.Context(), numBatchesToKeep, maxBatchPrunePerBlock) + require.NoError(b, err) + } +} + +func BenchmarkDataResultPruning(b *testing.B) { + f := initFixture(b) - err = pprof.StartCPUProfile(cpuFile) - require.NoError(b, err) - defer pprof.StopCPUProfile() + maxDataResultsToCheckForPrune := uint64(100) + + // Create 10 data results for each of 1000 batches + for i := range uint64(100) { + f.AddBlock() + + dataResults := generateDataResults(b, 10) + for _, dataResult := range dataResults { + err := f.batchingKeeper.SetDataResultForBatching(f.Context(), dataResult) + require.NoError(b, err) + err = f.batchingKeeper.MarkDataResultAsBatched(f.Context(), dataResult, i) + require.NoError(b, err) + } + } for b.Loop() { - err = f.batchingKeeper.PruneBatches(f.Context()) + f.AddBlock() + f.SetRandomLastCommitHash() + + err := f.batchingKeeper.PruneDataResults(f.Context(), maxDataResultsToCheckForPrune, 2000) require.NoError(b, err) } } diff --git a/x/batching/keeper/data_result.go b/x/batching/keeper/data_result.go index c6596adc..d3f7fa9b 100644 --- a/x/batching/keeper/data_result.go +++ b/x/batching/keeper/data_result.go @@ -15,6 +15,11 @@ func (k Keeper) SetDataResultForBatching(ctx context.Context, result types.DataR return k.dataResults.Set(ctx, collections.Join3(false, result.DrId, result.DrBlockHeight), result) } +// RemoveDataResult removes a data result from the store. +func (k Keeper) RemoveDataResult(ctx context.Context, batched bool, dataReqID string, dataReqHeight uint64) error { + return k.dataResults.Remove(ctx, collections.Join3(batched, dataReqID, dataReqHeight)) +} + // MarkDataResultAsBatched removes the "unbatched" variant of the given // data result and stores a "batched" variant. func (k Keeper) MarkDataResultAsBatched(ctx context.Context, result types.DataResult, batchNum uint64) error { @@ -147,6 +152,10 @@ func (k Keeper) GetBatchAssignment(ctx context.Context, dataReqID string, dataRe return k.batchAssignments.Get(ctx, collections.Join(dataReqID, dataReqHeight)) } +func (k Keeper) RemoveBatchAssignment(ctx context.Context, dataReqID string, dataReqHeight uint64) error { + return k.batchAssignments.Remove(ctx, collections.Join(dataReqID, dataReqHeight)) +} + // getAllBatchAssignments retrieves all batch assignments from the store. // Used for genesis export. func (k Keeper) getAllBatchAssignments(ctx context.Context) ([]types.BatchAssignment, error) { diff --git a/x/batching/keeper/endblock.go b/x/batching/keeper/endblock.go index 1d517c0a..2e509d5c 100644 --- a/x/batching/keeper/endblock.go +++ b/x/batching/keeper/endblock.go @@ -10,6 +10,7 @@ import ( "cosmossdk.io/collections" "cosmossdk.io/math" + "github.com/cosmos/cosmos-sdk/telemetry" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/sedaprotocol/seda-chain/app/utils" @@ -24,97 +25,44 @@ func (k Keeper) EndBlock(ctx sdk.Context) error { if err != nil { return err } - if !isActivated { - k.Logger(ctx).Info("skip batching since proving scheme has not been activated", "index", sedatypes.SEDAKeyIndexSecp256k1) - return nil - } - batch, dataEntries, valEntries, err := k.ConstructBatch(ctx) - if err != nil { - if errors.Is(err, types.ErrNoBatchingUpdate) { + if isActivated { + batch, dataEntries, valEntries, err := k.ConstructBatch(ctx) + if err != nil { + if !errors.Is(err, types.ErrNoBatchingUpdate) { + return err + } k.Logger(ctx).Info("skip batch creation due to no update", "height", ctx.BlockHeight()) - return nil + } else { + err = k.SetNewBatch(ctx, batch, dataEntries, valEntries) + if err != nil { + return err + } } - return err - } - - err = k.SetNewBatch(ctx, batch, dataEntries, valEntries) - if err != nil { - return err + } else { + k.Logger(ctx).Info("skip batching since proving scheme has not been activated", "index", sedatypes.SEDAKeyIndexSecp256k1) } - return k.PruneBatches(ctx) -} - -// PruneBatches prunes batches and their associated data based on module -// parameters NumBatchesToKeep and MaxBatchPrunePerBlock. -func (k Keeper) PruneBatches(ctx sdk.Context) error { params, err := k.GetParams(ctx) if err != nil { return err } - // Note the current batch number here has not been used yet. - currentBatchNum, err := k.GetCurrentBatchNum(ctx) - if err != nil { - return err - } - firstBatchNum, err := k.firstBatchNumber.Get(ctx) + lastPrunedBatchNum, err := k.PruneBatches(ctx, params.NumBatchesToKeep, params.MaxBatchPrunePerBlock) if err != nil { - return err - } - if currentBatchNum-firstBatchNum <= params.NumBatchesToKeep { - k.Logger(ctx).Info("skip batch pruning", "current_batch_num", currentBatchNum, "first_batch_num", firstBatchNum) + telemetry.SetGauge(1, types.TelemetryKeyBatchingPruningFail) + k.Logger(ctx).Error("error while pruning batches", "err", err) return nil } - // Prune range is [firstBatchNum, newFirstBatchNum) - firstBatchHeight, err := k.batches.Indexes.Number.MatchExact(ctx, firstBatchNum) - if err != nil { - return err - } - - newFirstBatchNum := min(firstBatchNum+params.MaxBatchPrunePerBlock, currentBatchNum-params.NumBatchesToKeep) - newFirstBatchHeight, err := k.batchIndex.Get(ctx, newFirstBatchNum) + err = k.PruneDataResults(ctx, params.MaxDataResultsToCheckForPrune, lastPrunedBatchNum) if err != nil { - return err - } - - // Clear batches and their associated data. - batchNumRng := new(collections.Range[uint64]).StartInclusive(firstBatchNum).EndExclusive(newFirstBatchNum) - err = k.batchIndex.Clear(ctx, batchNumRng) - if err != nil { - return err - } - err = k.dataResultTreeEntries.Clear(ctx, batchNumRng) - if err != nil { - return err - } - - batchHeightRng := new(collections.Range[int64]).StartInclusive(firstBatchHeight).EndExclusive(newFirstBatchHeight) - err = k.batchesMap.Clear(ctx, batchHeightRng) - if err != nil { - return err - } - - valRng := new(collections.Range[collections.Pair[uint64, []byte]]). - StartInclusive(collections.PairPrefix[uint64, []byte](firstBatchNum)). - EndExclusive(collections.PairPrefix[uint64, []byte](newFirstBatchNum)) - err = k.validatorTreeEntries.Clear(ctx, valRng) - if err != nil { - return err - } - err = k.batchSignatures.Clear(ctx, valRng) - if err != nil { - return err - } - - err = k.firstBatchNumber.Set(ctx, newFirstBatchNum) - if err != nil { - return err + telemetry.SetGauge(1, types.TelemetryKeyBatchingPruningFail) + k.Logger(ctx).Error("error while pruning data results", "err", err) + return nil } - k.Logger(ctx).Info("successfully pruned batch data") + telemetry.SetGauge(0, types.TelemetryKeyBatchingPruningFail) return nil } diff --git a/x/batching/keeper/endblock_pruning.go b/x/batching/keeper/endblock_pruning.go new file mode 100644 index 00000000..8530b802 --- /dev/null +++ b/x/batching/keeper/endblock_pruning.go @@ -0,0 +1,150 @@ +package keeper + +import ( + "encoding/hex" + + "golang.org/x/crypto/sha3" + + "cosmossdk.io/collections" + + sdk "github.com/cosmos/cosmos-sdk/types" +) + +func (k Keeper) PruneDataResults(ctx sdk.Context, maxDataResultsToCheckForPrune, lastRemovedBatchNum uint64) error { + if maxDataResultsToCheckForPrune == 0 || lastRemovedBatchNum == 0 { + k.Logger(ctx).Info("skip data result pruning", "max_data_results_to_check_for_prune", maxDataResultsToCheckForPrune, "last_removed_batch_num", lastRemovedBatchNum) + return nil + } + + // Use hash of last commit hash as starting point of the range. + hasher := sha3.NewLegacyKeccak256() + hasher.Write(ctx.BlockHeader().LastCommitHash) + hash := hasher.Sum(nil) + + var rng *collections.Range[collections.Triple[bool, string, uint64]] + if ctx.BlockHeight()%2 == 0 { + rng = new(collections.Range[collections.Triple[bool, string, uint64]]). + StartInclusive(collections.TripleSuperPrefix[bool, string, uint64](true, hex.EncodeToString(hash))) + } else { + rng = new(collections.Range[collections.Triple[bool, string, uint64]]). + EndInclusive(collections.TripleSuperPrefix[bool, string, uint64](true, hex.EncodeToString(hash))). + Descending() + } + + iter, err := k.dataResults.Iterate(ctx, rng) + if err != nil { + return err + } + defer iter.Close() + + var numChecked, numPruned uint64 + for ; iter.Valid(); iter.Next() { + kv, err := iter.KeyValue() + if err != nil { + return err + } + + batchNum, err := k.GetBatchAssignment(ctx, kv.Value.DrId, kv.Value.DrBlockHeight) + if err != nil { + return err + } + + if batchNum <= lastRemovedBatchNum { + err = k.RemoveDataResult(ctx, true, kv.Value.DrId, kv.Value.DrBlockHeight) + if err != nil { + return err + } + err = k.RemoveBatchAssignment(ctx, kv.Value.DrId, kv.Value.DrBlockHeight) + if err != nil { + return err + } + numPruned++ + } + + numChecked++ + if numChecked == maxDataResultsToCheckForPrune { + break + } + } + + k.Logger(ctx).Info("pruned data results", "num_checked", numChecked, "num_pruned", numPruned) + return nil +} + +// PruneBatches prunes batches and their associated data based on module +// parameters NumBatchesToKeep and MaxBatchPrunePerBlock. It returns the +// batch number of the last pruned batch. +func (k Keeper) PruneBatches(ctx sdk.Context, numBatchesToKeep, maxBatchPrunePerBlock uint64) (uint64, error) { + currentBatchNum, err := k.GetCurrentBatchNum(ctx) + if err != nil { + return 0, err + } + if currentBatchNum <= numBatchesToKeep { + k.Logger(ctx).Info("skip batch pruning", "current_batch_num", currentBatchNum, "num_batches_to_keep", numBatchesToKeep) + return 0, nil + } + + rng := new(collections.Range[uint64]).EndExclusive(currentBatchNum - numBatchesToKeep) + iter, err := k.batches.Indexes.Number.Iterate(ctx, rng) + if err != nil { + return 0, err + } + defer iter.Close() + + var firstKey *collections.Pair[uint64, int64] + var pruneCount uint64 + var lastPrunedBatchNum uint64 + for ; iter.Valid(); iter.Next() { + fullKey, err := iter.FullKey() + if err != nil { + return 0, err + } + if firstKey == nil { + firstKey = &fullKey + } + + batchNum, batchHeight := fullKey.K1(), fullKey.K2() + if batchNum >= currentBatchNum-numBatchesToKeep { + // Should not happen because of the range configuration. + break + } + + err = k.batches.Remove(ctx, batchHeight) + if err != nil { + return 0, err + } + k.Logger(ctx).Info("pruned batch", "batch_num", batchNum) + + lastPrunedBatchNum = batchNum + + pruneCount++ + if pruneCount == maxBatchPrunePerBlock { + break + } + } + + if firstKey == nil { + // This means nothing was pruned. + k.Logger(ctx).Info("no batches to prune") + return 0, nil + } + + dataRng := new(collections.Range[uint64]).EndExclusive(firstKey.K1() + pruneCount) + err = k.dataResultTreeEntries.Clear(ctx, dataRng) + if err != nil { + return 0, err + } + + valRng := new(collections.Range[collections.Pair[uint64, []byte]]). + EndExclusive(collections.PairPrefix[uint64, []byte](firstKey.K1() + pruneCount)) + err = k.validatorTreeEntries.Clear(ctx, valRng) + if err != nil { + return 0, err + } + err = k.batchSignatures.Clear(ctx, valRng) + if err != nil { + return 0, err + } + + return lastPrunedBatchNum, nil +} diff --git a/x/batching/keeper/endblock_test.go b/x/batching/keeper/endblock_test.go index b78a0bc5..adedbebe 100644 --- a/x/batching/keeper/endblock_test.go +++ b/x/batching/keeper/endblock_test.go @@ -30,119 +30,6 @@ import ( "github.com/sedaprotocol/seda-chain/x/batching/types" ) -func TestBatchPruning(t *testing.T) { - f := initFixture(t) - - f.addBatchSigningValidators(t, 10) - - err := f.batchingKeeper.SetParams(f.Context(), types.Params{ - NumBatchesToKeep: 75, - MaxBatchPrunePerBlock: 150, - }) - require.NoError(t, err) - - // Create 300 batches with random associated data. - for range 300 { - f.AddBlock() - - err := f.batchingKeeper.SetDataResultForBatching(f.Context(), generateDataResults(t, 1)[0]) - require.NoError(t, err) - batch, dataEntries, valEntries, err := f.batchingKeeper.ConstructBatch(f.Context()) - require.NoError(t, err) - err = f.batchingKeeper.SetNewBatch(f.Context(), batch, dataEntries, valEntries) - require.NoError(t, err) - err = f.batchingKeeper.SetBatchSigSecp256k1(f.Context(), batch.BatchNumber, valEntries[0].ValidatorAddress, generateRandomBytes(64)) - require.NoError(t, err) - } - - batches, err := f.batchingKeeper.GetAllBatches(f.Context()) - require.NoError(t, err) - require.Equal(t, 300, len(batches)) - - // Should prune first 150 batches. - err = f.batchingKeeper.PruneBatches(f.Context()) - require.NoError(t, err) - - batches, err = f.batchingKeeper.GetAllBatches(f.Context()) - require.NoError(t, err) - require.Equal(t, 150, len(batches)) - require.Equal(t, uint64(150), batches[0].BatchNumber) - require.Equal(t, uint64(299), batches[len(batches)-1].BatchNumber) - - for i := uint64(0); i <= 149; i++ { - f.checkNoBatchData(t, i) - } - for i := uint64(150); i <= 299; i++ { - f.checkBatchData(t, i) - } - - // Should prune second 75 batches. - err = f.batchingKeeper.PruneBatches(f.Context()) - require.NoError(t, err) - - batches, err = f.batchingKeeper.GetAllBatches(f.Context()) - require.NoError(t, err) - require.Equal(t, 75, len(batches)) - require.Equal(t, uint64(225), batches[0].BatchNumber) - require.Equal(t, uint64(299), batches[len(batches)-1].BatchNumber) - - for i := 0; i <= 224; i++ { - f.checkNoBatchData(t, uint64(i)) - } - for i := 225; i <= 299; i++ { - f.checkBatchData(t, uint64(i)) - } - - // Should prune nothing. - err = f.batchingKeeper.PruneBatches(f.Context()) - require.NoError(t, err) - - batches, err = f.batchingKeeper.GetAllBatches(f.Context()) - require.NoError(t, err) - require.Equal(t, 75, len(batches)) - require.Equal(t, uint64(225), batches[0].BatchNumber) - require.Equal(t, uint64(299), batches[len(batches)-1].BatchNumber) - - for i := 0; i <= 224; i++ { - f.checkNoBatchData(t, uint64(i)) - } - for i := 225; i <= 299; i++ { - f.checkBatchData(t, uint64(i)) - } -} - -func (f *fixture) checkNoBatchData(t *testing.T, batchNum uint64) { - batch, err := f.batchingKeeper.GetBatchByBatchNumber(f.Context(), batchNum) - require.ErrorIs(t, err, collections.ErrNotFound) - dataEntries, err := f.batchingKeeper.GetDataResultTreeEntries(f.Context(), batchNum) - require.ErrorIs(t, err, collections.ErrNotFound) - valEntries, _ := f.batchingKeeper.GetValidatorTreeEntries(f.Context(), batchNum) - // require.ErrorIs(t, err, collections.ErrNotFound) // this function does not error even if there are no entries. - sigs, _ := f.batchingKeeper.GetBatchSignatures(f.Context(), batchNum) - // require.ErrorIs(t, err, collections.ErrNotFound) // this function does not error even if there are no entries. - - require.Empty(t, batch, "batchNum: %d", batchNum) - require.Empty(t, dataEntries, "batchNum: %d", batchNum) - require.Empty(t, valEntries, "batchNum: %d", batchNum) - require.Empty(t, sigs, "batchNum: %d", batchNum) -} - -func (f *fixture) checkBatchData(t *testing.T, batchNum uint64) { - batch, err := f.batchingKeeper.GetBatchByBatchNumber(f.Context(), batchNum) - require.NoError(t, err) - dataEntries, err := f.batchingKeeper.GetDataResultTreeEntries(f.Context(), batchNum) - require.NoError(t, err) - valEntries, err := f.batchingKeeper.GetValidatorTreeEntries(f.Context(), batchNum) - require.NoError(t, err) - sigs, err := f.batchingKeeper.GetBatchSignatures(f.Context(), batchNum) - require.NoError(t, err) - - require.NotEmpty(t, batch) - require.NotEmpty(t, dataEntries) - require.NotEmpty(t, valEntries) - require.NotEmpty(t, sigs) -} - func Test_ConstructDataResultTree(t *testing.T) { f := initFixture(t) @@ -890,3 +777,166 @@ func (f *fixture) addBatchSigningValidatorsFromTestData(t *testing.T, testData [ } return addrs, secp256k1PubKeys, powers } + +func TestBatchPruning(t *testing.T) { + f := initFixture(t) + + f.addBatchSigningValidators(t, 10) + + numBatchesToKeep := uint64(75) + maxBatchPrunePerBlock := uint64(150) + + // Should prune nothing. + lastRemovedBatchNum, err := f.batchingKeeper.PruneBatches(f.Context(), numBatchesToKeep, maxBatchPrunePerBlock) + require.NoError(t, err) + require.Equal(t, uint64(0), lastRemovedBatchNum) + + // Create 300 batches with random associated data. + for range 300 { + f.AddBlock() + + err := f.batchingKeeper.SetDataResultForBatching(f.Context(), generateDataResults(t, 1)[0]) + require.NoError(t, err) + batch, dataEntries, valEntries, err := f.batchingKeeper.ConstructBatch(f.Context()) + require.NoError(t, err) + err = f.batchingKeeper.SetNewBatch(f.Context(), batch, dataEntries, valEntries) + require.NoError(t, err) + err = f.batchingKeeper.SetBatchSigSecp256k1(f.Context(), batch.BatchNumber, valEntries[0].ValidatorAddress, generateRandomBytes(64)) + require.NoError(t, err) + } + + batches, err := f.batchingKeeper.GetAllBatches(f.Context()) + require.NoError(t, err) + require.Equal(t, 300, len(batches)) + + // Should prune first 150 batches. + lastRemovedBatchNum, err = f.batchingKeeper.PruneBatches(f.Context(), numBatchesToKeep, maxBatchPrunePerBlock) + require.NoError(t, err) + require.Equal(t, uint64(149), lastRemovedBatchNum) + + batches, err = f.batchingKeeper.GetAllBatches(f.Context()) + require.NoError(t, err) + require.Equal(t, 150, len(batches)) + require.Equal(t, uint64(150), batches[0].BatchNumber) + require.Equal(t, uint64(299), batches[len(batches)-1].BatchNumber) + + for i := 0; i <= 149; i++ { + f.checkNoBatchData(t, uint64(i)) + } + for i := 150; i <= 299; i++ { + f.checkBatchData(t, uint64(i)) + } + + // Should prune second 75 batches. + lastRemovedBatchNum, err = f.batchingKeeper.PruneBatches(f.Context(), numBatchesToKeep, maxBatchPrunePerBlock) + require.NoError(t, err) + require.Equal(t, uint64(224), lastRemovedBatchNum) + + batches, err = f.batchingKeeper.GetAllBatches(f.Context()) + require.NoError(t, err) + require.Equal(t, 75, len(batches)) + require.Equal(t, uint64(225), batches[0].BatchNumber) + require.Equal(t, uint64(299), batches[len(batches)-1].BatchNumber) + + for i := 0; i <= 224; i++ { + f.checkNoBatchData(t, uint64(i)) + } + for i := 225; i <= 299; i++ { + f.checkBatchData(t, uint64(i)) + } + + // Should prune nothing. + lastRemovedBatchNum, err = f.batchingKeeper.PruneBatches(f.Context(), numBatchesToKeep, maxBatchPrunePerBlock) + require.NoError(t, err) + require.Equal(t, uint64(0), lastRemovedBatchNum) + + batches, err = f.batchingKeeper.GetAllBatches(f.Context()) + require.NoError(t, err) + require.Equal(t, 75, len(batches)) + require.Equal(t, uint64(225), batches[0].BatchNumber) + require.Equal(t, uint64(299), batches[len(batches)-1].BatchNumber) + + for i := 0; i <= 224; i++ { + f.checkNoBatchData(t, uint64(i)) + } + for i := 225; i <= 299; i++ { + f.checkBatchData(t, uint64(i)) + } +} + +func (f *fixture) checkNoBatchData(t *testing.T, batchNum uint64) { + batch, err := f.batchingKeeper.GetBatchByBatchNumber(f.Context(), batchNum) + require.ErrorIs(t, err, collections.ErrNotFound) + dataEntries, err := f.batchingKeeper.GetDataResultTreeEntries(f.Context(), batchNum) + require.ErrorIs(t, err, collections.ErrNotFound) + valEntries, _ := f.batchingKeeper.GetValidatorTreeEntries(f.Context(), batchNum) + // require.ErrorIs(t, err, collections.ErrNotFound) // this function does not error even if there are no entries. + sigs, _ := f.batchingKeeper.GetBatchSignatures(f.Context(), batchNum) + // require.ErrorIs(t, err, collections.ErrNotFound) // this function does not error even if there are no entries. + + require.Empty(t, batch, "batchNum: %d", batchNum) + require.Empty(t, dataEntries, "batchNum: %d", batchNum) + require.Empty(t, valEntries, "batchNum: %d", batchNum) + require.Empty(t, sigs, "batchNum: %d", batchNum) +} + +func (f *fixture) checkBatchData(t *testing.T, batchNum uint64) { + batch, err := f.batchingKeeper.GetBatchByBatchNumber(f.Context(), batchNum) + require.NoError(t, err) + dataEntries, err := f.batchingKeeper.GetDataResultTreeEntries(f.Context(), batchNum) + require.NoError(t, err) + valEntries, err := f.batchingKeeper.GetValidatorTreeEntries(f.Context(), batchNum) + require.NoError(t, err) + sigs, err := f.batchingKeeper.GetBatchSignatures(f.Context(), batchNum) + require.NoError(t, err) + + require.NotEmpty(t, batch) + require.NotEmpty(t, dataEntries) + require.NotEmpty(t, valEntries) + require.NotEmpty(t, sigs) +} + +func TestDataResultPruning(t *testing.T) { + f := initFixture(t) + + maxDataResultsToCheckForPrune := uint64(100) + + // Should prune nothing. + err := f.batchingKeeper.PruneDataResults(f.Context(), maxDataResultsToCheckForPrune, 0) + require.NoError(t, err) + + // Create 10 data results for each of 100 batches + for i := range uint64(100) { + f.AddBlock() + + dataResults := generateDataResults(t, 10) + for _, dataResult := range dataResults { + err := f.batchingKeeper.SetDataResultForBatching(f.Context(), dataResult) + require.NoError(t, err) + err = f.batchingKeeper.MarkDataResultAsBatched(f.Context(), dataResult, i) + require.NoError(t, err) + } + } + + dataResults, err := f.batchingKeeper.GetDataResults(f.Context(), true) + require.NoError(t, err) + require.Equal(t, 1000, len(dataResults)) + + i := 0 + for ; i < 30; i++ { + f.AddBlock() + f.SetRandomLastCommitHash() + + err = f.batchingKeeper.PruneDataResults(f.Context(), maxDataResultsToCheckForPrune, uint64(25+25*i)) + require.NoError(t, err) + + dataResults, err = f.batchingKeeper.GetDataResults(f.Context(), true) + require.NoError(t, err) + if len(dataResults) == 0 { + break + } + } + + require.Equal(t, 0, len(dataResults)) + t.Logf("test completed after %d iterations", i) +} diff --git a/x/batching/keeper/genesis.go b/x/batching/keeper/genesis.go index c267191a..3bf174c4 100644 --- a/x/batching/keeper/genesis.go +++ b/x/batching/keeper/genesis.go @@ -14,9 +14,6 @@ func (k Keeper) InitGenesis(ctx sdk.Context, data types.GenesisState) { if err != nil { panic(err) } - if err := k.firstBatchNumber.Set(ctx, data.FirstBatchNumber); err != nil { - panic(err) - } for _, batch := range data.Batches { err := k.setBatch(ctx, batch) if err != nil { @@ -72,10 +69,6 @@ func (k Keeper) ExportGenesis(ctx sdk.Context) types.GenesisState { if err != nil { panic(err) } - firstBatchNumber, err := k.firstBatchNumber.Get(ctx) - if err != nil { - panic(err) - } batches, err := k.GetAllBatches(ctx) if err != nil { panic(err) @@ -92,5 +85,5 @@ func (k Keeper) ExportGenesis(ctx sdk.Context) types.GenesisState { if err != nil { panic(err) } - return types.NewGenesisState(curBatchNum, firstBatchNumber, batches, batchData, dataResults, batchAssignments, params) + return types.NewGenesisState(curBatchNum, batches, batchData, dataResults, batchAssignments, params) } diff --git a/x/batching/keeper/keeper.go b/x/batching/keeper/keeper.go index d7bbdef1..8159f366 100644 --- a/x/batching/keeper/keeper.go +++ b/x/batching/keeper/keeper.go @@ -35,15 +35,10 @@ type Keeper struct { batchAssignments collections.Map[collections.Pair[string, uint64], uint64] currentBatchNumber collections.Sequence batches *collections.IndexedMap[int64, types.Batch, BatchIndexes] - firstBatchNumber collections.Item[uint64] validatorTreeEntries collections.Map[collections.Pair[uint64, []byte], types.ValidatorTreeEntry] dataResultTreeEntries collections.Map[uint64, types.DataResultTreeEntries] batchSignatures collections.Map[collections.Pair[uint64, []byte], types.BatchSignatures] params collections.Item[types.Params] - - // Additional maps for efficient pruning - batchesMap collections.Map[int64, types.Batch] - batchIndex collections.Map[uint64, int64] } func NewKeeper( @@ -73,7 +68,6 @@ func NewKeeper( batchAssignments: collections.NewMap(sb, types.BatchAssignmentsPrefix, "batch_assignments", collections.PairKeyCodec(collections.StringKey, collections.Uint64Key), collections.Uint64Value), currentBatchNumber: collections.NewSequence(sb, types.CurrentBatchNumberKey, "current_batch_number"), batches: collections.NewIndexedMap(sb, types.BatchesKeyPrefix, "batches", collections.Int64Key, codec.CollValue[types.Batch](cdc), NewBatchIndexes(sb)), - firstBatchNumber: collections.NewItem(sb, types.FirstBatchNumberKey, "first_batch_number", collections.Uint64Value), validatorTreeEntries: collections.NewMap(sb, types.ValidatorTreeEntriesKeyPrefix, "validator_tree_entries", collections.PairKeyCodec(collections.Uint64Key, collections.BytesKey), codec.CollValue[types.ValidatorTreeEntry](cdc)), dataResultTreeEntries: collections.NewMap(sb, types.DataResultTreeEntriesKeyPrefix, "data_result_tree_entries", collections.Uint64Key, codec.CollValue[types.DataResultTreeEntries](cdc)), batchSignatures: collections.NewMap(sb, types.BatchSignaturesKeyPrefix, "batch_signatures", collections.PairKeyCodec(collections.Uint64Key, collections.BytesKey), codec.CollValue[types.BatchSignatures](cdc)), @@ -86,11 +80,6 @@ func NewKeeper( } k.Schema = schema - // Additional maps for efficient pruning - sbTemp := collections.NewSchemaBuilder(storeService) - k.batchesMap = collections.NewMap(sbTemp, types.BatchesKeyPrefix, "batches", collections.Int64Key, codec.CollValue[types.Batch](cdc)) - k.batchIndex = collections.NewMap(sbTemp, types.BatchNumberKeyPrefix, "batch_by_number", collections.Uint64Key, collections.Int64Value) - return k } diff --git a/x/batching/types/batching.pb.go b/x/batching/types/batching.pb.go index 0eb5680f..35aa4453 100644 --- a/x/batching/types/batching.pb.go +++ b/x/batching/types/batching.pb.go @@ -441,6 +441,9 @@ type Params struct { // MaxBatchPrunePerBlock is the maximum number of batches to prune per // block. MaxBatchPrunePerBlock uint64 `protobuf:"varint,2,opt,name=max_batch_prune_per_block,json=maxBatchPrunePerBlock,proto3" json:"max_batch_prune_per_block,omitempty"` + // MaxDataResultsToCheckForPrune is the maximum number of data results to + // check for pruning per block. + MaxDataResultsToCheckForPrune uint64 `protobuf:"varint,3,opt,name=max_data_results_to_check_for_prune,json=maxDataResultsToCheckForPrune,proto3" json:"max_data_results_to_check_for_prune,omitempty"` } func (m *Params) Reset() { *m = Params{} } @@ -490,6 +493,13 @@ func (m *Params) GetMaxBatchPrunePerBlock() uint64 { return 0 } +func (m *Params) GetMaxDataResultsToCheckForPrune() uint64 { + if m != nil { + return m.MaxDataResultsToCheckForPrune + } + return 0 +} + func init() { proto.RegisterType((*Batch)(nil), "sedachain.batching.v1.Batch") proto.RegisterType((*DataResultTreeEntries)(nil), "sedachain.batching.v1.DataResultTreeEntries") @@ -504,61 +514,64 @@ func init() { } var fileDescriptor_5b2a028024867de2 = []byte{ - // 864 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x55, 0xcd, 0x6e, 0xeb, 0x44, - 0x14, 0xae, 0xd3, 0x34, 0x3f, 0x93, 0xa4, 0x0d, 0xd3, 0x1b, 0xe4, 0xde, 0x45, 0x1c, 0x22, 0xae, - 0x14, 0x40, 0x49, 0x08, 0x11, 0x3f, 0x12, 0x6c, 0x30, 0x20, 0x51, 0xa1, 0x7b, 0x15, 0x0d, 0xe5, - 0x2e, 0x58, 0x60, 0x4d, 0x3c, 0x23, 0xc7, 0x4a, 0xec, 0x31, 0x33, 0xe3, 0xd0, 0xbe, 0x05, 0x2f, - 0xc0, 0x86, 0x67, 0xe0, 0x1d, 0x60, 0x79, 0xc5, 0x0a, 0x81, 0x64, 0xa1, 0x76, 0x97, 0x47, 0x60, - 0x85, 0x7c, 0xc6, 0x71, 0xda, 0xb2, 0x66, 0xe5, 0x39, 0xdf, 0x77, 0xce, 0x99, 0x73, 0x66, 0xbe, - 0x33, 0x46, 0x6f, 0x2a, 0xce, 0xa8, 0xbf, 0xa2, 0x61, 0x3c, 0x5d, 0x52, 0xed, 0xaf, 0xc2, 0x38, - 0x98, 0x6e, 0x67, 0xe5, 0x7a, 0x92, 0x48, 0xa1, 0x05, 0xee, 0x95, 0x5e, 0x93, 0x92, 0xd9, 0xce, - 0x9e, 0x5e, 0xf8, 0x42, 0x45, 0x42, 0x79, 0xe0, 0x34, 0x35, 0x86, 0x89, 0x78, 0xfa, 0x24, 0x10, - 0x81, 0x30, 0x78, 0xbe, 0x32, 0xe8, 0xf0, 0xa7, 0x0a, 0x3a, 0x71, 0xf3, 0x04, 0xf8, 0x0d, 0xd4, - 0x86, 0x4c, 0x5e, 0x9c, 0x46, 0x4b, 0x2e, 0x6d, 0x6b, 0x60, 0x8d, 0xaa, 0xa4, 0x05, 0xd8, 0x0b, - 0x80, 0xc0, 0x65, 0x23, 0xfc, 0xb5, 0xb7, 0xe2, 0x61, 0xb0, 0xd2, 0x76, 0x65, 0x60, 0x8d, 0x8e, - 0x49, 0x0b, 0xb0, 0x2f, 0x01, 0xc2, 0x1f, 0x22, 0xdb, 0x4f, 0xa5, 0xe4, 0xb1, 0xf6, 0x18, 0xd5, - 0xd4, 0x93, 0x5c, 0xa5, 0x1b, 0xed, 0x49, 0x21, 0xb4, 0x7d, 0x3c, 0xb0, 0x46, 0x4d, 0xd2, 0x2b, - 0xf8, 0xcf, 0xa9, 0xa6, 0x04, 0x58, 0x22, 0x84, 0xc6, 0x23, 0xd4, 0xfd, 0x4f, 0x40, 0x15, 0x02, - 0x4e, 0xd9, 0x43, 0xcf, 0x67, 0xe8, 0x74, 0x4b, 0x37, 0x21, 0xa3, 0x5a, 0x48, 0xe3, 0x77, 0x02, - 0x7e, 0x9d, 0x12, 0x05, 0xb7, 0x0b, 0xd4, 0x30, 0xfd, 0x84, 0xcc, 0xae, 0x0d, 0xac, 0x51, 0x9b, - 0xd4, 0xc1, 0xbe, 0x64, 0xf8, 0x2d, 0xd4, 0x4d, 0xa4, 0xd8, 0x86, 0x71, 0xe0, 0x45, 0x5c, 0xd3, - 0x3c, 0xbf, 0x5d, 0x07, 0x97, 0xb3, 0x02, 0x7f, 0x5e, 0xc0, 0xc3, 0x19, 0xea, 0x1d, 0x0a, 0xbd, - 0x92, 0x9c, 0x7f, 0x11, 0x6b, 0x19, 0x72, 0x85, 0x6d, 0x54, 0xe7, 0x66, 0x69, 0x5b, 0x83, 0xe3, - 0x3c, 0x7b, 0x61, 0x0e, 0x7f, 0xb5, 0x10, 0x7e, 0xb9, 0x2f, 0x65, 0x1f, 0x72, 0x83, 0xbf, 0x43, - 0xaf, 0x1d, 0xca, 0xa6, 0x8c, 0x49, 0xae, 0x14, 0x1c, 0x72, 0xdb, 0x9d, 0xfd, 0x93, 0x39, 0xe3, - 0x20, 0xd4, 0xab, 0x74, 0x39, 0xf1, 0x45, 0x54, 0xdc, 0x5b, 0xf1, 0x19, 0x2b, 0xb6, 0x9e, 0xea, - 0x9b, 0x84, 0xab, 0xc9, 0x4b, 0xba, 0xf9, 0xd4, 0x04, 0x92, 0x6e, 0x99, 0xab, 0x40, 0xf0, 0xbb, - 0xe8, 0xc9, 0x56, 0xe8, 0xbc, 0xa7, 0x44, 0xfc, 0xc0, 0xa5, 0x97, 0x70, 0xe9, 0xf3, 0xd8, 0x5c, - 0x52, 0x87, 0x60, 0xc3, 0x2d, 0x72, 0x6a, 0x61, 0x18, 0xec, 0xa0, 0x16, 0xd7, 0xab, 0xb2, 0x96, - 0x63, 0x38, 0x01, 0xc4, 0xf5, 0xaa, 0x48, 0x39, 0xfc, 0xd9, 0x42, 0x67, 0x20, 0x8e, 0xaf, 0xc3, - 0x20, 0xa6, 0x3a, 0x95, 0x5c, 0xfd, 0xef, 0x6d, 0x4c, 0xd1, 0xb9, 0xe2, 0x7e, 0xf2, 0xde, 0xfb, - 0x1f, 0xac, 0x67, 0x9e, 0xda, 0xef, 0x0b, 0x5d, 0xb4, 0x09, 0x2e, 0xa9, 0xb2, 0xa2, 0xe1, 0x5f, - 0x55, 0x84, 0x0e, 0x57, 0x84, 0x5f, 0x47, 0x95, 0x90, 0x41, 0x41, 0x4d, 0xb7, 0xb6, 0xcb, 0x9c, - 0x4a, 0xc8, 0x48, 0x25, 0x64, 0xb8, 0x8f, 0x4e, 0x98, 0xcc, 0xb5, 0x50, 0x01, 0xaa, 0xb9, 0xcb, - 0x1c, 0x03, 0x90, 0x2a, 0x93, 0x97, 0x0c, 0x7f, 0x8c, 0xce, 0x98, 0xf4, 0x1e, 0xc8, 0x3b, 0x3f, - 0x90, 0xaa, 0x7b, 0xbe, 0xcb, 0x9c, 0xc7, 0x14, 0xe9, 0x30, 0xe9, 0xde, 0x53, 0xfd, 0x33, 0x54, - 0xdf, 0x72, 0xa9, 0x42, 0x11, 0x1b, 0xcd, 0xba, 0xad, 0x5d, 0xe6, 0xec, 0x21, 0xb2, 0x5f, 0xe0, - 0xf9, 0xa3, 0xf9, 0x39, 0x81, 0x0d, 0xba, 0xbb, 0xcc, 0x79, 0x80, 0x3f, 0x9c, 0xa8, 0x4f, 0xd0, - 0x99, 0x21, 0x75, 0x18, 0x71, 0xa5, 0x69, 0x94, 0x80, 0x9c, 0x8b, 0xc2, 0x1e, 0x51, 0xe4, 0x14, - 0x80, 0xab, 0xbd, 0x8d, 0xdf, 0x46, 0x4d, 0x7e, 0x1d, 0x6a, 0xcf, 0x17, 0x8c, 0x83, 0xc6, 0x3b, - 0x6e, 0x67, 0x97, 0x39, 0x07, 0x90, 0x34, 0xf2, 0xe5, 0x67, 0x82, 0x71, 0xfc, 0x02, 0x35, 0x02, - 0xaa, 0xbc, 0x54, 0x71, 0x66, 0x37, 0xa0, 0x8d, 0xf9, 0x9f, 0x99, 0xd3, 0x33, 0xf7, 0xa7, 0xd8, - 0x7a, 0x12, 0x8a, 0x69, 0x44, 0xf5, 0x6a, 0x72, 0x19, 0xeb, 0x5d, 0xe6, 0x94, 0xce, 0xbf, 0xff, - 0x32, 0x46, 0xc5, 0x53, 0x73, 0x19, 0x6b, 0x52, 0x0f, 0xa8, 0xfa, 0x46, 0x71, 0x86, 0x87, 0xa8, - 0x66, 0xa6, 0xd9, 0x6e, 0x82, 0x3e, 0xd0, 0x2e, 0x73, 0x0a, 0x84, 0x14, 0xdf, 0xbc, 0xbb, 0x84, - 0xde, 0x2c, 0xa9, 0xbf, 0x2e, 0xc5, 0x84, 0x60, 0x6b, 0xe8, 0xee, 0x11, 0x45, 0x4e, 0x0b, 0x60, - 0x2f, 0x96, 0x39, 0x6a, 0xe7, 0xef, 0xa0, 0x97, 0xd0, 0x9b, 0x8d, 0xa0, 0xcc, 0x6e, 0x41, 0x28, - 0x1c, 0xe8, 0x7d, 0x9c, 0xb4, 0x72, 0x6b, 0x61, 0x0c, 0xfc, 0x0e, 0x6a, 0xfa, 0x22, 0x56, 0x3c, - 0x56, 0xa9, 0xb2, 0xdb, 0x03, 0x6b, 0xd4, 0x30, 0x47, 0x52, 0x82, 0xe4, 0xb0, 0x1c, 0x7e, 0x8f, - 0x6a, 0x0b, 0x2a, 0x69, 0xa4, 0xf0, 0x18, 0x9d, 0xc7, 0x69, 0xe4, 0xc1, 0x1b, 0xc2, 0x95, 0xa7, - 0x85, 0xb7, 0xe6, 0x3c, 0x29, 0x9e, 0xc9, 0x6e, 0x9c, 0x46, 0xae, 0x61, 0xae, 0xc4, 0x57, 0x9c, - 0x27, 0xf8, 0x23, 0x74, 0x11, 0xd1, 0x6b, 0xe3, 0xee, 0x25, 0x32, 0x8d, 0x79, 0x3e, 0x91, 0x46, - 0x45, 0xa0, 0xc1, 0x2a, 0xe9, 0x45, 0xf4, 0x1a, 0x82, 0x16, 0x39, 0xbd, 0xe0, 0x46, 0x52, 0xee, - 0xf3, 0xdf, 0x6e, 0xfb, 0xd6, 0xab, 0xdb, 0xbe, 0xf5, 0xf7, 0x6d, 0xdf, 0xfa, 0xf1, 0xae, 0x7f, - 0xf4, 0xea, 0xae, 0x7f, 0xf4, 0xc7, 0x5d, 0xff, 0xe8, 0xdb, 0xf9, 0xbd, 0xe1, 0xca, 0x3b, 0x82, - 0x27, 0xdc, 0x17, 0x1b, 0x30, 0xc6, 0xe6, 0x9f, 0x71, 0x7d, 0xf8, 0x6b, 0xc0, 0xb4, 0x2d, 0x6b, - 0xe0, 0x35, 0xff, 0x37, 0x00, 0x00, 0xff, 0xff, 0x45, 0xd9, 0xf2, 0x15, 0x58, 0x06, 0x00, 0x00, + // 904 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x55, 0xcd, 0x8e, 0xe3, 0x44, + 0x10, 0x5e, 0x67, 0x32, 0xf9, 0xe9, 0x24, 0x33, 0xa1, 0x67, 0x83, 0x3c, 0x2b, 0x11, 0x87, 0xc0, + 0x4a, 0x01, 0x94, 0x84, 0x10, 0xf1, 0x23, 0xc1, 0x85, 0x2c, 0x20, 0x06, 0xb4, 0xab, 0xa8, 0x19, + 0xf6, 0xc0, 0x01, 0xab, 0xe3, 0x6e, 0x62, 0x2b, 0xb1, 0xdb, 0xea, 0x6e, 0x87, 0xcc, 0x5b, 0xf0, + 0x02, 0x5c, 0x78, 0x06, 0xae, 0x9c, 0xe1, 0xb8, 0xe2, 0x84, 0x40, 0xb2, 0xd0, 0xcc, 0x2d, 0x8f, + 0xc0, 0x09, 0xb9, 0xda, 0x71, 0x66, 0x86, 0x33, 0x27, 0x77, 0x7d, 0x5f, 0x55, 0x75, 0x55, 0xf7, + 0x57, 0x6d, 0xf4, 0xba, 0xe2, 0x8c, 0x7a, 0x3e, 0x0d, 0xa2, 0xf1, 0x82, 0x6a, 0xcf, 0x0f, 0xa2, + 0xe5, 0x78, 0x33, 0x29, 0xd6, 0xa3, 0x58, 0x0a, 0x2d, 0x70, 0xa7, 0xf0, 0x1a, 0x15, 0xcc, 0x66, + 0xf2, 0xe8, 0xdc, 0x13, 0x2a, 0x14, 0xca, 0x05, 0xa7, 0xb1, 0x31, 0x4c, 0xc4, 0xa3, 0x87, 0x4b, + 0xb1, 0x14, 0x06, 0xcf, 0x56, 0x06, 0xed, 0xff, 0x58, 0x42, 0xc7, 0xb3, 0x2c, 0x01, 0x7e, 0x15, + 0x35, 0x21, 0x93, 0x1b, 0x25, 0xe1, 0x82, 0x4b, 0xdb, 0xea, 0x59, 0x83, 0x32, 0x69, 0x00, 0xf6, + 0x0c, 0x20, 0x70, 0x59, 0x0b, 0x6f, 0xe5, 0xfa, 0x3c, 0x58, 0xfa, 0xda, 0x2e, 0xf5, 0xac, 0xc1, + 0x11, 0x69, 0x00, 0xf6, 0x39, 0x40, 0xf8, 0x7d, 0x64, 0x7b, 0x89, 0x94, 0x3c, 0xd2, 0x2e, 0xa3, + 0x9a, 0xba, 0x92, 0xab, 0x64, 0xad, 0x5d, 0x29, 0x84, 0xb6, 0x8f, 0x7a, 0xd6, 0xa0, 0x4e, 0x3a, + 0x39, 0xff, 0x09, 0xd5, 0x94, 0x00, 0x4b, 0x84, 0xd0, 0x78, 0x80, 0xda, 0xff, 0x09, 0x28, 0x43, + 0xc0, 0x09, 0xbb, 0xeb, 0xf9, 0x18, 0x9d, 0x6c, 0xe8, 0x3a, 0x60, 0x54, 0x0b, 0x69, 0xfc, 0x8e, + 0xc1, 0xaf, 0x55, 0xa0, 0xe0, 0x76, 0x8e, 0x6a, 0xa6, 0x9f, 0x80, 0xd9, 0x95, 0x9e, 0x35, 0x68, + 0x92, 0x2a, 0xd8, 0x17, 0x0c, 0xbf, 0x81, 0xda, 0xb1, 0x14, 0x9b, 0x20, 0x5a, 0xba, 0x21, 0xd7, + 0x34, 0xcb, 0x6f, 0x57, 0xc1, 0xe5, 0x34, 0xc7, 0x9f, 0xe6, 0x70, 0x7f, 0x82, 0x3a, 0x87, 0x42, + 0x2f, 0x25, 0xe7, 0x9f, 0x46, 0x5a, 0x06, 0x5c, 0x61, 0x1b, 0x55, 0xb9, 0x59, 0xda, 0x56, 0xef, + 0x28, 0xcb, 0x9e, 0x9b, 0xfd, 0x5f, 0x2d, 0x84, 0x9f, 0xef, 0x4b, 0xd9, 0x87, 0x5c, 0xe1, 0x6f, + 0xd1, 0x4b, 0x87, 0xb2, 0x29, 0x63, 0x92, 0x2b, 0x05, 0x87, 0xdc, 0x9c, 0x4d, 0xfe, 0x49, 0x9d, + 0xe1, 0x32, 0xd0, 0x7e, 0xb2, 0x18, 0x79, 0x22, 0xcc, 0xef, 0x2d, 0xff, 0x0c, 0x15, 0x5b, 0x8d, + 0xf5, 0x55, 0xcc, 0xd5, 0xe8, 0x39, 0x5d, 0x7f, 0x6c, 0x02, 0x49, 0xbb, 0xc8, 0x95, 0x23, 0xf8, + 0x6d, 0xf4, 0x70, 0x23, 0x74, 0xd6, 0x53, 0x2c, 0xbe, 0xe7, 0xd2, 0x8d, 0xb9, 0xf4, 0x78, 0x64, + 0x2e, 0xa9, 0x45, 0xb0, 0xe1, 0xe6, 0x19, 0x35, 0x37, 0x0c, 0x76, 0x50, 0x83, 0x6b, 0xbf, 0xa8, + 0xe5, 0x08, 0x4e, 0x00, 0x71, 0xed, 0xe7, 0x29, 0xfb, 0x3f, 0x59, 0xe8, 0x14, 0xc4, 0xf1, 0x55, + 0xb0, 0x8c, 0xa8, 0x4e, 0x24, 0x57, 0xff, 0x7b, 0x1b, 0x63, 0x74, 0xa6, 0xb8, 0x17, 0xbf, 0xf3, + 0xee, 0x7b, 0xab, 0x89, 0xab, 0xf6, 0xfb, 0x42, 0x17, 0x4d, 0x82, 0x0b, 0xaa, 0xa8, 0xa8, 0xff, + 0x57, 0x19, 0xa1, 0xc3, 0x15, 0xe1, 0x97, 0x51, 0x29, 0x60, 0x50, 0x50, 0x7d, 0x56, 0xd9, 0xa5, + 0x4e, 0x29, 0x60, 0xa4, 0x14, 0x30, 0xdc, 0x45, 0xc7, 0x4c, 0x66, 0x5a, 0x28, 0x01, 0x55, 0xdf, + 0xa5, 0x8e, 0x01, 0x48, 0x99, 0xc9, 0x0b, 0x86, 0x3f, 0x44, 0xa7, 0x4c, 0xba, 0x77, 0xe4, 0x9d, + 0x1d, 0x48, 0x79, 0x76, 0xb6, 0x4b, 0x9d, 0xfb, 0x14, 0x69, 0x31, 0x39, 0xbb, 0xa5, 0xfa, 0xc7, + 0xa8, 0xba, 0xe1, 0x52, 0x05, 0x22, 0x32, 0x9a, 0x9d, 0x35, 0x76, 0xa9, 0xb3, 0x87, 0xc8, 0x7e, + 0x81, 0xa7, 0xf7, 0xe6, 0xe7, 0x18, 0x36, 0x68, 0xef, 0x52, 0xe7, 0x0e, 0x7e, 0x77, 0xa2, 0x3e, + 0x42, 0xa7, 0x86, 0xd4, 0x41, 0xc8, 0x95, 0xa6, 0x61, 0x0c, 0x72, 0xce, 0x0b, 0xbb, 0x47, 0x91, + 0x13, 0x00, 0x2e, 0xf7, 0x36, 0x7e, 0x13, 0xd5, 0xf9, 0x36, 0xd0, 0xae, 0x27, 0x18, 0x07, 0x8d, + 0xb7, 0x66, 0xad, 0x5d, 0xea, 0x1c, 0x40, 0x52, 0xcb, 0x96, 0x4f, 0x04, 0xe3, 0xf8, 0x19, 0xaa, + 0x2d, 0xa9, 0x72, 0x13, 0xc5, 0x99, 0x5d, 0x83, 0x36, 0xa6, 0x7f, 0xa6, 0x4e, 0xc7, 0xdc, 0x9f, + 0x62, 0xab, 0x51, 0x20, 0xc6, 0x21, 0xd5, 0xfe, 0xe8, 0x22, 0xd2, 0xbb, 0xd4, 0x29, 0x9c, 0x7f, + 0xff, 0x79, 0x88, 0xf2, 0xa7, 0xe6, 0x22, 0xd2, 0xa4, 0xba, 0xa4, 0xea, 0x6b, 0xc5, 0x19, 0xee, + 0xa3, 0x8a, 0x99, 0x66, 0xbb, 0x0e, 0xfa, 0x40, 0xbb, 0xd4, 0xc9, 0x11, 0x92, 0x7f, 0xb3, 0xee, + 0x62, 0x7a, 0xb5, 0xa0, 0xde, 0xaa, 0x10, 0x13, 0x82, 0xad, 0xa1, 0xbb, 0x7b, 0x14, 0x39, 0xc9, + 0x81, 0xbd, 0x58, 0xa6, 0xa8, 0x99, 0xbd, 0x83, 0x6e, 0x4c, 0xaf, 0xd6, 0x82, 0x32, 0xbb, 0x01, + 0xa1, 0x70, 0xa0, 0xb7, 0x71, 0xd2, 0xc8, 0xac, 0xb9, 0x31, 0xf0, 0x5b, 0xa8, 0xee, 0x89, 0x48, + 0xf1, 0x48, 0x25, 0xca, 0x6e, 0xf6, 0xac, 0x41, 0xcd, 0x1c, 0x49, 0x01, 0x92, 0xc3, 0xb2, 0xff, + 0x8b, 0x85, 0x2a, 0x73, 0x2a, 0x69, 0xa8, 0xf0, 0x10, 0x9d, 0x45, 0x49, 0xe8, 0xc2, 0x23, 0xc2, + 0x95, 0xab, 0x85, 0xbb, 0xe2, 0x3c, 0xce, 0xdf, 0xc9, 0x76, 0x94, 0x84, 0x33, 0xc3, 0x5c, 0x8a, + 0x2f, 0x39, 0x8f, 0xf1, 0x07, 0xe8, 0x3c, 0xa4, 0x5b, 0xe3, 0xee, 0xc6, 0x32, 0x89, 0x78, 0x36, + 0x92, 0x46, 0x46, 0x20, 0xc2, 0x32, 0xe9, 0x84, 0x74, 0x0b, 0x41, 0xf3, 0x8c, 0x9e, 0x73, 0xa3, + 0x29, 0xfc, 0x05, 0x7a, 0x2d, 0x8b, 0xbc, 0xf5, 0x1c, 0xc2, 0x6e, 0x9e, 0xcf, 0xbd, 0x95, 0xfb, + 0x9d, 0x90, 0x26, 0x9b, 0x91, 0x27, 0x79, 0x25, 0xa4, 0xdb, 0x83, 0xfc, 0xd5, 0xa5, 0x78, 0x92, + 0xb9, 0x7d, 0x26, 0x24, 0xe4, 0x9c, 0x3d, 0xfd, 0xed, 0xba, 0x6b, 0xbd, 0xb8, 0xee, 0x5a, 0x7f, + 0x5f, 0x77, 0xad, 0x1f, 0x6e, 0xba, 0x0f, 0x5e, 0xdc, 0x74, 0x1f, 0xfc, 0x71, 0xd3, 0x7d, 0xf0, + 0xcd, 0xf4, 0xd6, 0xa4, 0x66, 0xc7, 0x03, 0xff, 0x03, 0x4f, 0xac, 0xc1, 0x18, 0x9a, 0x1f, 0xd0, + 0xf6, 0xf0, 0x0b, 0x82, 0xd1, 0x5d, 0x54, 0xc0, 0x6b, 0xfa, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, + 0xc7, 0x3e, 0x8b, 0x07, 0xa5, 0x06, 0x00, 0x00, } func (m *Batch) Marshal() (dAtA []byte, err error) { @@ -867,6 +880,11 @@ func (m *Params) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.MaxDataResultsToCheckForPrune != 0 { + i = encodeVarintBatching(dAtA, i, uint64(m.MaxDataResultsToCheckForPrune)) + i-- + dAtA[i] = 0x18 + } if m.MaxBatchPrunePerBlock != 0 { i = encodeVarintBatching(dAtA, i, uint64(m.MaxBatchPrunePerBlock)) i-- @@ -1042,6 +1060,9 @@ func (m *Params) Size() (n int) { if m.MaxBatchPrunePerBlock != 0 { n += 1 + sovBatching(uint64(m.MaxBatchPrunePerBlock)) } + if m.MaxDataResultsToCheckForPrune != 0 { + n += 1 + sovBatching(uint64(m.MaxDataResultsToCheckForPrune)) + } return n } @@ -2083,6 +2104,25 @@ func (m *Params) Unmarshal(dAtA []byte) error { break } } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field MaxDataResultsToCheckForPrune", wireType) + } + m.MaxDataResultsToCheckForPrune = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBatching + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.MaxDataResultsToCheckForPrune |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipBatching(dAtA[iNdEx:]) diff --git a/x/batching/types/genesis.go b/x/batching/types/genesis.go index e2248b44..cae91cc8 100644 --- a/x/batching/types/genesis.go +++ b/x/batching/types/genesis.go @@ -13,7 +13,6 @@ import ( // NewGenesisState constructs a GenesisState object. func NewGenesisState( curBatchNum uint64, - firstBatchNumber uint64, batches []Batch, batchData []BatchData, dataResults []GenesisDataResult, @@ -22,7 +21,6 @@ func NewGenesisState( ) GenesisState { return GenesisState{ CurrentBatchNumber: curBatchNum, - FirstBatchNumber: firstBatchNumber, Batches: batches, BatchData: batchData, DataResults: dataResults, @@ -33,16 +31,12 @@ func NewGenesisState( // DefaultGenesisState creates a default GenesisState object. func DefaultGenesisState() *GenesisState { - state := NewGenesisState(collections.DefaultSequenceStart, 0, nil, nil, nil, nil, DefaultParams()) + state := NewGenesisState(collections.DefaultSequenceStart, nil, nil, nil, nil, DefaultParams()) return &state } // ValidateGenesis validates batching genesis data. func ValidateGenesis(gs GenesisState) error { - if gs.CurrentBatchNumber != uint64(len(gs.Batches)) { - return fmt.Errorf("current batch number %d should be equal to number of batches %d", gs.CurrentBatchNumber, len(gs.Batches)) - } - for _, batch := range gs.Batches { if batch.BatchNumber > gs.CurrentBatchNumber { return fmt.Errorf("batch number %d should not exceed current batch number %d", batch.BatchNumber, gs.CurrentBatchNumber) diff --git a/x/batching/types/genesis.pb.go b/x/batching/types/genesis.pb.go index 0a1c8be6..1c0170d4 100644 --- a/x/batching/types/genesis.pb.go +++ b/x/batching/types/genesis.pb.go @@ -33,7 +33,6 @@ type GenesisState struct { DataResults []GenesisDataResult `protobuf:"bytes,4,rep,name=data_results,json=dataResults,proto3" json:"data_results"` BatchAssignments []BatchAssignment `protobuf:"bytes,5,rep,name=batch_assignments,json=batchAssignments,proto3" json:"batch_assignments"` Params Params `protobuf:"bytes,6,opt,name=params,proto3" json:"params"` - FirstBatchNumber uint64 `protobuf:"varint,7,opt,name=first_batch_number,json=firstBatchNumber,proto3" json:"first_batch_number,omitempty"` } func (m *GenesisState) Reset() { *m = GenesisState{} } @@ -111,13 +110,6 @@ func (m *GenesisState) GetParams() Params { return Params{} } -func (m *GenesisState) GetFirstBatchNumber() uint64 { - if m != nil { - return m.FirstBatchNumber - } - return 0 -} - // BatchAssignment represents a batch assignment for genesis export // and import. type BatchAssignment struct { @@ -314,43 +306,42 @@ func init() { } var fileDescriptor_eccca5d98d3cb479 = []byte{ - // 567 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xc1, 0x6e, 0xd3, 0x40, - 0x10, 0x8d, 0x9b, 0x90, 0x92, 0x49, 0x50, 0x9a, 0xa5, 0x48, 0x56, 0x05, 0x26, 0x0d, 0xa8, 0x0a, - 0x52, 0x71, 0x68, 0x7b, 0x84, 0x0b, 0x11, 0x15, 0xe5, 0x00, 0x82, 0x14, 0x81, 0x40, 0x48, 0xd6, - 0x3a, 0x5e, 0x6c, 0x4b, 0x89, 0x1d, 0x76, 0xd7, 0x81, 0xfe, 0x03, 0x07, 0x3e, 0xab, 0xdc, 0x7a, - 0xe4, 0x54, 0xa1, 0xe4, 0x47, 0x90, 0xc7, 0xbb, 0x4e, 0x52, 0x92, 0xc0, 0x2d, 0x99, 0x79, 0xf3, - 0x66, 0xfc, 0xde, 0xec, 0xc0, 0x3d, 0xc1, 0x3c, 0xda, 0x0f, 0x68, 0x18, 0x75, 0x5c, 0x2a, 0xfb, - 0x41, 0x18, 0xf9, 0x9d, 0xf1, 0x41, 0xc7, 0x67, 0x11, 0x13, 0xa1, 0xb0, 0x47, 0x3c, 0x96, 0x31, - 0xb9, 0x95, 0x83, 0x6c, 0x0d, 0xb2, 0xc7, 0x07, 0x3b, 0xdb, 0x7e, 0xec, 0xc7, 0x88, 0xe8, 0xa4, - 0xbf, 0x32, 0xf0, 0xce, 0xfd, 0xe5, 0x8c, 0x79, 0x21, 0xa2, 0x5a, 0x97, 0x45, 0xa8, 0x3d, 0xcf, - 0x9a, 0x9c, 0x4a, 0x2a, 0x19, 0x79, 0x04, 0xdb, 0xfd, 0x84, 0x73, 0x16, 0x49, 0x07, 0xa1, 0x4e, - 0x94, 0x0c, 0x5d, 0xc6, 0x4d, 0xa3, 0x69, 0xb4, 0x4b, 0x3d, 0xa2, 0x72, 0xdd, 0x34, 0xf5, 0x0a, - 0x33, 0xe4, 0x09, 0x6c, 0x22, 0x92, 0x09, 0x73, 0xa3, 0x59, 0x6c, 0x57, 0x0f, 0x6f, 0xdb, 0x4b, - 0xe7, 0xb4, 0xb1, 0xa8, 0x5b, 0x3a, 0xbf, 0xbc, 0x5b, 0xe8, 0xe9, 0x12, 0x72, 0x0c, 0x90, 0xf5, - 0xf1, 0xa8, 0xa4, 0x66, 0x11, 0x09, 0x9a, 0xeb, 0x08, 0x9e, 0x51, 0x49, 0x15, 0x49, 0xc5, 0xd5, - 0x01, 0xf2, 0x06, 0x6a, 0x29, 0x81, 0xc3, 0x99, 0x48, 0x06, 0x52, 0x98, 0x25, 0x24, 0x6a, 0xaf, - 0x20, 0x52, 0x5f, 0x9c, 0x56, 0xf6, 0xb0, 0x40, 0x11, 0x56, 0xbd, 0x3c, 0x22, 0xc8, 0x07, 0x68, - 0x64, 0x93, 0x51, 0x21, 0x42, 0x3f, 0x1a, 0xb2, 0x48, 0x0a, 0xf3, 0x1a, 0xf2, 0xee, 0xad, 0x1b, - 0xf0, 0x69, 0x0e, 0x57, 0xac, 0x5b, 0xee, 0x62, 0x58, 0x90, 0xc7, 0x50, 0x1e, 0x51, 0x4e, 0x87, - 0xc2, 0x2c, 0x37, 0x8d, 0x76, 0xf5, 0xf0, 0xce, 0x0a, 0xbe, 0xd7, 0x08, 0x52, 0x34, 0xaa, 0x84, - 0xec, 0x03, 0xf9, 0x1c, 0x72, 0x71, 0xc5, 0x9f, 0x4d, 0xf4, 0x67, 0x0b, 0x33, 0x73, 0xee, 0xb4, - 0xbe, 0x1b, 0x50, 0xbf, 0x32, 0x16, 0xd9, 0x85, 0xda, 0x12, 0x6f, 0xab, 0xee, 0x9c, 0xa9, 0x7b, - 0x50, 0x57, 0x7a, 0x7e, 0x49, 0x98, 0x90, 0x4e, 0xe8, 0x99, 0x1b, 0x4d, 0xa3, 0x5d, 0xe9, 0xdd, - 0xc8, 0x24, 0xc2, 0xe8, 0x0b, 0x8f, 0xd8, 0x70, 0x73, 0x01, 0x17, 0xb0, 0xd0, 0x0f, 0xa4, 0x59, - 0x44, 0xc6, 0xc6, 0x1c, 0xf6, 0x04, 0x13, 0xad, 0x9f, 0x1b, 0x50, 0xc9, 0x6d, 0xfc, 0x9f, 0x41, - 0xdc, 0xbc, 0x41, 0xea, 0x8a, 0xc3, 0x22, 0xc9, 0x43, 0xdc, 0xb4, 0x54, 0xb7, 0xfd, 0x15, 0xba, - 0xcd, 0x8c, 0x7d, 0xcb, 0x19, 0x3b, 0xce, 0x6a, 0x94, 0x8c, 0x8d, 0x99, 0xc7, 0x2a, 0x41, 0x3e, - 0x41, 0x63, 0x4c, 0x07, 0xa1, 0x47, 0x65, 0xcc, 0xf3, 0x0e, 0xd9, 0x2a, 0x3e, 0x58, 0xd1, 0xe1, - 0x9d, 0xc6, 0xeb, 0x06, 0x67, 0xda, 0xec, 0x9c, 0x49, 0xb3, 0xbf, 0x87, 0x6c, 0x01, 0x9c, 0x54, - 0x7f, 0x2a, 0x13, 0xce, 0xf4, 0x7a, 0xae, 0x5d, 0xa3, 0xd3, 0x1c, 0xad, 0x98, 0xeb, 0xee, 0x62, - 0xb8, 0xf5, 0x15, 0x1a, 0x7f, 0x2d, 0x32, 0x31, 0xf5, 0x6b, 0xf4, 0x50, 0xcd, 0xeb, 0xfa, 0xa5, - 0x79, 0xe4, 0x04, 0xaa, 0x73, 0x4a, 0x2a, 0x05, 0x77, 0xff, 0xa9, 0xa0, 0xea, 0x0e, 0x33, 0xd9, - 0xba, 0x2f, 0xcf, 0x27, 0x96, 0x71, 0x31, 0xb1, 0x8c, 0xdf, 0x13, 0xcb, 0xf8, 0x31, 0xb5, 0x0a, - 0x17, 0x53, 0xab, 0xf0, 0x6b, 0x6a, 0x15, 0x3e, 0x1e, 0xf9, 0xa1, 0x0c, 0x12, 0xd7, 0xee, 0xc7, - 0xc3, 0x4e, 0x4a, 0x8c, 0x47, 0xa6, 0x1f, 0x0f, 0xf0, 0xcf, 0xc3, 0xec, 0x1a, 0x7d, 0x9b, 0xdd, - 0x23, 0x79, 0x36, 0x62, 0xc2, 0x2d, 0x23, 0xea, 0xe8, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xd6, - 0xac, 0xf5, 0x7d, 0x04, 0x05, 0x00, 0x00, + // 554 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x94, 0xc1, 0x6e, 0xd3, 0x4c, + 0x10, 0xc7, 0xe3, 0x24, 0x5f, 0x3e, 0xb2, 0x09, 0x0a, 0x5e, 0x8a, 0x64, 0x55, 0x60, 0xdc, 0x80, + 0x2a, 0x23, 0x81, 0x4d, 0xdb, 0x23, 0x5c, 0x88, 0xa8, 0x28, 0x07, 0x10, 0xa4, 0x08, 0x04, 0x42, + 0xb2, 0xd6, 0xf6, 0xca, 0xb6, 0x94, 0xd8, 0x61, 0x77, 0x1d, 0xe8, 0x3b, 0x70, 0xe0, 0x51, 0x78, + 0x8c, 0x72, 0xeb, 0x91, 0x13, 0x42, 0xc9, 0x8b, 0x20, 0x8f, 0x77, 0x9d, 0x04, 0x92, 0xc0, 0x2d, + 0x99, 0xf9, 0xcf, 0x6f, 0xd6, 0xff, 0x99, 0x5d, 0x74, 0x8b, 0xd3, 0x90, 0x04, 0x31, 0x49, 0x52, + 0xd7, 0x27, 0x22, 0x88, 0x93, 0x34, 0x72, 0xa7, 0x07, 0x6e, 0x44, 0x53, 0xca, 0x13, 0xee, 0x4c, + 0x58, 0x26, 0x32, 0x7c, 0xad, 0x12, 0x39, 0x4a, 0xe4, 0x4c, 0x0f, 0x76, 0x77, 0xa2, 0x2c, 0xca, + 0x40, 0xe1, 0x16, 0xbf, 0x4a, 0xf1, 0xee, 0xed, 0xf5, 0xc4, 0xaa, 0x10, 0x54, 0xfd, 0xaf, 0x0d, + 0xd4, 0x7d, 0x52, 0x36, 0x39, 0x15, 0x44, 0x50, 0x7c, 0x1f, 0xed, 0x04, 0x39, 0x63, 0x34, 0x15, + 0x1e, 0x48, 0xbd, 0x34, 0x1f, 0xfb, 0x94, 0x19, 0x9a, 0xa5, 0xd9, 0xcd, 0x21, 0x96, 0xb9, 0x41, + 0x91, 0x7a, 0x0e, 0x19, 0xfc, 0x10, 0xfd, 0x0f, 0x4a, 0xca, 0x8d, 0xba, 0xd5, 0xb0, 0x3b, 0x87, + 0xd7, 0x9d, 0xb5, 0xe7, 0x74, 0xa0, 0x68, 0xd0, 0x3c, 0xff, 0x71, 0xb3, 0x36, 0x54, 0x25, 0xf8, + 0x18, 0xa1, 0xb2, 0x4f, 0x48, 0x04, 0x31, 0x1a, 0x00, 0xb0, 0xb6, 0x01, 0x1e, 0x13, 0x41, 0x24, + 0xa4, 0xed, 0xab, 0x00, 0x7e, 0x89, 0xba, 0x05, 0xc0, 0x63, 0x94, 0xe7, 0x23, 0xc1, 0x8d, 0x26, + 0x80, 0xec, 0x0d, 0x20, 0xf9, 0xc5, 0x45, 0xe5, 0x10, 0x0a, 0x24, 0xb0, 0x13, 0x56, 0x11, 0x8e, + 0xdf, 0x22, 0xbd, 0x3c, 0x19, 0xe1, 0x3c, 0x89, 0xd2, 0x31, 0x4d, 0x05, 0x37, 0xfe, 0x03, 0xee, + 0xfe, 0xb6, 0x03, 0x3e, 0xaa, 0xe4, 0x92, 0x7a, 0xc5, 0x5f, 0x0d, 0x73, 0xfc, 0x00, 0xb5, 0x26, + 0x84, 0x91, 0x31, 0x37, 0x5a, 0x96, 0x66, 0x77, 0x0e, 0x6f, 0x6c, 0xe0, 0xbd, 0x00, 0x91, 0xc4, + 0xc8, 0x92, 0xfe, 0x67, 0x0d, 0xf5, 0x7e, 0x6b, 0x84, 0xf7, 0x50, 0x77, 0xcd, 0xb4, 0x3a, 0xfe, + 0xd2, 0x98, 0xf6, 0x51, 0x4f, 0x3a, 0xf4, 0x21, 0xa7, 0x5c, 0x78, 0x49, 0x68, 0xd4, 0x2d, 0xcd, + 0x6e, 0x0f, 0x2f, 0x97, 0x1f, 0x0d, 0xd1, 0xa7, 0x21, 0x76, 0xd0, 0xd5, 0x15, 0x5d, 0x4c, 0x93, + 0x28, 0x16, 0x46, 0x03, 0x88, 0xfa, 0x92, 0xf6, 0x04, 0x12, 0xfd, 0x6f, 0x75, 0xd4, 0xae, 0x06, + 0xf3, 0x2f, 0x07, 0xf1, 0xab, 0x06, 0x85, 0xcf, 0x1e, 0x4d, 0x05, 0x4b, 0x60, 0x77, 0x0a, 0x27, + 0xee, 0x6e, 0x70, 0x62, 0x31, 0xaa, 0x57, 0x8c, 0xd2, 0xe3, 0xb2, 0x46, 0x1a, 0xa3, 0x2f, 0xa6, + 0x26, 0x13, 0xf8, 0x3d, 0xd2, 0xa7, 0x64, 0x94, 0x84, 0x44, 0x64, 0xac, 0xea, 0x50, 0x2e, 0xd7, + 0x9d, 0x0d, 0x1d, 0x5e, 0x2b, 0xbd, 0x6a, 0x70, 0xa6, 0xc6, 0x57, 0x91, 0x14, 0xfd, 0x0d, 0x2a, + 0x47, 0xea, 0x15, 0xfe, 0x13, 0x91, 0x33, 0xaa, 0x16, 0x6e, 0xeb, 0x62, 0x9c, 0x56, 0x6a, 0x49, + 0xee, 0xf9, 0xab, 0xe1, 0xfe, 0x47, 0xa4, 0xff, 0xb1, 0x9a, 0xd8, 0x50, 0xf7, 0x2b, 0x04, 0x37, + 0x2f, 0xa9, 0xbb, 0x13, 0xe2, 0x13, 0xd4, 0x59, 0x72, 0x52, 0x3a, 0xb8, 0xf7, 0x57, 0x07, 0x65, + 0x77, 0xb4, 0xb0, 0x6d, 0xf0, 0xec, 0x7c, 0x66, 0x6a, 0x17, 0x33, 0x53, 0xfb, 0x39, 0x33, 0xb5, + 0x2f, 0x73, 0xb3, 0x76, 0x31, 0x37, 0x6b, 0xdf, 0xe7, 0x66, 0xed, 0xdd, 0x51, 0x94, 0x88, 0x38, + 0xf7, 0x9d, 0x20, 0x1b, 0xbb, 0x05, 0x18, 0x9e, 0x8d, 0x20, 0x1b, 0xc1, 0x9f, 0x7b, 0xe5, 0xfb, + 0xf2, 0x69, 0xf1, 0xc2, 0x88, 0xb3, 0x09, 0xe5, 0x7e, 0x0b, 0x54, 0x47, 0xbf, 0x02, 0x00, 0x00, + 0xff, 0xff, 0xd3, 0x84, 0x10, 0x33, 0xd6, 0x04, 0x00, 0x00, } func (m *GenesisState) Marshal() (dAtA []byte, err error) { @@ -373,11 +364,6 @@ func (m *GenesisState) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.FirstBatchNumber != 0 { - i = encodeVarintGenesis(dAtA, i, uint64(m.FirstBatchNumber)) - i-- - dAtA[i] = 0x38 - } { size, err := m.Params.MarshalToSizedBuffer(dAtA[:i]) if err != nil { @@ -647,9 +633,6 @@ func (m *GenesisState) Size() (n int) { } l = m.Params.Size() n += 1 + l + sovGenesis(uint64(l)) - if m.FirstBatchNumber != 0 { - n += 1 + sovGenesis(uint64(m.FirstBatchNumber)) - } return n } @@ -935,25 +918,6 @@ func (m *GenesisState) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex - case 7: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field FirstBatchNumber", wireType) - } - m.FirstBatchNumber = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowGenesis - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.FirstBatchNumber |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } default: iNdEx = preIndex skippy, err := skipGenesis(dAtA[iNdEx:]) diff --git a/x/batching/types/keys.go b/x/batching/types/keys.go index 27f06d75..ea464fd6 100644 --- a/x/batching/types/keys.go +++ b/x/batching/types/keys.go @@ -20,5 +20,4 @@ var ( DataResultTreeEntriesKeyPrefix = collections.NewPrefix(6) BatchSignaturesKeyPrefix = collections.NewPrefix(7) ParamsKey = collections.NewPrefix(8) - FirstBatchNumberKey = collections.NewPrefix(9) ) diff --git a/x/batching/types/params.go b/x/batching/types/params.go index e9e84c8f..5e3e4fb2 100644 --- a/x/batching/types/params.go +++ b/x/batching/types/params.go @@ -5,15 +5,17 @@ import ( ) const ( - DefaultNumBatchesToKeep = 10000 - DefaultMaxBatchPrunePerBlock = 100 + DefaultNumBatchesToKeep = 10000 + DefaultMaxBatchPrunePerBlock = 100 + DefaultMaxDataResultsToCheckForPrune = 100 ) // DefaultParams returns default batching module parameters. func DefaultParams() Params { return Params{ - NumBatchesToKeep: DefaultNumBatchesToKeep, - MaxBatchPrunePerBlock: DefaultMaxBatchPrunePerBlock, + NumBatchesToKeep: DefaultNumBatchesToKeep, + MaxBatchPrunePerBlock: DefaultMaxBatchPrunePerBlock, + MaxDataResultsToCheckForPrune: DefaultMaxDataResultsToCheckForPrune, } } diff --git a/x/batching/types/telemetry.go b/x/batching/types/telemetry.go new file mode 100644 index 00000000..ad5aa1b9 --- /dev/null +++ b/x/batching/types/telemetry.go @@ -0,0 +1,5 @@ +package types + +const ( + TelemetryKeyBatchingPruningFail = "seda_tally_end_block_batching_pruning_fail" +) From 659a6392968c2243d2efff592b762920d92478cb Mon Sep 17 00:00:00 2001 From: Hyoung-yoon Kim Date: Fri, 28 Nov 2025 13:05:01 -0500 Subject: [PATCH 2/8] feat(x/batching): implementation of batching module pruning Implementation of batching module pruning as described in #673. --- app/app.go | 7 +- app/upgrades/mainnet/v1.0.7/constants.go | 54 ++ proto/sedachain/batching/v1/batching.proto | 20 +- proto/sedachain/batching/v1/genesis.proto | 4 + testutil/integration.go | 13 - x/batching/README.md | 7 +- x/batching/keeper/batch.go | 33 +- x/batching/keeper/batch_assignments.go | 67 +++ x/batching/keeper/benchmark_endblock_test.go | 32 +- x/batching/keeper/data_result.go | 243 +++++---- x/batching/keeper/endblock.go | 65 ++- x/batching/keeper/endblock_pruning.go | 150 ------ x/batching/keeper/endblock_test.go | 164 ------ x/batching/keeper/evidence_test.go | 12 +- x/batching/keeper/export_test.go | 131 +++++ x/batching/keeper/genesis.go | 35 +- x/batching/keeper/genesis_test.go | 13 + x/batching/keeper/integration_test.go | 2 +- x/batching/keeper/keeper.go | 23 +- x/batching/keeper/keeper_test.go | 4 +- x/batching/keeper/pruning.go | 212 ++++++++ x/batching/keeper/pruning_test.go | 484 +++++++++++++++++ x/batching/keeper/querier.go | 4 +- x/batching/types/batching.pb.go | 537 ++++++++++++++++--- x/batching/types/genesis.go | 24 +- x/batching/types/genesis.pb.go | 227 ++++++-- x/batching/types/keys.go | 6 +- x/batching/types/params.go | 12 +- x/tally/keeper/endblock_test.go | 10 +- 29 files changed, 1958 insertions(+), 637 deletions(-) create mode 100644 app/upgrades/mainnet/v1.0.7/constants.go create mode 100644 x/batching/keeper/batch_assignments.go delete mode 100644 x/batching/keeper/endblock_pruning.go create mode 100644 x/batching/keeper/export_test.go create mode 100644 x/batching/keeper/pruning.go create mode 100644 x/batching/keeper/pruning_test.go diff --git a/app/app.go b/app/app.go index 2850a93a..2b84b90a 100644 --- a/app/app.go +++ b/app/app.go @@ -480,17 +480,12 @@ func NewApp( app.AccountKeeper.AddressCodec(), ) - groupConfig := group.DefaultConfig() - /* - Example of setting group params: - groupConfig.MaxMetadataLen = 1000 - */ app.GroupKeeper = groupkeeper.NewKeeper( keys[group.StoreKey], appCodec, app.MsgServiceRouter(), app.AccountKeeper, - groupConfig, + group.DefaultConfig(), ) homePath := cast.ToString(appOpts.Get(flags.FlagHome)) diff --git a/app/upgrades/mainnet/v1.0.7/constants.go b/app/upgrades/mainnet/v1.0.7/constants.go new file mode 100644 index 00000000..60840df8 --- /dev/null +++ b/app/upgrades/mainnet/v1.0.7/constants.go @@ -0,0 +1,54 @@ +package v1 + +import ( + "context" + + storetypes "cosmossdk.io/store/types" + upgradetypes "cosmossdk.io/x/upgrade/types" + + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/cosmos/cosmos-sdk/types/module" + + "github.com/sedaprotocol/seda-chain/app/keepers" + "github.com/sedaprotocol/seda-chain/app/upgrades" +) + +const ( + UpgradeName = "v" // TODO Update name and register this handler. +) + +var Upgrade = upgrades.Upgrade{ + UpgradeName: UpgradeName, + CreateUpgradeHandler: CreateUpgradeHandler, + StoreUpgrades: storetypes.StoreUpgrades{ + Added: []string{}, + Deleted: []string{}, + }, +} + +func CreateUpgradeHandler( + mm upgrades.ModuleManager, + configurator module.Configurator, + keepers *keepers.AppKeepers, +) upgradetypes.UpgradeHandler { + return func(context context.Context, _ upgradetypes.Plan, fromVM module.VersionMap) (module.VersionMap, error) { + ctx := sdk.UnwrapSDKContext(context) + + // Run module migrations. + migrations, err := mm.RunMigrations(ctx, configurator, fromVM) + if err != nil { + return nil, err + } + + err = keepers.BatchingKeeper.SetBatchNumberAtUpgrade(ctx) + if err != nil { + return nil, err + } + err = keepers.BatchingKeeper.SetHasPruningCaughtUp(ctx, false) + if err != nil { + return nil, err + } + + return migrations, nil + } +} diff --git a/proto/sedachain/batching/v1/batching.proto b/proto/sedachain/batching/v1/batching.proto index 1023872e..3b413bf7 100644 --- a/proto/sedachain/batching/v1/batching.proto +++ b/proto/sedachain/batching/v1/batching.proto @@ -86,6 +86,20 @@ message DataResult { bool consensus = 12 [ (gogoproto.jsontag) = "consensus" ]; } +// DataRequestIDHeights is a collection of DataRequestIDHeight objects. +message DataRequestIDHeights { + repeated DataRequestIDHeight data_request_id_heights = 1 + [ (gogoproto.nullable) = false ]; +} + +// DataRequestIDHeight is a pair of data request ID and its posted height. +message DataRequestIDHeight { + // DataRequestID is the hex-encoded data request ID. + string data_request_id = 1; + // DataRequestHeight is the height at which the data request was submitted. + uint64 data_request_height = 2; +} + // Params defines the parameters for the batching module. message Params { // NumBatchesToKeep is the number of batches to keep in the state without @@ -94,7 +108,7 @@ message Params { // MaxBatchPrunePerBlock is the maximum number of batches to prune per // block. uint64 max_batch_prune_per_block = 2; - // MaxDataResultsToCheckForPrune is the maximum number of data results to - // check for pruning per block. - uint64 max_data_results_to_check_for_prune = 3; + // MaxLegacyDataResultPrunePerBlock is the maximum number of legacy data + // results to be checked for pruning per block. + uint64 max_legacy_data_result_prune_per_block = 3; } diff --git a/proto/sedachain/batching/v1/genesis.proto b/proto/sedachain/batching/v1/genesis.proto index 9f077f3a..9005ae41 100644 --- a/proto/sedachain/batching/v1/genesis.proto +++ b/proto/sedachain/batching/v1/genesis.proto @@ -17,6 +17,10 @@ message GenesisState { repeated BatchAssignment batch_assignments = 5 [ (gogoproto.nullable) = false ]; Params params = 6 [ (gogoproto.nullable) = false ]; + repeated GenesisDataResult legacy_data_results = 7 + [ (gogoproto.nullable) = false ]; + bool has_pruning_caught_up = 8; + uint64 batch_number_at_upgrade = 9; } // BatchAssignment represents a batch assignment for genesis export diff --git a/testutil/integration.go b/testutil/integration.go index 27c2243f..422ba67c 100644 --- a/testutil/integration.go +++ b/testutil/integration.go @@ -1,7 +1,6 @@ package testutil import ( - "crypto/rand" "fmt" "time" @@ -212,18 +211,6 @@ func (app *IntegationApp) AddBlock() { app.ctx = app.ctx.WithBlockHeight(app.ctx.BlockHeader().Height + 1) } -func (app *IntegationApp) SetRandomLastCommitHash() { - randomBytes := make([]byte, 32) - _, err := rand.Read(randomBytes) - if err != nil { - panic(err) - } - - newHeader := app.ctx.BlockHeader() - newHeader.LastCommitHash = randomBytes - app.ctx = app.ctx.WithBlockHeader(newHeader) -} - // QueryHelper returns the application query helper. // It can be used when registering query services. func (app *IntegationApp) QueryHelper() *baseapp.QueryServiceTestHelper { diff --git a/x/batching/README.md b/x/batching/README.md index bf9ddd01..477cb7cf 100644 --- a/x/batching/README.md +++ b/x/batching/README.md @@ -5,7 +5,7 @@ The batching module collects data rseults, current validators, and their signatu ## State ``` -0x00 | is_batched | data_request_id | data_request_height -> data_result +0x00 | is_batched | data_request_id | data_request_height -> legacy_data_results 0x01 | data_request_id | data_request_height -> batch_number 0x02 -> current_batch_number 0x03 | block_height -> batch @@ -13,6 +13,11 @@ The batching module collects data rseults, current validators, and their signatu 0x05 | batch_number | validator_address -> validator_tree_entries 0x06 | batch_number -> data_tree_entries 0x07 | batch_number | validator_address -> batch_signature +0x08 -> parameters +0x09 | is_batched | data_request_id | data_request_height -> data_result +0x10 | batch_number -> data_request_id | data_request_height +0x11 -> batch_number_at_upgrade +0x12 -> has_pruning_caught_up ``` ### Batches diff --git a/x/batching/keeper/batch.go b/x/batching/keeper/batch.go index cfac841b..4d772da4 100644 --- a/x/batching/keeper/batch.go +++ b/x/batching/keeper/batch.go @@ -34,47 +34,48 @@ func (k Keeper) setBatch(ctx context.Context, batch types.Batch) error { return k.batches.Set(ctx, batch.BlockHeight, batch) } -// SetNewBatch increments the current batch number and stores a given -// batch at that index. It also stores the given data result tree -// entries, validator tree entries, and batch signature entries (at -// the next batch index, to be populated with signatures later). It -// returns an error if a batch already exists at the given batch's -// block height or if the given batch's batch number does not match -// the next batch number. -func (k Keeper) SetNewBatch(ctx context.Context, batch types.Batch, dataEntries types.DataResultTreeEntries, valEntries []types.ValidatorTreeEntry) error { +// SetNewBatch stores a new batch and its associated data at current batch number +// and increments the current batch number. If successful, it returns the batch number +// of the newly created batch. +func (k Keeper) SetNewBatch(ctx sdk.Context, batch types.Batch, dataEntries types.DataResultTreeEntries, valEntries []types.ValidatorTreeEntry) (uint64, error) { found, err := k.batches.Has(ctx, batch.BlockHeight) if err != nil { - return err + return 0, err } if found { - return types.ErrBatchAlreadyExists.Wrapf("batch block height %d", batch.BlockHeight) + return 0, types.ErrBatchAlreadyExists.Wrapf("batch block height %d", batch.BlockHeight) } batchNum, err := k.GetCurrentBatchNum(ctx) if err != nil { - return err + return 0, err } if batch.BatchNumber != batchNum { - return types.ErrInvalidBatchNumber.Wrapf("got %d; expected %d", batch.BatchNumber, batchNum) + return 0, types.ErrInvalidBatchNumber.Wrapf("got %d; expected %d", batch.BatchNumber, batchNum) } err = k.setDataResultTreeEntry(ctx, batchNum, dataEntries) if err != nil { - return err + return 0, err } for _, valEntry := range valEntries { err = k.setValidatorTreeEntry(ctx, batchNum, valEntry) if err != nil { - return err + return 0, err } } _, err = k.incrementCurrentBatchNum(ctx) if err != nil { - return err + return 0, err } - return k.batches.Set(ctx, batch.BlockHeight, batch) + + err = k.batches.Set(ctx, batch.BlockHeight, batch) + if err != nil { + return 0, err + } + return batchNum, nil } func (k Keeper) GetBatchForHeight(ctx context.Context, blockHeight int64) (types.Batch, error) { diff --git a/x/batching/keeper/batch_assignments.go b/x/batching/keeper/batch_assignments.go new file mode 100644 index 00000000..58c95811 --- /dev/null +++ b/x/batching/keeper/batch_assignments.go @@ -0,0 +1,67 @@ +package keeper + +import ( + "context" + "errors" + + "cosmossdk.io/collections" + + "github.com/sedaprotocol/seda-chain/x/batching/types" +) + +// SetBatchAssignment stores mapping between data request ID - posted height pair +// and assigned batch number in both directions. +func (k Keeper) SetBatchAssignment(ctx context.Context, dataReqID string, dataReqHeight, batchNumber uint64) error { + items, err := k.batchDataResults.Get(ctx, batchNumber) + if err != nil { + if !errors.Is(err, collections.ErrNotFound) { + return err + } + items.DataRequestIdHeights = make([]types.DataRequestIDHeight, 0) + } + items.DataRequestIdHeights = append(items.DataRequestIdHeights, types.DataRequestIDHeight{ + DataRequestId: dataReqID, + DataRequestHeight: dataReqHeight, + }) + err = k.batchDataResults.Set(ctx, batchNumber, items) + if err != nil { + return err + } + + return k.batchAssignments.Set(ctx, collections.Join(dataReqID, dataReqHeight), batchNumber) +} + +func (k Keeper) GetBatchAssignment(ctx context.Context, dataReqID string, dataReqHeight uint64) (uint64, error) { + return k.batchAssignments.Get(ctx, collections.Join(dataReqID, dataReqHeight)) +} + +func (k Keeper) RemoveBatchAssignment(ctx context.Context, dataReqID string, dataReqHeight uint64) error { + return k.batchAssignments.Remove(ctx, collections.Join(dataReqID, dataReqHeight)) +} + +// getAllBatchAssignments retrieves all batch assignments from the store. +// Used for genesis export. +func (k Keeper) getAllBatchAssignments(ctx context.Context) ([]types.BatchAssignment, error) { + var batchAssignments []types.BatchAssignment + err := k.batchAssignments.Walk(ctx, nil, func(key collections.Pair[string, uint64], value uint64) (stop bool, err error) { + batchAssignments = append(batchAssignments, types.BatchAssignment{ + BatchNumber: value, + DataRequestId: key.K1(), + DataRequestHeight: key.K2(), + }) + return false, nil + }) + return batchAssignments, err +} + +func (k Keeper) SetBatchDataResults(ctx context.Context, batchNumber uint64, dataRequestIDHeights types.DataRequestIDHeights) error { + return k.batchDataResults.Set(ctx, batchNumber, dataRequestIDHeights) +} + +func (k Keeper) GetBatchDataResults(ctx context.Context, batchNumber uint64) (types.DataRequestIDHeights, error) { + return k.batchDataResults.Get(ctx, batchNumber) +} + +func (k Keeper) RemoveBatchDataResults(ctx context.Context, batchNumber uint64) error { + return k.batchDataResults.Remove(ctx, batchNumber) +} diff --git a/x/batching/keeper/benchmark_endblock_test.go b/x/batching/keeper/benchmark_endblock_test.go index 8a28566c..911b6da0 100644 --- a/x/batching/keeper/benchmark_endblock_test.go +++ b/x/batching/keeper/benchmark_endblock_test.go @@ -14,6 +14,7 @@ func BenchmarkBatchPruning(b *testing.B) { numBatchesToKeep := uint64(1000) maxBatchPrunePerBlock := uint64(100) + var lastBatchNum uint64 for range numBatches { f.AddBlock() @@ -21,39 +22,12 @@ func BenchmarkBatchPruning(b *testing.B) { require.NoError(b, err) batch, dataEntries, valEntries, err := f.batchingKeeper.ConstructBatch(f.Context()) require.NoError(b, err) - err = f.batchingKeeper.SetNewBatch(f.Context(), batch, dataEntries, valEntries) + lastBatchNum, err = f.batchingKeeper.SetNewBatch(f.Context(), batch, dataEntries, valEntries) require.NoError(b, err) } for b.Loop() { - _, err := f.batchingKeeper.PruneBatches(f.Context(), numBatchesToKeep, maxBatchPrunePerBlock) - require.NoError(b, err) - } -} - -func BenchmarkDataResultPruning(b *testing.B) { - f := initFixture(b) - - maxDataResultsToCheckForPrune := uint64(100) - - // Create 10 data results for each of 1000 batches - for i := range uint64(100) { - f.AddBlock() - - dataResults := generateDataResults(b, 10) - for _, dataResult := range dataResults { - err := f.batchingKeeper.SetDataResultForBatching(f.Context(), dataResult) - require.NoError(b, err) - err = f.batchingKeeper.MarkDataResultAsBatched(f.Context(), dataResult, i) - require.NoError(b, err) - } - } - - for b.Loop() { - f.AddBlock() - f.SetRandomLastCommitHash() - - err := f.batchingKeeper.PruneDataResults(f.Context(), maxDataResultsToCheckForPrune, 2000) + _, err := f.batchingKeeper.BatchPruneBatches(f.Context(), numBatchesToKeep, maxBatchPrunePerBlock, lastBatchNum) require.NoError(b, err) } } diff --git a/x/batching/keeper/data_result.go b/x/batching/keeper/data_result.go index d3f7fa9b..94bee208 100644 --- a/x/batching/keeper/data_result.go +++ b/x/batching/keeper/data_result.go @@ -9,108 +9,108 @@ import ( "github.com/sedaprotocol/seda-chain/x/batching/types" ) -// SetDataResultForBatching stores a data result so that it is ready -// to be batched. +// SetDataResultForBatching stores a data result so that it is ready to be batched. func (k Keeper) SetDataResultForBatching(ctx context.Context, result types.DataResult) error { return k.dataResults.Set(ctx, collections.Join3(false, result.DrId, result.DrBlockHeight), result) } +// SetDataResultAsBatched stores a data result under "batched" status. +func (k Keeper) SetDataResultAsBatched(ctx context.Context, result types.DataResult) error { + return k.dataResults.Set(ctx, collections.Join3(true, result.DrId, result.DrBlockHeight), result) +} + // RemoveDataResult removes a data result from the store. func (k Keeper) RemoveDataResult(ctx context.Context, batched bool, dataReqID string, dataReqHeight uint64) error { return k.dataResults.Remove(ctx, collections.Join3(batched, dataReqID, dataReqHeight)) } -// MarkDataResultAsBatched removes the "unbatched" variant of the given -// data result and stores a "batched" variant. +// RemoveLegacyDataResult removes a data result from the legacy store. +func (k Keeper) RemoveLegacyDataResult(ctx context.Context, batched bool, dataReqID string, dataReqHeight uint64) error { + return k.legacyDataResults.Remove(ctx, collections.Join3(batched, dataReqID, dataReqHeight)) +} + +// MarkDataResultAsBatched updates the data result status to "batched" and stores +// the batch number assignment. func (k Keeper) MarkDataResultAsBatched(ctx context.Context, result types.DataResult, batchNum uint64) error { - err := k.SetBatchAssignment(ctx, result.DrId, result.DrBlockHeight, batchNum) + err := k.RemoveDataResult(ctx, false, result.DrId, result.DrBlockHeight) if err != nil { return err } - err = k.dataResults.Remove(ctx, collections.Join3(false, result.DrId, result.DrBlockHeight)) + err = k.SetDataResultAsBatched(ctx, result) if err != nil { return err } - return k.dataResults.Set(ctx, collections.Join3(true, result.DrId, result.DrBlockHeight), result) + return k.SetBatchAssignment(ctx, result.DrId, result.DrBlockHeight, batchNum) } -// GetDataResult returns a data result given the associated data request's -// ID and height. -func (k Keeper) GetDataResult(ctx context.Context, dataReqID string, dataReqHeight uint64) (*types.DataResult, error) { - dataResult, err := k.dataResults.Get(ctx, collections.Join3(false, dataReqID, dataReqHeight)) - if err != nil { - if errors.Is(err, collections.ErrNotFound) { - // Look among batched data requests. - dataResult, err := k.dataResults.Get(ctx, collections.Join3(true, dataReqID, dataReqHeight)) - if err != nil { - return nil, err - } - return &dataResult, nil - } - return nil, err - } - return &dataResult, err +// GetDataResults returns a list of data results under a given status +// (batched or not). +func (k Keeper) GetDataResults(ctx context.Context, batched bool) ([]types.DataResult, error) { + var results []types.DataResult + err := k.IterateDataResults(ctx, batched, func(_ collections.Triple[bool, string, uint64], value types.DataResult) (bool, error) { + results = append(results, value) + return false, nil + }) + return results, err } -// GetLatestDataResult returns the latest data result given the associated -// data request's ID. -func (k Keeper) GetLatestDataResult(ctx context.Context, dataReqID string) (*types.DataResult, error) { - dataResult, err := k.getLatestDataResult(ctx, false, dataReqID) +// IterateDataResults iterates over all data results under a given +// status (batched or not) and performs a given callback function. +func (k Keeper) IterateDataResults(ctx context.Context, batched bool, cb func(key collections.Triple[bool, string, uint64], value types.DataResult) (bool, error)) error { + rng := collections.NewPrefixedTripleRange[bool, string, uint64](batched) + return k.dataResults.Walk(ctx, rng, cb) +} + +// getAllGenesisDataResults returns all data results from the store regardless +// of their batched status. Used for genesis export. +func (k Keeper) getAllGenesisDataResults(ctx context.Context) ([]types.GenesisDataResult, error) { + dataResults := make([]types.GenesisDataResult, 0) + unbatched, err := k.GetDataResults(ctx, false) if err != nil { - if errors.Is(err, collections.ErrNotFound) { - // Look among batched data requests. - dataResult, err := k.getLatestDataResult(ctx, true, dataReqID) - if err != nil { - return nil, err - } - return dataResult, nil - } return nil, err } - - return dataResult, nil -} - -func (k Keeper) getLatestDataResult(ctx context.Context, batched bool, dataReqID string) (*types.DataResult, error) { - // The triple pair ranger does not expose the Descending() method, - // so we manually create the range using the same prefix that the - // collections.NewSuperPrefixedTripleRange uses internally. - drRange := &collections.Range[collections.Triple[bool, string, uint64]]{} - drRange.Prefix(collections.TripleSuperPrefix[bool, string, uint64](batched, dataReqID)).Descending() - - itr, err := k.dataResults.Iterate(ctx, drRange) + for _, result := range unbatched { + dataResults = append(dataResults, types.GenesisDataResult{ + Batched: false, + DataResult: result, + }) + } + batched, err := k.GetDataResults(ctx, true) if err != nil { return nil, err } - defer itr.Close() - - if itr.Valid() { - kv, err := itr.KeyValue() - if err != nil { - return nil, err - } - return &kv.Value, nil + for _, result := range batched { + dataResults = append(dataResults, types.GenesisDataResult{ + Batched: true, + DataResult: result, + }) } - - return nil, collections.ErrNotFound + return dataResults, nil } -// GetDataResults returns a list of data results under a given status +// GetLegacyDataResults returns a list of legacy data results under a given status // (batched or not). -func (k Keeper) GetDataResults(ctx context.Context, batched bool) ([]types.DataResult, error) { +func (k Keeper) GetLegacyDataResults(ctx context.Context, batched bool) ([]types.DataResult, error) { var results []types.DataResult - err := k.IterateDataResults(ctx, batched, func(_ collections.Triple[bool, string, uint64], value types.DataResult) (bool, error) { + err := k.IterateLegacyDataResults(ctx, batched, func(_ collections.Triple[bool, string, uint64], value types.DataResult) (bool, error) { results = append(results, value) return false, nil }) return results, err } -// getAllDataResults returns all data results from the store regardless +// IterateLegacyDataResults iterates over all legacy data results under a given +// status (batched or not) and performs a given callback function. +func (k Keeper) IterateLegacyDataResults(ctx context.Context, batched bool, cb func(key collections.Triple[bool, string, uint64], value types.DataResult) (bool, error)) error { + rng := collections.NewPrefixedTripleRange[bool, string, uint64](batched) + return k.legacyDataResults.Walk(ctx, rng, cb) +} + +// getAllGenesisLegacyDataResults returns all data results from the legacy store regardless // of their batched status. Used for genesis export. -func (k Keeper) getAllGenesisDataResults(ctx context.Context) ([]types.GenesisDataResult, error) { +func (k Keeper) getAllGenesisLegacyDataResults(ctx context.Context) ([]types.GenesisDataResult, error) { dataResults := make([]types.GenesisDataResult, 0) - unbatched, err := k.GetDataResults(ctx, false) + unbatched, err := k.GetLegacyDataResults(ctx, false) if err != nil { return nil, err } @@ -120,7 +120,7 @@ func (k Keeper) getAllGenesisDataResults(ctx context.Context) ([]types.GenesisDa DataResult: result, }) } - batched, err := k.GetDataResults(ctx, true) + batched, err := k.GetLegacyDataResults(ctx, true) if err != nil { return nil, err } @@ -133,40 +133,95 @@ func (k Keeper) getAllGenesisDataResults(ctx context.Context) ([]types.GenesisDa return dataResults, nil } -// IterateDataResults iterates over all data results under a given -// status (batched or not) and performs a given callback function. -func (k Keeper) IterateDataResults(ctx context.Context, batched bool, cb func(key collections.Triple[bool, string, uint64], value types.DataResult) (bool, error)) error { - rng := collections.NewPrefixedTripleRange[bool, string, uint64](batched) - return k.dataResults.Walk(ctx, rng, cb) -} +// GetDataResult returns a data result given the associated data request's +// ID and height. +// NOTE: Checks both legacy and new collections. +func (k Keeper) GetDataResult(ctx context.Context, dataReqID string, dataReqHeight uint64) (types.DataResult, error) { + var dataResult types.DataResult + var err error + + // Check in the following order: + // Legacy unbatched -> Legacy batched -> Unbatched -> Batched + dataResult, err = k.legacyDataResults.Get(ctx, collections.Join3(false, dataReqID, dataReqHeight)) + if err != nil && !errors.Is(err, collections.ErrNotFound) { + return dataResult, err + } else if err == nil { + return dataResult, nil + } + dataResult, err = k.legacyDataResults.Get(ctx, collections.Join3(true, dataReqID, dataReqHeight)) + if err != nil && !errors.Is(err, collections.ErrNotFound) { + return dataResult, err + } else if err == nil { + return dataResult, nil + } -// SetBatchAssignment assigns a given batch number to the given data -// request ID and data request height. -func (k Keeper) SetBatchAssignment(ctx context.Context, dataReqID string, dataReqHeight uint64, batchNumber uint64) error { - return k.batchAssignments.Set(ctx, collections.Join(dataReqID, dataReqHeight), batchNumber) + dataResult, err = k.dataResults.Get(ctx, collections.Join3(false, dataReqID, dataReqHeight)) + if err != nil && !errors.Is(err, collections.ErrNotFound) { + return dataResult, err + } else if err == nil { + return dataResult, nil + } + return k.dataResults.Get(ctx, collections.Join3(true, dataReqID, dataReqHeight)) } -// GetBatchAssignment returns the given data request's assigned batch -// number for a given height. -func (k Keeper) GetBatchAssignment(ctx context.Context, dataReqID string, dataReqHeight uint64) (uint64, error) { - return k.batchAssignments.Get(ctx, collections.Join(dataReqID, dataReqHeight)) +// GetLatestDataResult returns the latest data result given the associated +// data request's ID. +// NOTE: Checks both legacy and new collections. +func (k Keeper) GetLatestDataResult(ctx context.Context, dataReqID string) (types.DataResult, error) { + dataResult, err := k.getLatestDataResult(ctx, false, dataReqID) + if err != nil { + if errors.Is(err, collections.ErrNotFound) { + // Look among batched data requests. + dataResult, err := k.getLatestDataResult(ctx, true, dataReqID) + if err != nil { + return types.DataResult{}, err + } + return dataResult, nil + } + return types.DataResult{}, err + } + return dataResult, nil } -func (k Keeper) RemoveBatchAssignment(ctx context.Context, dataReqID string, dataReqHeight uint64) error { - return k.batchAssignments.Remove(ctx, collections.Join(dataReqID, dataReqHeight)) -} +// getLatestDataResult returns the latest data result given the associated +// data request's ID and its batched status. +// NOTE: Checks both legacy and new collections. +func (k Keeper) getLatestDataResult(ctx context.Context, batched bool, dataReqID string) (types.DataResult, error) { + // The triple pair ranger does not expose the Descending() method, + // so we manually create the range using the same prefix that the + // collections.NewSuperPrefixedTripleRange uses internally. + drRange := &collections.Range[collections.Triple[bool, string, uint64]]{} + drRange.Prefix(collections.TripleSuperPrefix[bool, string, uint64](batched, dataReqID)).Descending() -// getAllBatchAssignments retrieves all batch assignments from the store. -// Used for genesis export. -func (k Keeper) getAllBatchAssignments(ctx context.Context) ([]types.BatchAssignment, error) { - var batchAssignments []types.BatchAssignment - err := k.batchAssignments.Walk(ctx, nil, func(key collections.Pair[string, uint64], value uint64) (stop bool, err error) { - batchAssignments = append(batchAssignments, types.BatchAssignment{ - BatchNumber: value, - DataRequestId: key.K1(), - DataRequestHeight: key.K2(), - }) - return false, nil - }) - return batchAssignments, err + // Check the current store first. + itr, err := k.dataResults.Iterate(ctx, drRange) + if err != nil { + return types.DataResult{}, err + } + defer itr.Close() + + if itr.Valid() { + kv, err := itr.KeyValue() + if err != nil { + return types.DataResult{}, err + } + return kv.Value, nil + } + + // Check the legacy store. + itr, err = k.legacyDataResults.Iterate(ctx, drRange) + if err != nil { + return types.DataResult{}, err + } + defer itr.Close() + + if itr.Valid() { + kv, err := itr.KeyValue() + if err != nil { + return types.DataResult{}, err + } + return kv.Value, nil + } + + return types.DataResult{}, collections.ErrNotFound } diff --git a/x/batching/keeper/endblock.go b/x/batching/keeper/endblock.go index 2e509d5c..ca2ba491 100644 --- a/x/batching/keeper/endblock.go +++ b/x/batching/keeper/endblock.go @@ -19,6 +19,15 @@ import ( ) func (k Keeper) EndBlock(ctx sdk.Context) error { + params, err := k.GetParams(ctx) + if err != nil { + return err + } + batchNumAtUpgrade, err := k.GetBatchNumberAtUpgrade(ctx) + if err != nil { + return err + } + // Since we're only using the secp256k1 key for batching, we only // need to check if the secp256k1 proving scheme is activated. isActivated, err := k.pubKeyKeeper.IsProvingSchemeActivated(ctx, sedatypes.SEDAKeyIndexSecp256k1) @@ -34,32 +43,60 @@ func (k Keeper) EndBlock(ctx sdk.Context) error { } k.Logger(ctx).Info("skip batch creation due to no update", "height", ctx.BlockHeight()) } else { - err = k.SetNewBatch(ctx, batch, dataEntries, valEntries) + newBatchNum, err := k.SetNewBatch(ctx, batch, dataEntries, valEntries) if err != nil { return err } + + // Try pruning a batch. + // If there has been an upgrade (batchNumAtUpgrade is not 0), + // then prune only if the batch was created after the upgrade. + batchNumToPrune := newBatchNum - params.NumBatchesToKeep + if newBatchNum >= params.NumBatchesToKeep && + (batchNumAtUpgrade == 0 || batchNumToPrune > batchNumAtUpgrade) { + err = k.TryPruneBatch(ctx, batchNumToPrune) + if err != nil { + return err + } + } } } else { k.Logger(ctx).Info("skip batching since proving scheme has not been activated", "index", sedatypes.SEDAKeyIndexSecp256k1) } - params, err := k.GetParams(ctx) + hasCaughtUp, err := k.HasPruningCaughtUp(ctx) if err != nil { return err } + if !hasCaughtUp { + // Batch prune MaxBatchPrunePerBlock batches and switch HasPruningCaughtUp + // to true if all batches up to the batch number at the time of the upgrade + // have been pruned. + // Note this operation does not prune data results, which will be pruned + // separately in the else clause. + lastPrunedBatchNum, err := k.BatchPruneBatches(ctx, params.NumBatchesToKeep, params.MaxBatchPrunePerBlock, batchNumAtUpgrade) + if err != nil { + telemetry.SetGauge(1, types.TelemetryKeyBatchingPruningFail) + k.Logger(ctx).Error("error while pruning batches", "err", err) + return nil + } - lastPrunedBatchNum, err := k.PruneBatches(ctx, params.NumBatchesToKeep, params.MaxBatchPrunePerBlock) - if err != nil { - telemetry.SetGauge(1, types.TelemetryKeyBatchingPruningFail) - k.Logger(ctx).Error("error while pruning batches", "err", err) - return nil - } - - err = k.PruneDataResults(ctx, params.MaxDataResultsToCheckForPrune, lastPrunedBatchNum) - if err != nil { - telemetry.SetGauge(1, types.TelemetryKeyBatchingPruningFail) - k.Logger(ctx).Error("error while pruning data results", "err", err) - return nil + if lastPrunedBatchNum >= batchNumAtUpgrade { + err = k.SetHasPruningCaughtUp(ctx, true) + if err != nil { + return err + } + k.Logger(ctx).Info("batch pruning has caught up") + } + } else { + // Now batch pruning of batches is terminated, and we start pruning legacy + // data results collection, which is no longer used. + err = k.PruneLegacyDataResults(ctx, params.MaxLegacyDataResultPrunePerBlock) + if err != nil { + telemetry.SetGauge(1, types.TelemetryKeyBatchingPruningFail) + k.Logger(ctx).Error("error while pruning legacy data results", "err", err) + return nil + } } telemetry.SetGauge(0, types.TelemetryKeyBatchingPruningFail) diff --git a/x/batching/keeper/endblock_pruning.go b/x/batching/keeper/endblock_pruning.go deleted file mode 100644 index 8530b802..00000000 --- a/x/batching/keeper/endblock_pruning.go +++ /dev/null @@ -1,150 +0,0 @@ -package keeper - -import ( - "encoding/hex" - - "golang.org/x/crypto/sha3" - - "cosmossdk.io/collections" - - sdk "github.com/cosmos/cosmos-sdk/types" -) - -func (k Keeper) PruneDataResults(ctx sdk.Context, maxDataResultsToCheckForPrune, lastRemovedBatchNum uint64) error { - if maxDataResultsToCheckForPrune == 0 || lastRemovedBatchNum == 0 { - k.Logger(ctx).Info("skip data result pruning", "max_data_results_to_check_for_prune", maxDataResultsToCheckForPrune, "last_removed_batch_num", lastRemovedBatchNum) - return nil - } - - // Use hash of last commit hash as starting point of the range. - hasher := sha3.NewLegacyKeccak256() - hasher.Write(ctx.BlockHeader().LastCommitHash) - hash := hasher.Sum(nil) - - var rng *collections.Range[collections.Triple[bool, string, uint64]] - if ctx.BlockHeight()%2 == 0 { - rng = new(collections.Range[collections.Triple[bool, string, uint64]]). - StartInclusive(collections.TripleSuperPrefix[bool, string, uint64](true, hex.EncodeToString(hash))) - } else { - rng = new(collections.Range[collections.Triple[bool, string, uint64]]). - EndInclusive(collections.TripleSuperPrefix[bool, string, uint64](true, hex.EncodeToString(hash))). - Descending() - } - - iter, err := k.dataResults.Iterate(ctx, rng) - if err != nil { - return err - } - defer iter.Close() - - var numChecked, numPruned uint64 - for ; iter.Valid(); iter.Next() { - kv, err := iter.KeyValue() - if err != nil { - return err - } - - batchNum, err := k.GetBatchAssignment(ctx, kv.Value.DrId, kv.Value.DrBlockHeight) - if err != nil { - return err - } - - if batchNum <= lastRemovedBatchNum { - err = k.RemoveDataResult(ctx, true, kv.Value.DrId, kv.Value.DrBlockHeight) - if err != nil { - return err - } - err = k.RemoveBatchAssignment(ctx, kv.Value.DrId, kv.Value.DrBlockHeight) - if err != nil { - return err - } - numPruned++ - } - - numChecked++ - if numChecked == maxDataResultsToCheckForPrune { - break - } - } - - k.Logger(ctx).Info("pruned data results", "num_checked", numChecked, "num_pruned", numPruned) - return nil -} - -// PruneBatches prunes batches and their associated data based on module -// parameters NumBatchesToKeep and MaxBatchPrunePerBlock. It returns the -// batch number of the last pruned batch. -func (k Keeper) PruneBatches(ctx sdk.Context, numBatchesToKeep, maxBatchPrunePerBlock uint64) (uint64, error) { - currentBatchNum, err := k.GetCurrentBatchNum(ctx) - if err != nil { - return 0, err - } - if currentBatchNum <= numBatchesToKeep { - k.Logger(ctx).Info("skip batch pruning", "current_batch_num", currentBatchNum, "num_batches_to_keep", numBatchesToKeep) - return 0, nil - } - - rng := new(collections.Range[uint64]).EndExclusive(currentBatchNum - numBatchesToKeep) - iter, err := k.batches.Indexes.Number.Iterate(ctx, rng) - if err != nil { - return 0, err - } - defer iter.Close() - - var firstKey *collections.Pair[uint64, int64] - var pruneCount uint64 - var lastPrunedBatchNum uint64 - for ; iter.Valid(); iter.Next() { - fullKey, err := iter.FullKey() - if err != nil { - return 0, err - } - if firstKey == nil { - firstKey = &fullKey - } - - batchNum, batchHeight := fullKey.K1(), fullKey.K2() - if batchNum >= currentBatchNum-numBatchesToKeep { - // Should not happen because of the range configuration. - break - } - - err = k.batches.Remove(ctx, batchHeight) - if err != nil { - return 0, err - } - k.Logger(ctx).Info("pruned batch", "batch_num", batchNum) - - lastPrunedBatchNum = batchNum - - pruneCount++ - if pruneCount == maxBatchPrunePerBlock { - break - } - } - - if firstKey == nil { - // This means nothing was pruned. - k.Logger(ctx).Info("no batches to prune") - return 0, nil - } - - dataRng := new(collections.Range[uint64]).EndExclusive(firstKey.K1() + pruneCount) - err = k.dataResultTreeEntries.Clear(ctx, dataRng) - if err != nil { - return 0, err - } - - valRng := new(collections.Range[collections.Pair[uint64, []byte]]). - EndExclusive(collections.PairPrefix[uint64, []byte](firstKey.K1() + pruneCount)) - err = k.validatorTreeEntries.Clear(ctx, valRng) - if err != nil { - return 0, err - } - err = k.batchSignatures.Clear(ctx, valRng) - if err != nil { - return 0, err - } - - return lastPrunedBatchNum, nil -} diff --git a/x/batching/keeper/endblock_test.go b/x/batching/keeper/endblock_test.go index adedbebe..6562811f 100644 --- a/x/batching/keeper/endblock_test.go +++ b/x/batching/keeper/endblock_test.go @@ -16,7 +16,6 @@ import ( ethcrypto "github.com/ethereum/go-ethereum/crypto" "github.com/stretchr/testify/require" - "cosmossdk.io/collections" "cosmossdk.io/math" "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" @@ -777,166 +776,3 @@ func (f *fixture) addBatchSigningValidatorsFromTestData(t *testing.T, testData [ } return addrs, secp256k1PubKeys, powers } - -func TestBatchPruning(t *testing.T) { - f := initFixture(t) - - f.addBatchSigningValidators(t, 10) - - numBatchesToKeep := uint64(75) - maxBatchPrunePerBlock := uint64(150) - - // Should prune nothing. - lastRemovedBatchNum, err := f.batchingKeeper.PruneBatches(f.Context(), numBatchesToKeep, maxBatchPrunePerBlock) - require.NoError(t, err) - require.Equal(t, uint64(0), lastRemovedBatchNum) - - // Create 300 batches with random associated data. - for range 300 { - f.AddBlock() - - err := f.batchingKeeper.SetDataResultForBatching(f.Context(), generateDataResults(t, 1)[0]) - require.NoError(t, err) - batch, dataEntries, valEntries, err := f.batchingKeeper.ConstructBatch(f.Context()) - require.NoError(t, err) - err = f.batchingKeeper.SetNewBatch(f.Context(), batch, dataEntries, valEntries) - require.NoError(t, err) - err = f.batchingKeeper.SetBatchSigSecp256k1(f.Context(), batch.BatchNumber, valEntries[0].ValidatorAddress, generateRandomBytes(64)) - require.NoError(t, err) - } - - batches, err := f.batchingKeeper.GetAllBatches(f.Context()) - require.NoError(t, err) - require.Equal(t, 300, len(batches)) - - // Should prune first 150 batches. - lastRemovedBatchNum, err = f.batchingKeeper.PruneBatches(f.Context(), numBatchesToKeep, maxBatchPrunePerBlock) - require.NoError(t, err) - require.Equal(t, uint64(149), lastRemovedBatchNum) - - batches, err = f.batchingKeeper.GetAllBatches(f.Context()) - require.NoError(t, err) - require.Equal(t, 150, len(batches)) - require.Equal(t, uint64(150), batches[0].BatchNumber) - require.Equal(t, uint64(299), batches[len(batches)-1].BatchNumber) - - for i := 0; i <= 149; i++ { - f.checkNoBatchData(t, uint64(i)) - } - for i := 150; i <= 299; i++ { - f.checkBatchData(t, uint64(i)) - } - - // Should prune second 75 batches. - lastRemovedBatchNum, err = f.batchingKeeper.PruneBatches(f.Context(), numBatchesToKeep, maxBatchPrunePerBlock) - require.NoError(t, err) - require.Equal(t, uint64(224), lastRemovedBatchNum) - - batches, err = f.batchingKeeper.GetAllBatches(f.Context()) - require.NoError(t, err) - require.Equal(t, 75, len(batches)) - require.Equal(t, uint64(225), batches[0].BatchNumber) - require.Equal(t, uint64(299), batches[len(batches)-1].BatchNumber) - - for i := 0; i <= 224; i++ { - f.checkNoBatchData(t, uint64(i)) - } - for i := 225; i <= 299; i++ { - f.checkBatchData(t, uint64(i)) - } - - // Should prune nothing. - lastRemovedBatchNum, err = f.batchingKeeper.PruneBatches(f.Context(), numBatchesToKeep, maxBatchPrunePerBlock) - require.NoError(t, err) - require.Equal(t, uint64(0), lastRemovedBatchNum) - - batches, err = f.batchingKeeper.GetAllBatches(f.Context()) - require.NoError(t, err) - require.Equal(t, 75, len(batches)) - require.Equal(t, uint64(225), batches[0].BatchNumber) - require.Equal(t, uint64(299), batches[len(batches)-1].BatchNumber) - - for i := 0; i <= 224; i++ { - f.checkNoBatchData(t, uint64(i)) - } - for i := 225; i <= 299; i++ { - f.checkBatchData(t, uint64(i)) - } -} - -func (f *fixture) checkNoBatchData(t *testing.T, batchNum uint64) { - batch, err := f.batchingKeeper.GetBatchByBatchNumber(f.Context(), batchNum) - require.ErrorIs(t, err, collections.ErrNotFound) - dataEntries, err := f.batchingKeeper.GetDataResultTreeEntries(f.Context(), batchNum) - require.ErrorIs(t, err, collections.ErrNotFound) - valEntries, _ := f.batchingKeeper.GetValidatorTreeEntries(f.Context(), batchNum) - // require.ErrorIs(t, err, collections.ErrNotFound) // this function does not error even if there are no entries. - sigs, _ := f.batchingKeeper.GetBatchSignatures(f.Context(), batchNum) - // require.ErrorIs(t, err, collections.ErrNotFound) // this function does not error even if there are no entries. - - require.Empty(t, batch, "batchNum: %d", batchNum) - require.Empty(t, dataEntries, "batchNum: %d", batchNum) - require.Empty(t, valEntries, "batchNum: %d", batchNum) - require.Empty(t, sigs, "batchNum: %d", batchNum) -} - -func (f *fixture) checkBatchData(t *testing.T, batchNum uint64) { - batch, err := f.batchingKeeper.GetBatchByBatchNumber(f.Context(), batchNum) - require.NoError(t, err) - dataEntries, err := f.batchingKeeper.GetDataResultTreeEntries(f.Context(), batchNum) - require.NoError(t, err) - valEntries, err := f.batchingKeeper.GetValidatorTreeEntries(f.Context(), batchNum) - require.NoError(t, err) - sigs, err := f.batchingKeeper.GetBatchSignatures(f.Context(), batchNum) - require.NoError(t, err) - - require.NotEmpty(t, batch) - require.NotEmpty(t, dataEntries) - require.NotEmpty(t, valEntries) - require.NotEmpty(t, sigs) -} - -func TestDataResultPruning(t *testing.T) { - f := initFixture(t) - - maxDataResultsToCheckForPrune := uint64(100) - - // Should prune nothing. - err := f.batchingKeeper.PruneDataResults(f.Context(), maxDataResultsToCheckForPrune, 0) - require.NoError(t, err) - - // Create 10 data results for each of 100 batches - for i := range uint64(100) { - f.AddBlock() - - dataResults := generateDataResults(t, 10) - for _, dataResult := range dataResults { - err := f.batchingKeeper.SetDataResultForBatching(f.Context(), dataResult) - require.NoError(t, err) - err = f.batchingKeeper.MarkDataResultAsBatched(f.Context(), dataResult, i) - require.NoError(t, err) - } - } - - dataResults, err := f.batchingKeeper.GetDataResults(f.Context(), true) - require.NoError(t, err) - require.Equal(t, 1000, len(dataResults)) - - i := 0 - for ; i < 30; i++ { - f.AddBlock() - f.SetRandomLastCommitHash() - - err = f.batchingKeeper.PruneDataResults(f.Context(), maxDataResultsToCheckForPrune, uint64(25+25*i)) - require.NoError(t, err) - - dataResults, err = f.batchingKeeper.GetDataResults(f.Context(), true) - require.NoError(t, err) - if len(dataResults) == 0 { - break - } - } - - require.Equal(t, 0, len(dataResults)) - t.Logf("test completed after %d iterations", i) -} diff --git a/x/batching/keeper/evidence_test.go b/x/batching/keeper/evidence_test.go index bba70c63..cc7de715 100644 --- a/x/batching/keeper/evidence_test.go +++ b/x/batching/keeper/evidence_test.go @@ -43,7 +43,7 @@ func TestHandleEvidence(t *testing.T) { BatchNumber: doubleSignBatchNumber, BlockHeight: doubleSignBlockHeight, } - err := f.batchingKeeper.SetNewBatch(f.Context(), batchToDoubleSign, types.DataResultTreeEntries{}, []types.ValidatorTreeEntry{}) + _, err := f.batchingKeeper.SetNewBatch(f.Context(), batchToDoubleSign, types.DataResultTreeEntries{}, []types.ValidatorTreeEntry{}) require.NoError(t, err) f.stakingKeeper.SetHistoricalInfo(f.Context(), doubleSignBlockHeight, &sdkstakingtypes.HistoricalInfo{ @@ -115,7 +115,7 @@ func TestHandleEvidence_DifferentBlockHeight(t *testing.T) { BatchNumber: doubleSignBatchNumber, BlockHeight: doubleSignBlockHeight, } - err := f.batchingKeeper.SetNewBatch(f.Context(), batchToDoubleSign, types.DataResultTreeEntries{}, []types.ValidatorTreeEntry{}) + _, err := f.batchingKeeper.SetNewBatch(f.Context(), batchToDoubleSign, types.DataResultTreeEntries{}, []types.ValidatorTreeEntry{}) require.NoError(t, err) f.stakingKeeper.SetHistoricalInfo(f.Context(), doubleSignBlockHeight, &sdkstakingtypes.HistoricalInfo{ @@ -207,7 +207,7 @@ func TestHandleEvidence_LegitBatchID(t *testing.T) { legitBatchID, err := evidence.GetBatchID() require.NoError(t, err) - err = f.batchingKeeper.SetNewBatch(f.Context(), types.Batch{ + _, err = f.batchingKeeper.SetNewBatch(f.Context(), types.Batch{ BatchId: legitBatchID, BatchNumber: 1, BlockHeight: 2, @@ -273,7 +273,7 @@ func TestHandleEvidence_DifferentPrivateKey(t *testing.T) { BatchNumber: doubleSignBatchNumber, BlockHeight: doubleSignBlockHeight, } - err := f.batchingKeeper.SetNewBatch(f.Context(), batchToDoubleSign, types.DataResultTreeEntries{}, []types.ValidatorTreeEntry{}) + _, err := f.batchingKeeper.SetNewBatch(f.Context(), batchToDoubleSign, types.DataResultTreeEntries{}, []types.ValidatorTreeEntry{}) require.NoError(t, err) f.stakingKeeper.SetHistoricalInfo(f.Context(), doubleSignBlockHeight, &sdkstakingtypes.HistoricalInfo{ @@ -322,7 +322,7 @@ func TestHandleEvidence_StaleEvidence(t *testing.T) { BatchNumber: doubleSignBatchNumber, BlockHeight: doubleSignBlockHeight, } - err := f.batchingKeeper.SetNewBatch(f.Context(), batchToDoubleSign, types.DataResultTreeEntries{}, []types.ValidatorTreeEntry{}) + _, err := f.batchingKeeper.SetNewBatch(f.Context(), batchToDoubleSign, types.DataResultTreeEntries{}, []types.ValidatorTreeEntry{}) require.NoError(t, err) f.stakingKeeper.SetHistoricalInfo(f.Context(), doubleSignBlockHeight, &sdkstakingtypes.HistoricalInfo{ @@ -399,7 +399,7 @@ func TestHandleEvidence_TombstonedValidator(t *testing.T) { BatchNumber: doubleSignBatchNumber, BlockHeight: doubleSignBlockHeight, } - err := f.batchingKeeper.SetNewBatch(f.Context(), batchToDoubleSign, types.DataResultTreeEntries{}, []types.ValidatorTreeEntry{}) + _, err := f.batchingKeeper.SetNewBatch(f.Context(), batchToDoubleSign, types.DataResultTreeEntries{}, []types.ValidatorTreeEntry{}) require.NoError(t, err) f.stakingKeeper.SetHistoricalInfo(f.Context(), doubleSignBlockHeight, &sdkstakingtypes.HistoricalInfo{ diff --git a/x/batching/keeper/export_test.go b/x/batching/keeper/export_test.go new file mode 100644 index 00000000..dd9aea6f --- /dev/null +++ b/x/batching/keeper/export_test.go @@ -0,0 +1,131 @@ +/* + This file is added for test use only. +*/ + +package keeper + +import ( + "context" + "encoding/hex" + "errors" + + "golang.org/x/crypto/sha3" + + "cosmossdk.io/collections" + sdk "github.com/cosmos/cosmos-sdk/types" + + "github.com/sedaprotocol/seda-chain/app/utils" + sedatypes "github.com/sedaprotocol/seda-chain/types" + "github.com/sedaprotocol/seda-chain/x/batching/types" +) + +func (k Keeper) LegacySetDataResultForBatching(ctx context.Context, result types.DataResult) error { + return k.legacyDataResults.Set(ctx, collections.Join3(false, result.DrId, result.DrBlockHeight), result) +} + +func (k Keeper) LegacyConstructBatch(ctx sdk.Context) (types.Batch, types.DataResultTreeEntries, []types.ValidatorTreeEntry, error) { + var newBatchNum uint64 + var latestDataRootHex, latestValRootHex string + latestBatch, err := k.GetLatestBatch(ctx) + if err != nil { + if !errors.Is(err, types.ErrBatchingHasNotStarted) { + return types.Batch{}, types.DataResultTreeEntries{}, nil, err + } + newBatchNum = collections.DefaultSequenceStart + } else { + newBatchNum = latestBatch.BatchNumber + 1 + latestDataRootHex = latestBatch.DataResultRoot + latestValRootHex = latestBatch.ValidatorRoot + } + + // Compute current data result tree root and the "super root" + // of current and previous data result trees' roots. + dataEntries, dataRoot, err := k.LegacyConstructDataResultTree(ctx, newBatchNum) + if err != nil { + return types.Batch{}, types.DataResultTreeEntries{}, nil, err + } + latestDataRoot, err := hex.DecodeString(latestDataRootHex) + if err != nil { + return types.Batch{}, types.DataResultTreeEntries{}, nil, err + } + superRoot := utils.RootFromLeaves([][]byte{latestDataRoot, dataRoot}) + + // Compute validator tree root. + valEntries, valRoot, err := k.ConstructValidatorTree(ctx) + if err != nil { + return types.Batch{}, types.DataResultTreeEntries{}, nil, err + } + valRootHex := hex.EncodeToString(valRoot) + + // Skip batching if there is no update in data result root nor + // validator root. + if len(dataEntries.Entries) == 0 && valRootHex == latestValRootHex { + return types.Batch{}, types.DataResultTreeEntries{}, nil, types.ErrNoBatchingUpdate + } + + var provingMetaData, provingMetaDataHash []byte + if len(provingMetaData) == 0 { + provingMetaDataHash = make([]byte, 32) // zero hash + } else { + hasher := sha3.NewLegacyKeccak256() + hasher.Write(provingMetaData) + provingMetaDataHash = hasher.Sum(nil) + } + + batchID := types.ComputeBatchID(newBatchNum, ctx.BlockHeight(), valRoot, superRoot, provingMetaDataHash) + + return types.Batch{ + BatchNumber: newBatchNum, + BlockHeight: ctx.BlockHeight(), + CurrentDataResultRoot: hex.EncodeToString(dataRoot), + DataResultRoot: hex.EncodeToString(superRoot), + ValidatorRoot: valRootHex, + BatchId: batchID, + ProvingMetadata: provingMetaData, + }, dataEntries, valEntries, nil +} + +func (k Keeper) LegacyConstructDataResultTree(ctx sdk.Context, newBatchNum uint64) (types.DataResultTreeEntries, []byte, error) { + dataResults, err := k.GetLegacyDataResults(ctx, false) + if err != nil { + return types.DataResultTreeEntries{}, nil, err + } + + entries := make([][]byte, len(dataResults)) + treeEntries := make([][]byte, len(dataResults)) + for i, res := range dataResults { + resID, err := hex.DecodeString(res.Id) + if err != nil { + return types.DataResultTreeEntries{}, nil, err + } + entries[i] = resID + treeEntries[i] = append([]byte{sedatypes.SEDASeparatorDataResult}, resID...) + + err = k.LegacyMarkDataResultAsBatched(ctx, res, newBatchNum) + if err != nil { + return types.DataResultTreeEntries{}, nil, err + } + } + + return types.DataResultTreeEntries{Entries: entries}, utils.RootFromEntries(treeEntries), nil +} + +func (k Keeper) LegacyMarkDataResultAsBatched(ctx context.Context, result types.DataResult, batchNum uint64) error { + err := k.LegacyRemoveDataResult(ctx, false, result.DrId, result.DrBlockHeight) + if err != nil { + return err + } + err = k.legacySetDataResultAsBatched(ctx, result) + if err != nil { + return err + } + return k.SetBatchAssignment(ctx, result.DrId, result.DrBlockHeight, batchNum) +} + +func (k Keeper) LegacyRemoveDataResult(ctx context.Context, batched bool, dataReqID string, dataReqHeight uint64) error { + return k.legacyDataResults.Remove(ctx, collections.Join3(batched, dataReqID, dataReqHeight)) +} + +func (k Keeper) legacySetDataResultAsBatched(ctx context.Context, result types.DataResult) error { + return k.legacyDataResults.Set(ctx, collections.Join3(true, result.DrId, result.DrBlockHeight), result) +} diff --git a/x/batching/keeper/genesis.go b/x/batching/keeper/genesis.go index 3bf174c4..e7772d07 100644 --- a/x/batching/keeper/genesis.go +++ b/x/batching/keeper/genesis.go @@ -44,13 +44,28 @@ func (k Keeper) InitGenesis(ctx sdk.Context, data types.GenesisState) { panic(err) } } + for _, dr := range data.LegacyDataResults { + err := k.legacyDataResults.Set(ctx, collections.Join3(dr.Batched, dr.DataResult.DrId, dr.DataResult.DrBlockHeight), dr.DataResult) + if err != nil { + panic(err) + } + } for _, batchAssignment := range data.BatchAssignments { err := k.SetBatchAssignment(ctx, batchAssignment.DataRequestId, batchAssignment.DataRequestHeight, batchAssignment.BatchNumber) if err != nil { panic(err) } } - if err := k.SetParams(ctx, data.Params); err != nil { + err = k.SetParams(ctx, data.Params) + if err != nil { + panic(err) + } + err = k.SetHasPruningCaughtUp(ctx, data.HasPruningCaughtUp) + if err != nil { + panic(err) + } + err = k.batchNumberAtUpgrade.Set(ctx, data.BatchNumberAtUpgrade) + if err != nil { panic(err) } } @@ -61,6 +76,10 @@ func (k Keeper) ExportGenesis(ctx sdk.Context) types.GenesisState { if err != nil { panic(err) } + legacyDataResults, err := k.getAllGenesisLegacyDataResults(ctx) + if err != nil { + panic(err) + } batchAssignments, err := k.getAllBatchAssignments(ctx) if err != nil { panic(err) @@ -85,5 +104,17 @@ func (k Keeper) ExportGenesis(ctx sdk.Context) types.GenesisState { if err != nil { panic(err) } - return types.NewGenesisState(curBatchNum, batches, batchData, dataResults, batchAssignments, params) + hasPruningCaughtUp, err := k.HasPruningCaughtUp(ctx) + if err != nil { + panic(err) + } + batchNumAtUpgrade, err := k.GetBatchNumberAtUpgrade(ctx) + if err != nil { + panic(err) + } + return types.NewGenesisState( + curBatchNum, batches, batchData, + dataResults, legacyDataResults, batchAssignments, + params, hasPruningCaughtUp, batchNumAtUpgrade, + ) } diff --git a/x/batching/keeper/genesis_test.go b/x/batching/keeper/genesis_test.go index 16f4c11e..d5240e88 100644 --- a/x/batching/keeper/genesis_test.go +++ b/x/batching/keeper/genesis_test.go @@ -25,6 +25,12 @@ func TestExportGenesis(t *testing.T) { valAddrs, _, _ := f.addBatchSigningValidators(t, 10) + legacyDataResults := generateDataResults(t, 25) + for _, dr := range legacyDataResults { + err := f.batchingKeeper.LegacySetDataResultForBatching(f.Context(), dr) + require.NoError(t, err) + } + dataResults := generateDataResults(t, 25) for _, dr := range dataResults { err := f.batchingKeeper.SetDataResultForBatching(f.Context(), dr) @@ -53,6 +59,9 @@ func TestExportGenesis(t *testing.T) { batchSigsBefore, err := f.batchingKeeper.GetBatchSignatures(f.Context(), latestBatchBefore.BatchNumber) require.NoError(t, err) + legacyDataResultsBefore, err := f.batchingKeeper.GetLegacyDataResults(f.Context(), false) + require.NoError(t, err) + // Export and import genesis. exportGenesis := f.batchingKeeper.ExportGenesis(f.Context()) @@ -87,6 +96,10 @@ func TestExportGenesis(t *testing.T) { batchSigsAfter, err := f.batchingKeeper.GetBatchSignatures(f.Context(), latestBatchBefore.BatchNumber) require.NoError(t, err) require.ElementsMatch(t, batchSigsBefore, batchSigsAfter) + + legacyDataResultsAfter, err := f.batchingKeeper.GetLegacyDataResults(f.Context(), false) + require.NoError(t, err) + require.ElementsMatch(t, legacyDataResultsBefore, legacyDataResultsAfter) } func (suite *KeeperTestSuite) TestInitGenesis() { diff --git a/x/batching/keeper/integration_test.go b/x/batching/keeper/integration_test.go index 5efa7185..7e81edb5 100644 --- a/x/batching/keeper/integration_test.go +++ b/x/batching/keeper/integration_test.go @@ -324,7 +324,7 @@ func generateFirstBatch(t *testing.T, f *fixture, numValidators int) ([]sdk.ValA BatchNumber: collections.DefaultSequenceStart, BlockHeight: 1, } - err := f.batchingKeeper.SetNewBatch(f.Context(), batch, types.DataResultTreeEntries{}, validatorEntries) + _, err := f.batchingKeeper.SetNewBatch(f.Context(), batch, types.DataResultTreeEntries{}, validatorEntries) require.NoError(t, err) return validatorAddrs, privKeys, validators diff --git a/x/batching/keeper/keeper.go b/x/batching/keeper/keeper.go index 8159f366..28b8052f 100644 --- a/x/batching/keeper/keeper.go +++ b/x/batching/keeper/keeper.go @@ -30,8 +30,8 @@ type Keeper struct { wasmViewKeeper wasmtypes.ViewKeeper validatorAddressCodec addresscodec.Codec - Schema collections.Schema - dataResults collections.Map[collections.Triple[bool, string, uint64], types.DataResult] + Schema collections.Schema + batchAssignments collections.Map[collections.Pair[string, uint64], uint64] currentBatchNumber collections.Sequence batches *collections.IndexedMap[int64, types.Batch, BatchIndexes] @@ -39,6 +39,21 @@ type Keeper struct { dataResultTreeEntries collections.Map[uint64, types.DataResultTreeEntries] batchSignatures collections.Map[collections.Pair[uint64, []byte], types.BatchSignatures] params collections.Item[types.Params] + // dataResults is the newer version of dataResults. The items in this collection + // have corresponding items in batchDataResults. + dataResults collections.Map[collections.Triple[bool, string, uint64], types.DataResult] + // batchDataResults maps batch number to a list of data request ID - posted height + // pairs to support simple pruning of data results. + batchDataResults collections.Map[uint64, types.DataRequestIDHeights] + // legacyDataResults is the older version of dataResults. The items in this + // collection do not have corresponding items in batchDataResults. + legacyDataResults collections.Map[collections.Triple[bool, string, uint64], types.DataResult] + // hasPruningCaughtUp indicates that all batches up to batchNumberAtUpgrade have + // been pruned by batch pruning. + hasPruningCaughtUp collections.Item[bool] + // batchNumberAtUpgrade is the batch number of the latest batch at upgrade time + // except when its value is 0, in which case there was no upgrade. + batchNumberAtUpgrade collections.Item[uint64] } func NewKeeper( @@ -64,8 +79,12 @@ func NewKeeper( wasmKeeper: wk, wasmViewKeeper: wvk, validatorAddressCodec: validatorAddressCodec, + legacyDataResults: collections.NewMap(sb, types.LegacyDataResultsPrefix, "legacy_data_results", collections.TripleKeyCodec(collections.BoolKey, collections.StringKey, collections.Uint64Key), codec.CollValue[types.DataResult](cdc)), + hasPruningCaughtUp: collections.NewItem(sb, types.HasPruningCaughtUpKey, "has_pruning_caught_up", collections.BoolValue), dataResults: collections.NewMap(sb, types.DataResultsPrefix, "data_results", collections.TripleKeyCodec(collections.BoolKey, collections.StringKey, collections.Uint64Key), codec.CollValue[types.DataResult](cdc)), batchAssignments: collections.NewMap(sb, types.BatchAssignmentsPrefix, "batch_assignments", collections.PairKeyCodec(collections.StringKey, collections.Uint64Key), collections.Uint64Value), + batchDataResults: collections.NewMap(sb, types.BatchDataResultsPrefix, "batch_data_results", collections.Uint64Key, codec.CollValue[types.DataRequestIDHeights](cdc)), + batchNumberAtUpgrade: collections.NewItem(sb, types.BatchNumberAtUpgradeKey, "batch_number_at_upgrade", collections.Uint64Value), currentBatchNumber: collections.NewSequence(sb, types.CurrentBatchNumberKey, "current_batch_number"), batches: collections.NewIndexedMap(sb, types.BatchesKeyPrefix, "batches", collections.Int64Key, codec.CollValue[types.Batch](cdc), NewBatchIndexes(sb)), validatorTreeEntries: collections.NewMap(sb, types.ValidatorTreeEntriesKeyPrefix, "validator_tree_entries", collections.PairKeyCodec(collections.Uint64Key, collections.BytesKey), codec.CollValue[types.ValidatorTreeEntry](cdc)), diff --git a/x/batching/keeper/keeper_test.go b/x/batching/keeper/keeper_test.go index 2e05c3f7..674a5a9d 100644 --- a/x/batching/keeper/keeper_test.go +++ b/x/batching/keeper/keeper_test.go @@ -87,7 +87,7 @@ func (s *KeeperTestSuite) TestKeeper_GetLatestSignedBatch() { // Height 4 // - Batch 0 is created. s.ctx = s.ctx.WithBlockHeight(s.ctx.BlockHeight() + 1) - err = s.keeper.SetNewBatch(s.ctx, types.Batch{ + _, err = s.keeper.SetNewBatch(s.ctx, types.Batch{ BatchNumber: 0, BlockHeight: s.ctx.BlockHeight(), }, types.DataResultTreeEntries{}, nil) @@ -102,7 +102,7 @@ func (s *KeeperTestSuite) TestKeeper_GetLatestSignedBatch() { // - Batch 1 is created. // - Signatures for batch 0 has been collected. s.ctx = s.ctx.WithBlockHeight(s.ctx.BlockHeight() + 1) - err = s.keeper.SetNewBatch(s.ctx, types.Batch{ + _, err = s.keeper.SetNewBatch(s.ctx, types.Batch{ BatchNumber: 1, BlockHeight: s.ctx.BlockHeight(), }, types.DataResultTreeEntries{}, nil) diff --git a/x/batching/keeper/pruning.go b/x/batching/keeper/pruning.go new file mode 100644 index 00000000..95a5f7fd --- /dev/null +++ b/x/batching/keeper/pruning.go @@ -0,0 +1,212 @@ +package keeper + +import ( + "errors" + + "cosmossdk.io/collections" + + sdk "github.com/cosmos/cosmos-sdk/types" +) + +// SetBatchNumberAtUpgrade sets the latest batch number at the time of the upgrade. +func (k Keeper) SetBatchNumberAtUpgrade(ctx sdk.Context) error { + // Latest batch number is the current batch number minus 1 because + // the current batch number has not been used yet. + currentBatchNum, err := k.GetCurrentBatchNum(ctx) + if err != nil { + return err + } + return k.batchNumberAtUpgrade.Set(ctx, currentBatchNum-1) +} + +func (k Keeper) GetBatchNumberAtUpgrade(ctx sdk.Context) (uint64, error) { + return k.batchNumberAtUpgrade.Get(ctx) +} + +func (k Keeper) SetHasPruningCaughtUp(ctx sdk.Context, hasCaughtUp bool) error { + return k.hasPruningCaughtUp.Set(ctx, hasCaughtUp) +} + +func (k Keeper) HasPruningCaughtUp(ctx sdk.Context) (bool, error) { + return k.hasPruningCaughtUp.Get(ctx) +} + +// TryPruneBatch attempts to prune the given batch and all of its associated data. +func (k Keeper) TryPruneBatch(ctx sdk.Context, batchNum uint64) error { + batch, err := k.GetBatchByBatchNumber(ctx, batchNum) + if err != nil { + return err + } + batchHeight := batch.BlockHeight + + err = k.batches.Remove(ctx, batchHeight) + if err != nil { + return err + } + err = k.dataResultTreeEntries.Remove(ctx, batchNum) + if err != nil { + return err + } + + valRng := new(collections.Range[collections.Pair[uint64, []byte]]).Prefix(collections.PairPrefix[uint64, []byte](batchNum)) + err = k.validatorTreeEntries.Clear(ctx, valRng) + if err != nil { + return err + } + err = k.batchSignatures.Clear(ctx, valRng) + if err != nil { + return err + } + + dataResults, err := k.GetBatchDataResults(ctx, batchNum) + if err != nil { + if errors.Is(err, collections.ErrNotFound) { + k.Logger(ctx).Info("cannot prune batch because schema change has not been applied", "batch_num", batchNum) + return nil + } + return err + } + for _, item := range dataResults.DataRequestIdHeights { + err = k.RemoveBatchAssignment(ctx, item.DataRequestId, item.DataRequestHeight) + if err != nil { + return err + } + err = k.RemoveDataResult(ctx, true, item.DataRequestId, item.DataRequestHeight) + if err != nil { + return err + } + } + err = k.RemoveBatchDataResults(ctx, batchNum) + if err != nil { + return err + } + + k.Logger(ctx).Info("single pruned batch", "batch_num", batchNum) + return nil +} + +// BatchPruneBatches prunes batches and their associated data, except for data +// results, in batches based on the module parameters NumBatchesToKeep and +// MaxBatchPrunePerBlock. +// It returns the batch number of the last batch that has been confirmed to have +// been pruned. +func (k Keeper) BatchPruneBatches(ctx sdk.Context, numBatchesToKeep, maxBatchPrunePerBlock, batchNumAtUpgrade uint64) (uint64, error) { + if maxBatchPrunePerBlock == 0 { + k.Logger(ctx).Info("skip batch pruning", "max_batch_prune_per_block", maxBatchPrunePerBlock) + return 0, nil + } + + // Prune up to, but not including, current batch number minus numBatchesToKeep. + currentBatchNum, err := k.GetCurrentBatchNum(ctx) + if err != nil { + return 0, err + } + if currentBatchNum <= numBatchesToKeep { + k.Logger(ctx).Info("skip batch pruning", "current_batch_num", currentBatchNum, "num_batches_to_keep", numBatchesToKeep) + return 0, nil + } + + rngEnd := min(currentBatchNum-numBatchesToKeep, batchNumAtUpgrade+1) + rng := new(collections.Range[uint64]).EndExclusive(rngEnd) + iter, err := k.batches.Indexes.Number.Iterate(ctx, rng) + if err != nil { + return 0, err + } + defer iter.Close() + + var firstKey *collections.Pair[uint64, int64] + var pruneCount uint64 + var lastPrunedBatchNum uint64 + for ; iter.Valid(); iter.Next() { + fullKey, err := iter.FullKey() + if err != nil { + return 0, err + } + if firstKey == nil { + firstKey = &fullKey + } + + batchNum, batchHeight := fullKey.K1(), fullKey.K2() + if batchNum >= currentBatchNum-numBatchesToKeep { + // Should not happen given the range configuration. + break + } + + err = k.batches.Remove(ctx, batchHeight) + if err != nil { + return 0, err + } + k.Logger(ctx).Info("pruned batch", "batch_num", batchNum) + + lastPrunedBatchNum = batchNum + + pruneCount++ + if pruneCount == maxBatchPrunePerBlock { + break + } + } + + if firstKey == nil { + k.Logger(ctx).Info("no batches to prune") + // This means all batches up to batch number rngEnd - 1 have been pruned. + // Note we subtract 1 because rngEnd is exclusive. + return rngEnd - 1, nil + } + + dataRng := new(collections.Range[uint64]).EndExclusive(firstKey.K1() + pruneCount) + err = k.dataResultTreeEntries.Clear(ctx, dataRng) + if err != nil { + return 0, err + } + + valRng := new(collections.Range[collections.Pair[uint64, []byte]]). + EndExclusive(collections.PairPrefix[uint64, []byte](firstKey.K1() + pruneCount)) + err = k.validatorTreeEntries.Clear(ctx, valRng) + if err != nil { + return 0, err + } + err = k.batchSignatures.Clear(ctx, valRng) + if err != nil { + return 0, err + } + + return lastPrunedBatchNum, nil +} + +func (k Keeper) PruneLegacyDataResults(ctx sdk.Context, maxDataResultPrunePerBlock uint64) error { + if maxDataResultPrunePerBlock == 0 { + k.Logger(ctx).Info("skip legacy data result pruning", "max_data_results_to_check_for_prune", maxDataResultPrunePerBlock) + return nil + } + + iter, err := k.legacyDataResults.Iterate(ctx, nil) + if err != nil { + return err + } + defer iter.Close() + + var numPruned uint64 + for ; iter.Valid(); iter.Next() { + kv, err := iter.KeyValue() + if err != nil { + return err + } + + err = k.RemoveLegacyDataResult(ctx, true, kv.Value.DrId, kv.Value.DrBlockHeight) + if err != nil { + return err + } + err = k.RemoveBatchAssignment(ctx, kv.Value.DrId, kv.Value.DrBlockHeight) + if err != nil { + return err + } + + numPruned++ + if numPruned == maxDataResultPrunePerBlock { + break + } + } + + k.Logger(ctx).Info("pruned legacy data results", "num_pruned", numPruned) + return nil +} diff --git a/x/batching/keeper/pruning_test.go b/x/batching/keeper/pruning_test.go new file mode 100644 index 00000000..a237e7a7 --- /dev/null +++ b/x/batching/keeper/pruning_test.go @@ -0,0 +1,484 @@ +package keeper_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "cosmossdk.io/collections" + + sedatypes "github.com/sedaprotocol/seda-chain/types" + "github.com/sedaprotocol/seda-chain/x/batching/types" + pubkeytypes "github.com/sedaprotocol/seda-chain/x/pubkey/types" +) + +func TestBatchPruneBatches(t *testing.T) { + f := initFixture(t) + + f.addBatchSigningValidators(t, 10) + + numBatchesToKeep := uint64(75) + maxBatchPrunePerBlock := uint64(150) + + // Should prune nothing. + lastRemovedBatchNum, err := f.batchingKeeper.BatchPruneBatches(f.Context(), numBatchesToKeep, maxBatchPrunePerBlock, 0) + require.NoError(t, err) + require.Equal(t, uint64(0), lastRemovedBatchNum) + + // Create 300 batches with random associated data. + var lastBatchNum uint64 + for range 300 { + f.AddBlock() + + err := f.batchingKeeper.SetDataResultForBatching(f.Context(), generateDataResults(t, 1)[0]) + require.NoError(t, err) + batch, dataEntries, valEntries, err := f.batchingKeeper.ConstructBatch(f.Context()) + require.NoError(t, err) + lastBatchNum, err = f.batchingKeeper.SetNewBatch(f.Context(), batch, dataEntries, valEntries) + require.NoError(t, err) + err = f.batchingKeeper.SetBatchSigSecp256k1(f.Context(), batch.BatchNumber, valEntries[0].ValidatorAddress, generateRandomBytes(64)) + require.NoError(t, err) + } + + batches, err := f.batchingKeeper.GetAllBatches(f.Context()) + require.NoError(t, err) + require.Equal(t, 300, len(batches)) + + // Suppose an upgrade happens here and sets batchNumberAtUpgrade. + // Should prune first 150 batches (0-149) + lastRemovedBatchNum, err = f.batchingKeeper.BatchPruneBatches(f.Context(), numBatchesToKeep, maxBatchPrunePerBlock, lastBatchNum) + require.NoError(t, err) + require.Equal(t, uint64(149), lastRemovedBatchNum) + + batches, err = f.batchingKeeper.GetAllBatches(f.Context()) + require.NoError(t, err) + require.Equal(t, 150, len(batches)) + require.Equal(t, uint64(150), batches[0].BatchNumber) + require.Equal(t, uint64(299), batches[len(batches)-1].BatchNumber) + + for i := 0; i <= 149; i++ { + f.checkNoBatchData(t, uint64(i)) + } + for i := 150; i <= 299; i++ { + f.checkBatchData(t, uint64(i), true) + } + + // Should prune second 75 batches (150-224) + lastRemovedBatchNum, err = f.batchingKeeper.BatchPruneBatches(f.Context(), numBatchesToKeep, maxBatchPrunePerBlock, lastBatchNum) + require.NoError(t, err) + require.Equal(t, uint64(224), lastRemovedBatchNum) + + batches, err = f.batchingKeeper.GetAllBatches(f.Context()) + require.NoError(t, err) + require.Equal(t, 75, len(batches)) + require.Equal(t, uint64(225), batches[0].BatchNumber) + require.Equal(t, uint64(299), batches[len(batches)-1].BatchNumber) + + for i := 0; i <= 224; i++ { + f.checkNoBatchData(t, uint64(i)) + } + for i := 225; i <= 299; i++ { + f.checkBatchData(t, uint64(i), true) + } + + // Should prune nothing + lastRemovedBatchNum, err = f.batchingKeeper.BatchPruneBatches(f.Context(), numBatchesToKeep, maxBatchPrunePerBlock, lastBatchNum) + require.NoError(t, err) + require.Equal(t, uint64(224), lastRemovedBatchNum) + + batches, err = f.batchingKeeper.GetAllBatches(f.Context()) + require.NoError(t, err) + require.Equal(t, 75, len(batches)) + require.Equal(t, uint64(225), batches[0].BatchNumber) + require.Equal(t, uint64(299), batches[len(batches)-1].BatchNumber) + + for i := 0; i <= 224; i++ { + f.checkNoBatchData(t, uint64(i)) + } + for i := 225; i <= 299; i++ { + f.checkBatchData(t, uint64(i), true) + } +} + +// TestLegacyDataResultPruning creates 1000 legacy data results and tests their +// pruning without creating batches. +func TestLegacyDataResultPruning(t *testing.T) { + f := initFixture(t) + + err := f.pubKeyKeeper.SetProvingScheme(f.Context(), pubkeytypes.ProvingScheme{ + Index: uint32(sedatypes.SEDAKeyIndexSecp256k1), + IsActivated: true, + }) + require.NoError(t, err) + + err = f.batchingKeeper.SetHasPruningCaughtUp(f.Context(), false) + require.NoError(t, err) + + err = f.batchingKeeper.SetParams(f.Context(), types.Params{ + MaxLegacyDataResultPrunePerBlock: 101, + NumBatchesToKeep: 10, + }) + require.NoError(t, err) + + // Create 10 data results for each of 100 batches + for i := range uint64(100) { + dataResults := generateDataResults(t, 10) + for _, dataResult := range dataResults { + err := f.batchingKeeper.LegacySetDataResultForBatching(f.Context(), dataResult) + require.NoError(t, err) + err = f.batchingKeeper.LegacyMarkDataResultAsBatched(f.Context(), dataResult, i) + require.NoError(t, err) + } + } + + dataResults, err := f.batchingKeeper.GetLegacyDataResults(f.Context(), true) + require.NoError(t, err) + require.Equal(t, 1000, len(dataResults)) + + for _, dataResult := range dataResults { + _, err := f.batchingKeeper.GetDataResult(f.Context(), dataResult.DrId, dataResult.DrBlockHeight) + require.NoError(t, err) + _, err = f.batchingKeeper.GetBatchAssignment(f.Context(), dataResult.DrId, dataResult.DrBlockHeight) + require.NoError(t, err) + } + + // Activate pruning of legacy data results. + err = f.batchingKeeper.SetHasPruningCaughtUp(f.Context(), true) + require.NoError(t, err) + err = f.batchingKeeper.SetBatchNumberAtUpgrade(f.Context()) + require.NoError(t, err) + + for i := 0; i < 10; i++ { + f.AddBlock() + + err = f.batchingKeeper.EndBlock(f.Context()) + require.NoError(t, err) + + res, err := f.batchingKeeper.GetLegacyDataResults(f.Context(), true) + require.NoError(t, err) + + expectedCount := max(1000-101*(i+1), 0) + require.Equal(t, expectedCount, len(res)) + } + + for _, dataResult := range dataResults { + _, err := f.batchingKeeper.GetDataResult(f.Context(), dataResult.DrId, dataResult.DrBlockHeight) + require.ErrorIs(t, err, collections.ErrNotFound) + _, err = f.batchingKeeper.GetBatchAssignment(f.Context(), dataResult.DrId, dataResult.DrBlockHeight) + require.ErrorIs(t, err, collections.ErrNotFound) + } +} + +// TestSimplePruning tests simple pruning with batch pruning disabled. +func TestSimplePruning(t *testing.T) { + f := initFixture(t) + + f.addBatchSigningValidators(t, 10) + + err := f.pubKeyKeeper.SetProvingScheme(f.Context(), pubkeytypes.ProvingScheme{ + Index: uint32(sedatypes.SEDAKeyIndexSecp256k1), + IsActivated: true, + }) + require.NoError(t, err) + + params := types.Params{ + NumBatchesToKeep: 15, + MaxBatchPrunePerBlock: 0, // disable batch pruning + } + err = f.batchingKeeper.SetParams(f.Context(), params) + require.NoError(t, err) + + // Create 30 batches with random associated data. + for range 30 { + f.AddBlock() + + err := f.batchingKeeper.SetDataResultForBatching(f.Context(), generateDataResults(t, 1)[0]) + require.NoError(t, err) + batch, dataEntries, valEntries, err := f.batchingKeeper.ConstructBatch(f.Context()) + require.NoError(t, err) + _, err = f.batchingKeeper.SetNewBatch(f.Context(), batch, dataEntries, valEntries) + require.NoError(t, err) + err = f.batchingKeeper.SetBatchSigSecp256k1(f.Context(), batch.BatchNumber, valEntries[0].ValidatorAddress, generateRandomBytes(64)) + require.NoError(t, err) + } + + batches, err := f.batchingKeeper.GetAllBatches(f.Context()) + require.NoError(t, err) + require.Equal(t, 30, len(batches)) + + // Should not prune anything because no batch is created. + f.AddBlock() + err = f.batchingKeeper.EndBlock(f.Context()) + require.NoError(t, err) + + batches, err = f.batchingKeeper.GetAllBatches(f.Context()) + require.NoError(t, err) + require.Equal(t, 30, len(batches)) + + // Should create 31st batch Batch 30 and prune Batch 15. + f.AddBlock() + err = f.batchingKeeper.SetDataResultForBatching(f.Context(), generateDataResults(t, 1)[0]) + require.NoError(t, err) + err = f.batchingKeeper.EndBlock(f.Context()) + require.NoError(t, err) + + batches, err = f.batchingKeeper.GetAllBatches(f.Context()) + require.NoError(t, err) + require.Equal(t, 30, len(batches)) + + for i := 0; i < 15; i++ { + f.checkBatchData(t, uint64(i), true) + } + f.checkNoBatchData(t, 15) + for i := 16; i <= 30; i++ { + f.checkBatchData(t, uint64(i), false) + } +} + +func TestNoSimplePruningUntilNumBatchesToKeepIsReached(t *testing.T) { + f := initFixture(t) + + f.addBatchSigningValidators(t, 10) + + err := f.pubKeyKeeper.SetProvingScheme(f.Context(), pubkeytypes.ProvingScheme{ + Index: uint32(sedatypes.SEDAKeyIndexSecp256k1), + IsActivated: true, + }) + require.NoError(t, err) + + err = f.batchingKeeper.SetParams(f.Context(), types.Params{ + NumBatchesToKeep: 11, + MaxBatchPrunePerBlock: 0, + }) + require.NoError(t, err) + + // Create 10 batches with random associated data. + for range 10 { + f.AddBlock() + + err := f.batchingKeeper.SetDataResultForBatching(f.Context(), generateDataResults(t, 1)[0]) + require.NoError(t, err) + batch, dataEntries, valEntries, err := f.batchingKeeper.ConstructBatch(f.Context()) + require.NoError(t, err) + _, err = f.batchingKeeper.SetNewBatch(f.Context(), batch, dataEntries, valEntries) + require.NoError(t, err) + err = f.batchingKeeper.SetBatchSigSecp256k1(f.Context(), batch.BatchNumber, valEntries[0].ValidatorAddress, generateRandomBytes(64)) + require.NoError(t, err) + } + + batches, err := f.batchingKeeper.GetAllBatches(f.Context()) + require.NoError(t, err) + require.Equal(t, 10, len(batches)) + + // Should create 11th batch Batch 10 and not prune anything. + f.AddBlock() + err = f.batchingKeeper.SetDataResultForBatching(f.Context(), generateDataResults(t, 1)[0]) + require.NoError(t, err) + err = f.batchingKeeper.EndBlock(f.Context()) + require.NoError(t, err) + + batches, err = f.batchingKeeper.GetAllBatches(f.Context()) + require.NoError(t, err) + require.Equal(t, 11, len(batches)) + + // Should create 12th batch Batch 11 and prune Batch 0. + f.AddBlock() + err = f.batchingKeeper.SetDataResultForBatching(f.Context(), generateDataResults(t, 1)[0]) + require.NoError(t, err) + err = f.batchingKeeper.EndBlock(f.Context()) + require.NoError(t, err) + + batches, err = f.batchingKeeper.GetAllBatches(f.Context()) + require.NoError(t, err) + require.Equal(t, 11, len(batches)) + require.Equal(t, uint64(1), batches[0].BatchNumber) + f.checkNoBatchData(t, 0) + for i := 1; i <= 9; i++ { + f.checkBatchData(t, uint64(i), true) + } + f.checkBatchData(t, 10, false) // did not add signatures for latest two batches + f.checkBatchData(t, 11, false) +} + +func TestPruningMockedUpgrade(t *testing.T) { + f := initFixture(t) + + f.addBatchSigningValidators(t, 10) + + err := f.pubKeyKeeper.SetProvingScheme(f.Context(), pubkeytypes.ProvingScheme{ + Index: uint32(sedatypes.SEDAKeyIndexSecp256k1), + IsActivated: true, + }) + require.NoError(t, err) + + err = f.batchingKeeper.SetParams(f.Context(), types.Params{ + NumBatchesToKeep: 10, + MaxBatchPrunePerBlock: 15, + MaxLegacyDataResultPrunePerBlock: 80, + }) + require.NoError(t, err) + + // Create 30 batches with 10 data results each before mock upgrade. + // We simulate the chain before the upgrade by using legacy functions. + for range 30 { + f.AddBlock() + + dataResults := generateDataResults(t, 10) + for _, dataResult := range dataResults { + err := f.batchingKeeper.LegacySetDataResultForBatching(f.Context(), dataResult) + require.NoError(t, err) + } + batch, dataEntries, valEntries, err := f.batchingKeeper.LegacyConstructBatch(f.Context()) + require.NoError(t, err) + _, err = f.batchingKeeper.SetNewBatch(f.Context(), batch, dataEntries, valEntries) + require.NoError(t, err) + err = f.batchingKeeper.SetBatchSigSecp256k1(f.Context(), batch.BatchNumber, valEntries[0].ValidatorAddress, generateRandomBytes(64)) + require.NoError(t, err) + } + + // Mock upgrade at height 30: + // - Upgrade handler should set batchNumberAtUpgrade and hasPruningCaughtUp. + err = f.batchingKeeper.SetBatchNumberAtUpgrade(f.Context()) + require.NoError(t, err) + err = f.batchingKeeper.SetHasPruningCaughtUp(f.Context(), false) + require.NoError(t, err) + + // Block 31: + // - Creates 31st batch Batch 30 + // - Batch prunes Batches 0-14 + f.BatchingEndBlock(t, 10) + + batches, err := f.batchingKeeper.GetAllBatches(f.Context()) + require.NoError(t, err) + require.Equal(t, 16, len(batches)) + require.Equal(t, uint64(15), batches[0].BatchNumber) + require.Equal(t, uint64(30), batches[len(batches)-1].BatchNumber) + for i := 0; i <= 14; i++ { + f.checkNoBatchData(t, uint64(i)) + } + for i := 15; i <= 30; i++ { + f.checkBatchData(t, uint64(i), false) + } + f.checkNumLegacyDataResults(t, 300) + + hasCaughtUp, err := f.batchingKeeper.HasPruningCaughtUp(f.Context()) + require.NoError(t, err) + require.False(t, hasCaughtUp) + + // Block 32~39: + // - Creates 32nd batch Batch 31 + // - Batch pruning in effect but limited by NumBatchesToKeep + for i := range 8 { + f.BatchingEndBlock(t, 10) + + batches, err = f.batchingKeeper.GetAllBatches(f.Context()) + require.NoError(t, err) + require.Equal(t, 10, len(batches)) + require.Equal(t, uint64(22+i), batches[0].BatchNumber) + require.Equal(t, uint64(31+i), batches[len(batches)-1].BatchNumber) + for j := 0; j <= 21+i; j++ { + f.checkNoBatchData(t, uint64(j)) + } + for j := 22 + i; j <= 31+i; j++ { + f.checkBatchData(t, uint64(j), false) + } + f.checkNumLegacyDataResults(t, 300) + + hasCaughtUp, err = f.batchingKeeper.HasPruningCaughtUp(f.Context()) + require.NoError(t, err) + require.False(t, hasCaughtUp) + } + + // Block 40 - 43: + // - Batch creation at every block but number of batches stays at 10 with simple pruning. + // - HasPruningCaughtUp is now True and legacy data results pruning is in effect. + for i := range 4 { + f.BatchingEndBlock(t, 10) + + batches, err = f.batchingKeeper.GetAllBatches(f.Context()) + require.NoError(t, err) + require.Equal(t, 10, len(batches)) + require.Equal(t, uint64(30+i), batches[0].BatchNumber) + require.Equal(t, uint64(39+i), batches[len(batches)-1].BatchNumber) + for j := 0; j <= 29+i; j++ { + f.checkNoBatchData(t, uint64(j)) + } + for j := 30 + i; j <= 39+i; j++ { + f.checkBatchData(t, uint64(j), false) + } + f.checkNumLegacyDataResults(t, 300-80*i) + + hasCaughtUp, err = f.batchingKeeper.HasPruningCaughtUp(f.Context()) + require.NoError(t, err) + require.True(t, hasCaughtUp) + } + + // Block 44 without batch creation + f.BatchingEndBlock(t, 0) + f.checkNumLegacyDataResults(t, 0) + + // Block 45 without batch creation + f.BatchingEndBlock(t, 5) + f.checkNumLegacyDataResults(t, 0) + +} + +// BatchingEndBlock adds a given number of data results to the store and executes +// batching EndBlock. Note if there is no change in data result or validator tree +// root, no new batch is created. +func (f *fixture) BatchingEndBlock(t *testing.T, numDataResults int) { + f.AddBlock() + if numDataResults > 0 { + dataResults := generateDataResults(t, numDataResults) + for _, dataResult := range dataResults { + err := f.batchingKeeper.SetDataResultForBatching(f.Context(), dataResult) + require.NoError(t, err) + } + } + err := f.batchingKeeper.EndBlock(f.Context()) + require.NoError(t, err) +} + +func (f *fixture) checkNoBatchData(t *testing.T, batchNum uint64) { + batch, err := f.batchingKeeper.GetBatchByBatchNumber(f.Context(), batchNum) + require.ErrorIs(t, err, collections.ErrNotFound) + dataEntries, err := f.batchingKeeper.GetDataResultTreeEntries(f.Context(), batchNum) + require.ErrorIs(t, err, collections.ErrNotFound) + valEntries, _ := f.batchingKeeper.GetValidatorTreeEntries(f.Context(), batchNum) + // require.ErrorIs(t, err, collections.ErrNotFound) // this function does not error even if there are no entries. + sigs, _ := f.batchingKeeper.GetBatchSignatures(f.Context(), batchNum) + // require.ErrorIs(t, err, collections.ErrNotFound) // this function does not error even if there are no entries. + + // TODO batchDataResults and dataResults + + require.Empty(t, batch, "batchNum: %d", batchNum) + require.Empty(t, dataEntries, "batchNum: %d", batchNum) + require.Empty(t, valEntries, "batchNum: %d", batchNum) + require.Empty(t, sigs, "batchNum: %d", batchNum) +} + +func (f *fixture) checkBatchData(t *testing.T, batchNum uint64, checkSigs bool) { + batch, err := f.batchingKeeper.GetBatchByBatchNumber(f.Context(), batchNum) + require.NoError(t, err) + dataEntries, err := f.batchingKeeper.GetDataResultTreeEntries(f.Context(), batchNum) + require.NoError(t, err) + valEntries, err := f.batchingKeeper.GetValidatorTreeEntries(f.Context(), batchNum) + require.NoError(t, err) + + // TODO batchDataResults and dataResults + + require.NotEmpty(t, batch, "batch number %d batch should not be empty", batchNum) + require.NotEmpty(t, dataEntries, "batch number %d data entriesshould not be empty", batchNum) + require.NotEmpty(t, valEntries, "batch number %d validator entries should not be empty", batchNum) + if checkSigs { + sigs, err := f.batchingKeeper.GetBatchSignatures(f.Context(), batchNum) + require.NoError(t, err) + require.NotEmpty(t, sigs, "batch number %d signatures should not be empty", batchNum) + } +} + +func (f *fixture) checkNumLegacyDataResults(t *testing.T, expectedNum int) { + dataResults, err := f.batchingKeeper.GetLegacyDataResults(f.Context(), true) + require.NoError(t, err) + require.Equal(t, expectedNum, len(dataResults)) +} diff --git a/x/batching/keeper/querier.go b/x/batching/keeper/querier.go index a1d01942..2e5c4c38 100644 --- a/x/batching/keeper/querier.go +++ b/x/batching/keeper/querier.go @@ -90,7 +90,7 @@ func (q Querier) Batches(c context.Context, req *types.QueryBatchesRequest) (*ty func (q Querier) DataResult(c context.Context, req *types.QueryDataResultRequest) (*types.QueryDataResultResponse, error) { ctx := sdk.UnwrapSDKContext(c) - var dataResult *types.DataResult + var dataResult types.DataResult var err error if req.DataRequestHeight == 0 { dataResult, err = q.GetLatestDataResult(ctx, req.DataRequestId) @@ -106,7 +106,7 @@ func (q Querier) DataResult(c context.Context, req *types.QueryDataResultRequest } result := &types.QueryDataResultResponse{ - DataResult: dataResult, + DataResult: &dataResult, } batchNum, err := q.GetBatchAssignment(ctx, req.DataRequestId, dataResult.DrBlockHeight) diff --git a/x/batching/types/batching.pb.go b/x/batching/types/batching.pb.go index 35aa4453..c8f80d4d 100644 --- a/x/batching/types/batching.pb.go +++ b/x/batching/types/batching.pb.go @@ -433,6 +433,106 @@ func (m *DataResult) GetConsensus() bool { return false } +// DataRequestIDHeights is a collection of DataRequestIDHeight objects. +type DataRequestIDHeights struct { + DataRequestIdHeights []DataRequestIDHeight `protobuf:"bytes,1,rep,name=data_request_id_heights,json=dataRequestIdHeights,proto3" json:"data_request_id_heights"` +} + +func (m *DataRequestIDHeights) Reset() { *m = DataRequestIDHeights{} } +func (m *DataRequestIDHeights) String() string { return proto.CompactTextString(m) } +func (*DataRequestIDHeights) ProtoMessage() {} +func (*DataRequestIDHeights) Descriptor() ([]byte, []int) { + return fileDescriptor_5b2a028024867de2, []int{5} +} +func (m *DataRequestIDHeights) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *DataRequestIDHeights) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_DataRequestIDHeights.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *DataRequestIDHeights) XXX_Merge(src proto.Message) { + xxx_messageInfo_DataRequestIDHeights.Merge(m, src) +} +func (m *DataRequestIDHeights) XXX_Size() int { + return m.Size() +} +func (m *DataRequestIDHeights) XXX_DiscardUnknown() { + xxx_messageInfo_DataRequestIDHeights.DiscardUnknown(m) +} + +var xxx_messageInfo_DataRequestIDHeights proto.InternalMessageInfo + +func (m *DataRequestIDHeights) GetDataRequestIdHeights() []DataRequestIDHeight { + if m != nil { + return m.DataRequestIdHeights + } + return nil +} + +// DataRequestIDHeight is a pair of data request ID and its posted height. +type DataRequestIDHeight struct { + // DataRequestID is the hex-encoded data request ID. + DataRequestId string `protobuf:"bytes,1,opt,name=data_request_id,json=dataRequestId,proto3" json:"data_request_id,omitempty"` + // DataRequestHeight is the height at which the data request was submitted. + DataRequestHeight uint64 `protobuf:"varint,2,opt,name=data_request_height,json=dataRequestHeight,proto3" json:"data_request_height,omitempty"` +} + +func (m *DataRequestIDHeight) Reset() { *m = DataRequestIDHeight{} } +func (m *DataRequestIDHeight) String() string { return proto.CompactTextString(m) } +func (*DataRequestIDHeight) ProtoMessage() {} +func (*DataRequestIDHeight) Descriptor() ([]byte, []int) { + return fileDescriptor_5b2a028024867de2, []int{6} +} +func (m *DataRequestIDHeight) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *DataRequestIDHeight) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_DataRequestIDHeight.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *DataRequestIDHeight) XXX_Merge(src proto.Message) { + xxx_messageInfo_DataRequestIDHeight.Merge(m, src) +} +func (m *DataRequestIDHeight) XXX_Size() int { + return m.Size() +} +func (m *DataRequestIDHeight) XXX_DiscardUnknown() { + xxx_messageInfo_DataRequestIDHeight.DiscardUnknown(m) +} + +var xxx_messageInfo_DataRequestIDHeight proto.InternalMessageInfo + +func (m *DataRequestIDHeight) GetDataRequestId() string { + if m != nil { + return m.DataRequestId + } + return "" +} + +func (m *DataRequestIDHeight) GetDataRequestHeight() uint64 { + if m != nil { + return m.DataRequestHeight + } + return 0 +} + // Params defines the parameters for the batching module. type Params struct { // NumBatchesToKeep is the number of batches to keep in the state without @@ -441,16 +541,16 @@ type Params struct { // MaxBatchPrunePerBlock is the maximum number of batches to prune per // block. MaxBatchPrunePerBlock uint64 `protobuf:"varint,2,opt,name=max_batch_prune_per_block,json=maxBatchPrunePerBlock,proto3" json:"max_batch_prune_per_block,omitempty"` - // MaxDataResultsToCheckForPrune is the maximum number of data results to - // check for pruning per block. - MaxDataResultsToCheckForPrune uint64 `protobuf:"varint,3,opt,name=max_data_results_to_check_for_prune,json=maxDataResultsToCheckForPrune,proto3" json:"max_data_results_to_check_for_prune,omitempty"` + // MaxLegacyDataResultPrunePerBlock is the maximum number of legacy data + // results to be checked for pruning per block. + MaxLegacyDataResultPrunePerBlock uint64 `protobuf:"varint,3,opt,name=max_legacy_data_result_prune_per_block,json=maxLegacyDataResultPrunePerBlock,proto3" json:"max_legacy_data_result_prune_per_block,omitempty"` } func (m *Params) Reset() { *m = Params{} } func (m *Params) String() string { return proto.CompactTextString(m) } func (*Params) ProtoMessage() {} func (*Params) Descriptor() ([]byte, []int) { - return fileDescriptor_5b2a028024867de2, []int{5} + return fileDescriptor_5b2a028024867de2, []int{7} } func (m *Params) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -493,9 +593,9 @@ func (m *Params) GetMaxBatchPrunePerBlock() uint64 { return 0 } -func (m *Params) GetMaxDataResultsToCheckForPrune() uint64 { +func (m *Params) GetMaxLegacyDataResultPrunePerBlock() uint64 { if m != nil { - return m.MaxDataResultsToCheckForPrune + return m.MaxLegacyDataResultPrunePerBlock } return 0 } @@ -506,6 +606,8 @@ func init() { proto.RegisterType((*ValidatorTreeEntry)(nil), "sedachain.batching.v1.ValidatorTreeEntry") proto.RegisterType((*BatchSignatures)(nil), "sedachain.batching.v1.BatchSignatures") proto.RegisterType((*DataResult)(nil), "sedachain.batching.v1.DataResult") + proto.RegisterType((*DataRequestIDHeights)(nil), "sedachain.batching.v1.DataRequestIDHeights") + proto.RegisterType((*DataRequestIDHeight)(nil), "sedachain.batching.v1.DataRequestIDHeight") proto.RegisterType((*Params)(nil), "sedachain.batching.v1.Params") } @@ -514,64 +616,69 @@ func init() { } var fileDescriptor_5b2a028024867de2 = []byte{ - // 904 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x55, 0xcd, 0x8e, 0xe3, 0x44, - 0x10, 0x5e, 0x67, 0x32, 0xf9, 0xe9, 0x24, 0x33, 0xa1, 0x67, 0x83, 0x3c, 0x2b, 0x11, 0x87, 0xc0, - 0x4a, 0x01, 0x94, 0x84, 0x10, 0xf1, 0x23, 0xc1, 0x85, 0x2c, 0x20, 0x06, 0xb4, 0xab, 0xa8, 0x19, - 0xf6, 0xc0, 0x01, 0xab, 0xe3, 0x6e, 0x62, 0x2b, 0xb1, 0xdb, 0xea, 0x6e, 0x87, 0xcc, 0x5b, 0xf0, - 0x02, 0x5c, 0x78, 0x06, 0xae, 0x9c, 0xe1, 0xb8, 0xe2, 0x84, 0x40, 0xb2, 0xd0, 0xcc, 0x2d, 0x8f, - 0xc0, 0x09, 0xb9, 0xda, 0x71, 0x66, 0x86, 0x33, 0x27, 0x77, 0x7d, 0x5f, 0x55, 0x75, 0x55, 0xf7, - 0x57, 0x6d, 0xf4, 0xba, 0xe2, 0x8c, 0x7a, 0x3e, 0x0d, 0xa2, 0xf1, 0x82, 0x6a, 0xcf, 0x0f, 0xa2, - 0xe5, 0x78, 0x33, 0x29, 0xd6, 0xa3, 0x58, 0x0a, 0x2d, 0x70, 0xa7, 0xf0, 0x1a, 0x15, 0xcc, 0x66, - 0xf2, 0xe8, 0xdc, 0x13, 0x2a, 0x14, 0xca, 0x05, 0xa7, 0xb1, 0x31, 0x4c, 0xc4, 0xa3, 0x87, 0x4b, - 0xb1, 0x14, 0x06, 0xcf, 0x56, 0x06, 0xed, 0xff, 0x58, 0x42, 0xc7, 0xb3, 0x2c, 0x01, 0x7e, 0x15, - 0x35, 0x21, 0x93, 0x1b, 0x25, 0xe1, 0x82, 0x4b, 0xdb, 0xea, 0x59, 0x83, 0x32, 0x69, 0x00, 0xf6, - 0x0c, 0x20, 0x70, 0x59, 0x0b, 0x6f, 0xe5, 0xfa, 0x3c, 0x58, 0xfa, 0xda, 0x2e, 0xf5, 0xac, 0xc1, - 0x11, 0x69, 0x00, 0xf6, 0x39, 0x40, 0xf8, 0x7d, 0x64, 0x7b, 0x89, 0x94, 0x3c, 0xd2, 0x2e, 0xa3, - 0x9a, 0xba, 0x92, 0xab, 0x64, 0xad, 0x5d, 0x29, 0x84, 0xb6, 0x8f, 0x7a, 0xd6, 0xa0, 0x4e, 0x3a, - 0x39, 0xff, 0x09, 0xd5, 0x94, 0x00, 0x4b, 0x84, 0xd0, 0x78, 0x80, 0xda, 0xff, 0x09, 0x28, 0x43, - 0xc0, 0x09, 0xbb, 0xeb, 0xf9, 0x18, 0x9d, 0x6c, 0xe8, 0x3a, 0x60, 0x54, 0x0b, 0x69, 0xfc, 0x8e, - 0xc1, 0xaf, 0x55, 0xa0, 0xe0, 0x76, 0x8e, 0x6a, 0xa6, 0x9f, 0x80, 0xd9, 0x95, 0x9e, 0x35, 0x68, - 0x92, 0x2a, 0xd8, 0x17, 0x0c, 0xbf, 0x81, 0xda, 0xb1, 0x14, 0x9b, 0x20, 0x5a, 0xba, 0x21, 0xd7, - 0x34, 0xcb, 0x6f, 0x57, 0xc1, 0xe5, 0x34, 0xc7, 0x9f, 0xe6, 0x70, 0x7f, 0x82, 0x3a, 0x87, 0x42, - 0x2f, 0x25, 0xe7, 0x9f, 0x46, 0x5a, 0x06, 0x5c, 0x61, 0x1b, 0x55, 0xb9, 0x59, 0xda, 0x56, 0xef, - 0x28, 0xcb, 0x9e, 0x9b, 0xfd, 0x5f, 0x2d, 0x84, 0x9f, 0xef, 0x4b, 0xd9, 0x87, 0x5c, 0xe1, 0x6f, - 0xd1, 0x4b, 0x87, 0xb2, 0x29, 0x63, 0x92, 0x2b, 0x05, 0x87, 0xdc, 0x9c, 0x4d, 0xfe, 0x49, 0x9d, - 0xe1, 0x32, 0xd0, 0x7e, 0xb2, 0x18, 0x79, 0x22, 0xcc, 0xef, 0x2d, 0xff, 0x0c, 0x15, 0x5b, 0x8d, - 0xf5, 0x55, 0xcc, 0xd5, 0xe8, 0x39, 0x5d, 0x7f, 0x6c, 0x02, 0x49, 0xbb, 0xc8, 0x95, 0x23, 0xf8, - 0x6d, 0xf4, 0x70, 0x23, 0x74, 0xd6, 0x53, 0x2c, 0xbe, 0xe7, 0xd2, 0x8d, 0xb9, 0xf4, 0x78, 0x64, - 0x2e, 0xa9, 0x45, 0xb0, 0xe1, 0xe6, 0x19, 0x35, 0x37, 0x0c, 0x76, 0x50, 0x83, 0x6b, 0xbf, 0xa8, - 0xe5, 0x08, 0x4e, 0x00, 0x71, 0xed, 0xe7, 0x29, 0xfb, 0x3f, 0x59, 0xe8, 0x14, 0xc4, 0xf1, 0x55, - 0xb0, 0x8c, 0xa8, 0x4e, 0x24, 0x57, 0xff, 0x7b, 0x1b, 0x63, 0x74, 0xa6, 0xb8, 0x17, 0xbf, 0xf3, - 0xee, 0x7b, 0xab, 0x89, 0xab, 0xf6, 0xfb, 0x42, 0x17, 0x4d, 0x82, 0x0b, 0xaa, 0xa8, 0xa8, 0xff, - 0x57, 0x19, 0xa1, 0xc3, 0x15, 0xe1, 0x97, 0x51, 0x29, 0x60, 0x50, 0x50, 0x7d, 0x56, 0xd9, 0xa5, - 0x4e, 0x29, 0x60, 0xa4, 0x14, 0x30, 0xdc, 0x45, 0xc7, 0x4c, 0x66, 0x5a, 0x28, 0x01, 0x55, 0xdf, - 0xa5, 0x8e, 0x01, 0x48, 0x99, 0xc9, 0x0b, 0x86, 0x3f, 0x44, 0xa7, 0x4c, 0xba, 0x77, 0xe4, 0x9d, - 0x1d, 0x48, 0x79, 0x76, 0xb6, 0x4b, 0x9d, 0xfb, 0x14, 0x69, 0x31, 0x39, 0xbb, 0xa5, 0xfa, 0xc7, - 0xa8, 0xba, 0xe1, 0x52, 0x05, 0x22, 0x32, 0x9a, 0x9d, 0x35, 0x76, 0xa9, 0xb3, 0x87, 0xc8, 0x7e, - 0x81, 0xa7, 0xf7, 0xe6, 0xe7, 0x18, 0x36, 0x68, 0xef, 0x52, 0xe7, 0x0e, 0x7e, 0x77, 0xa2, 0x3e, - 0x42, 0xa7, 0x86, 0xd4, 0x41, 0xc8, 0x95, 0xa6, 0x61, 0x0c, 0x72, 0xce, 0x0b, 0xbb, 0x47, 0x91, - 0x13, 0x00, 0x2e, 0xf7, 0x36, 0x7e, 0x13, 0xd5, 0xf9, 0x36, 0xd0, 0xae, 0x27, 0x18, 0x07, 0x8d, - 0xb7, 0x66, 0xad, 0x5d, 0xea, 0x1c, 0x40, 0x52, 0xcb, 0x96, 0x4f, 0x04, 0xe3, 0xf8, 0x19, 0xaa, - 0x2d, 0xa9, 0x72, 0x13, 0xc5, 0x99, 0x5d, 0x83, 0x36, 0xa6, 0x7f, 0xa6, 0x4e, 0xc7, 0xdc, 0x9f, - 0x62, 0xab, 0x51, 0x20, 0xc6, 0x21, 0xd5, 0xfe, 0xe8, 0x22, 0xd2, 0xbb, 0xd4, 0x29, 0x9c, 0x7f, - 0xff, 0x79, 0x88, 0xf2, 0xa7, 0xe6, 0x22, 0xd2, 0xa4, 0xba, 0xa4, 0xea, 0x6b, 0xc5, 0x19, 0xee, - 0xa3, 0x8a, 0x99, 0x66, 0xbb, 0x0e, 0xfa, 0x40, 0xbb, 0xd4, 0xc9, 0x11, 0x92, 0x7f, 0xb3, 0xee, - 0x62, 0x7a, 0xb5, 0xa0, 0xde, 0xaa, 0x10, 0x13, 0x82, 0xad, 0xa1, 0xbb, 0x7b, 0x14, 0x39, 0xc9, - 0x81, 0xbd, 0x58, 0xa6, 0xa8, 0x99, 0xbd, 0x83, 0x6e, 0x4c, 0xaf, 0xd6, 0x82, 0x32, 0xbb, 0x01, - 0xa1, 0x70, 0xa0, 0xb7, 0x71, 0xd2, 0xc8, 0xac, 0xb9, 0x31, 0xf0, 0x5b, 0xa8, 0xee, 0x89, 0x48, - 0xf1, 0x48, 0x25, 0xca, 0x6e, 0xf6, 0xac, 0x41, 0xcd, 0x1c, 0x49, 0x01, 0x92, 0xc3, 0xb2, 0xff, - 0x8b, 0x85, 0x2a, 0x73, 0x2a, 0x69, 0xa8, 0xf0, 0x10, 0x9d, 0x45, 0x49, 0xe8, 0xc2, 0x23, 0xc2, - 0x95, 0xab, 0x85, 0xbb, 0xe2, 0x3c, 0xce, 0xdf, 0xc9, 0x76, 0x94, 0x84, 0x33, 0xc3, 0x5c, 0x8a, - 0x2f, 0x39, 0x8f, 0xf1, 0x07, 0xe8, 0x3c, 0xa4, 0x5b, 0xe3, 0xee, 0xc6, 0x32, 0x89, 0x78, 0x36, - 0x92, 0x46, 0x46, 0x20, 0xc2, 0x32, 0xe9, 0x84, 0x74, 0x0b, 0x41, 0xf3, 0x8c, 0x9e, 0x73, 0xa3, - 0x29, 0xfc, 0x05, 0x7a, 0x2d, 0x8b, 0xbc, 0xf5, 0x1c, 0xc2, 0x6e, 0x9e, 0xcf, 0xbd, 0x95, 0xfb, - 0x9d, 0x90, 0x26, 0x9b, 0x91, 0x27, 0x79, 0x25, 0xa4, 0xdb, 0x83, 0xfc, 0xd5, 0xa5, 0x78, 0x92, - 0xb9, 0x7d, 0x26, 0x24, 0xe4, 0x9c, 0x3d, 0xfd, 0xed, 0xba, 0x6b, 0xbd, 0xb8, 0xee, 0x5a, 0x7f, - 0x5f, 0x77, 0xad, 0x1f, 0x6e, 0xba, 0x0f, 0x5e, 0xdc, 0x74, 0x1f, 0xfc, 0x71, 0xd3, 0x7d, 0xf0, - 0xcd, 0xf4, 0xd6, 0xa4, 0x66, 0xc7, 0x03, 0xff, 0x03, 0x4f, 0xac, 0xc1, 0x18, 0x9a, 0x1f, 0xd0, - 0xf6, 0xf0, 0x0b, 0x82, 0xd1, 0x5d, 0x54, 0xc0, 0x6b, 0xfa, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, - 0xc7, 0x3e, 0x8b, 0x07, 0xa5, 0x06, 0x00, 0x00, + // 980 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x55, 0xcb, 0x6e, 0x23, 0x45, + 0x17, 0x4e, 0x3b, 0xce, 0xc5, 0xc7, 0x76, 0x92, 0xa9, 0x24, 0xff, 0xdf, 0x99, 0x85, 0xdb, 0x58, + 0xcc, 0xc8, 0x0c, 0x8a, 0x4d, 0x26, 0xe2, 0x22, 0xc1, 0x86, 0x66, 0x90, 0x88, 0x60, 0x46, 0x56, + 0x11, 0x66, 0xc1, 0x82, 0x56, 0xa5, 0xab, 0xd4, 0x6e, 0xd9, 0xdd, 0xd5, 0x54, 0x55, 0x1b, 0x7b, + 0xc5, 0x2b, 0xf0, 0x02, 0x6c, 0x78, 0x06, 0xde, 0x80, 0x05, 0xb3, 0x1c, 0xb1, 0x42, 0x20, 0xb5, + 0x50, 0xb2, 0xf3, 0x23, 0xb0, 0x42, 0x5d, 0xd5, 0x6e, 0x5f, 0xc8, 0x96, 0x55, 0x57, 0x7d, 0xdf, + 0x77, 0x4e, 0x9d, 0xea, 0x73, 0x29, 0x78, 0x53, 0x32, 0x4a, 0xfc, 0x21, 0x09, 0xe3, 0xfe, 0x0d, + 0x51, 0xfe, 0x30, 0x8c, 0x83, 0xfe, 0xe4, 0xa2, 0x5c, 0xf7, 0x12, 0xc1, 0x15, 0x47, 0xa7, 0xa5, + 0xaa, 0x57, 0x32, 0x93, 0x8b, 0x87, 0x67, 0x3e, 0x97, 0x11, 0x97, 0x9e, 0x16, 0xf5, 0xcd, 0xc6, + 0x58, 0x3c, 0x3c, 0x09, 0x78, 0xc0, 0x0d, 0x9e, 0xaf, 0x0c, 0xda, 0xf9, 0xb1, 0x02, 0x3b, 0x6e, + 0xee, 0x00, 0xbd, 0x01, 0x0d, 0xed, 0xc9, 0x8b, 0xd3, 0xe8, 0x86, 0x09, 0xdb, 0x6a, 0x5b, 0xdd, + 0x2a, 0xae, 0x6b, 0xec, 0x85, 0x86, 0xb4, 0x64, 0xcc, 0xfd, 0x91, 0x37, 0x64, 0x61, 0x30, 0x54, + 0x76, 0xa5, 0x6d, 0x75, 0xb7, 0x71, 0x5d, 0x63, 0x9f, 0x69, 0x08, 0xbd, 0x0f, 0xb6, 0x9f, 0x0a, + 0xc1, 0x62, 0xe5, 0x51, 0xa2, 0x88, 0x27, 0x98, 0x4c, 0xc7, 0xca, 0x13, 0x9c, 0x2b, 0x7b, 0xbb, + 0x6d, 0x75, 0x6b, 0xf8, 0xb4, 0xe0, 0x9f, 0x11, 0x45, 0xb0, 0x66, 0x31, 0xe7, 0x0a, 0x75, 0xe1, + 0xe8, 0x5f, 0x06, 0x55, 0x6d, 0x70, 0x40, 0xd7, 0x95, 0x8f, 0xe0, 0x60, 0x42, 0xc6, 0x21, 0x25, + 0x8a, 0x0b, 0xa3, 0xdb, 0xd1, 0xba, 0x66, 0x89, 0x6a, 0xd9, 0x19, 0xec, 0x9b, 0xfb, 0x84, 0xd4, + 0xde, 0x6d, 0x5b, 0xdd, 0x06, 0xde, 0xd3, 0xfb, 0x2b, 0x8a, 0xde, 0x82, 0xa3, 0x44, 0xf0, 0x49, + 0x18, 0x07, 0x5e, 0xc4, 0x14, 0xc9, 0xfd, 0xdb, 0x7b, 0x5a, 0x72, 0x58, 0xe0, 0xcf, 0x0b, 0xb8, + 0x73, 0x01, 0xa7, 0xcb, 0x40, 0xaf, 0x05, 0x63, 0x9f, 0xc6, 0x4a, 0x84, 0x4c, 0x22, 0x1b, 0xf6, + 0x98, 0x59, 0xda, 0x56, 0x7b, 0x3b, 0xf7, 0x5e, 0x6c, 0x3b, 0xbf, 0x5a, 0x80, 0x5e, 0x2e, 0x42, + 0x59, 0x98, 0xcc, 0xd0, 0x37, 0xf0, 0x60, 0x19, 0x36, 0xa1, 0x54, 0x30, 0x29, 0xf5, 0x4f, 0x6e, + 0xb8, 0x17, 0x7f, 0x67, 0xce, 0x79, 0x10, 0xaa, 0x61, 0x7a, 0xd3, 0xf3, 0x79, 0x54, 0xe4, 0xad, + 0xf8, 0x9c, 0x4b, 0x3a, 0xea, 0xab, 0x59, 0xc2, 0x64, 0xef, 0x25, 0x19, 0x7f, 0x6c, 0x0c, 0xf1, + 0x51, 0xe9, 0xab, 0x40, 0xd0, 0x3b, 0x70, 0x32, 0xe1, 0x2a, 0xbf, 0x53, 0xc2, 0xbf, 0x63, 0xc2, + 0x4b, 0x98, 0xf0, 0x59, 0x6c, 0x92, 0xd4, 0xc4, 0xc8, 0x70, 0x83, 0x9c, 0x1a, 0x18, 0x06, 0x39, + 0x50, 0x67, 0x6a, 0x58, 0xc6, 0xb2, 0xad, 0xff, 0x00, 0x30, 0x35, 0x2c, 0x5c, 0x76, 0x7e, 0xb2, + 0xe0, 0x50, 0x17, 0xc7, 0x97, 0x61, 0x10, 0x13, 0x95, 0x0a, 0x26, 0xff, 0xf3, 0x6b, 0xf4, 0xe1, + 0x58, 0x32, 0x3f, 0x79, 0xfa, 0xee, 0x7b, 0xa3, 0x0b, 0x4f, 0x2e, 0xce, 0xd5, 0xb7, 0x68, 0x60, + 0x54, 0x52, 0x65, 0x44, 0x9d, 0x3f, 0xab, 0x00, 0xcb, 0x14, 0xa1, 0xff, 0x41, 0x25, 0xa4, 0x3a, + 0xa0, 0x9a, 0xbb, 0x3b, 0xcf, 0x9c, 0x4a, 0x48, 0x71, 0x25, 0xa4, 0xa8, 0x05, 0x3b, 0x54, 0xe4, + 0xb5, 0x50, 0xd1, 0x54, 0x6d, 0x9e, 0x39, 0x06, 0xc0, 0x55, 0x2a, 0xae, 0x28, 0xfa, 0x10, 0x0e, + 0xa9, 0xf0, 0xd6, 0xca, 0x3b, 0xff, 0x21, 0x55, 0xf7, 0x78, 0x9e, 0x39, 0x9b, 0x14, 0x6e, 0x52, + 0xe1, 0xae, 0x54, 0xfd, 0x23, 0xd8, 0x9b, 0x30, 0x21, 0x43, 0x1e, 0x9b, 0x9a, 0x75, 0xeb, 0xf3, + 0xcc, 0x59, 0x40, 0x78, 0xb1, 0x40, 0x97, 0x1b, 0xfd, 0xb3, 0xa3, 0x0f, 0x38, 0x9a, 0x67, 0xce, + 0x1a, 0xbe, 0xde, 0x51, 0x1f, 0xc1, 0xa1, 0x21, 0x55, 0x18, 0x31, 0xa9, 0x48, 0x94, 0xe8, 0x72, + 0x2e, 0x02, 0xdb, 0xa0, 0xf0, 0x81, 0x06, 0xae, 0x17, 0x7b, 0xf4, 0x04, 0x6a, 0x6c, 0x1a, 0x2a, + 0xcf, 0xe7, 0x94, 0xe9, 0x1a, 0x6f, 0xba, 0xcd, 0x79, 0xe6, 0x2c, 0x41, 0xbc, 0x9f, 0x2f, 0x3f, + 0xe1, 0x94, 0xa1, 0x17, 0xb0, 0x1f, 0x10, 0xe9, 0xa5, 0x92, 0x51, 0x7b, 0x5f, 0x5f, 0xe3, 0xf2, + 0x8f, 0xcc, 0x39, 0x35, 0xf9, 0x93, 0x74, 0xd4, 0x0b, 0x79, 0x3f, 0x22, 0x6a, 0xd8, 0xbb, 0x8a, + 0xd5, 0x3c, 0x73, 0x4a, 0xf1, 0x6f, 0x3f, 0x9f, 0x43, 0x31, 0x6a, 0xae, 0x62, 0x85, 0xf7, 0x02, + 0x22, 0xbf, 0x92, 0x8c, 0xa2, 0x0e, 0xec, 0x9a, 0x6e, 0xb6, 0x6b, 0xba, 0x3e, 0x60, 0x9e, 0x39, + 0x05, 0x82, 0x8b, 0x6f, 0x7e, 0xbb, 0x84, 0xcc, 0x6e, 0x88, 0x3f, 0x2a, 0x8b, 0x09, 0xf4, 0xd1, + 0xfa, 0x76, 0x1b, 0x14, 0x3e, 0x28, 0x80, 0x45, 0xb1, 0x5c, 0x42, 0x23, 0x9f, 0x83, 0x5e, 0x42, + 0x66, 0x63, 0x4e, 0xa8, 0x5d, 0xd7, 0xa6, 0xfa, 0x87, 0xae, 0xe2, 0xb8, 0x9e, 0xef, 0x06, 0x66, + 0x83, 0xde, 0x86, 0x9a, 0xcf, 0x63, 0xc9, 0x62, 0x99, 0x4a, 0xbb, 0xd1, 0xb6, 0xba, 0xfb, 0xe6, + 0x97, 0x94, 0x20, 0x5e, 0x2e, 0x3b, 0xdf, 0xc3, 0x89, 0x29, 0xae, 0x6f, 0x53, 0x26, 0xd5, 0xd5, + 0x33, 0x93, 0x14, 0x89, 0x02, 0xf8, 0x7f, 0x31, 0xae, 0x34, 0xe1, 0x85, 0xb4, 0x48, 0x9e, 0x19, + 0x07, 0xf5, 0xa7, 0x4f, 0x7a, 0xf7, 0x4e, 0xe8, 0xde, 0x3d, 0xde, 0xdc, 0xea, 0xab, 0xcc, 0xd9, + 0xc2, 0x27, 0x74, 0x85, 0xa2, 0xc5, 0x41, 0x9d, 0x08, 0x8e, 0xef, 0x31, 0x41, 0x8f, 0xe1, 0x70, + 0xe3, 0x7c, 0x53, 0xf3, 0xb8, 0xb9, 0xe6, 0x05, 0xf5, 0xe0, 0x78, 0x4d, 0xb7, 0x32, 0xb9, 0xab, + 0xf8, 0xc1, 0x8a, 0xd6, 0xf8, 0xed, 0xfc, 0x62, 0xc1, 0xee, 0x80, 0x08, 0x12, 0x49, 0x74, 0x0e, + 0xc7, 0x71, 0x1a, 0x79, 0x3a, 0x78, 0x26, 0x3d, 0xc5, 0xbd, 0x11, 0x63, 0x49, 0xf1, 0x2e, 0x1c, + 0xc5, 0x69, 0xe4, 0x1a, 0xe6, 0x9a, 0x7f, 0xce, 0x58, 0x82, 0x3e, 0x80, 0xb3, 0x88, 0x4c, 0x8d, + 0xdc, 0x4b, 0x44, 0x1a, 0xb3, 0x7c, 0x04, 0x99, 0xb6, 0x29, 0xce, 0x3b, 0x8d, 0xc8, 0x54, 0x1b, + 0x0d, 0x72, 0x7a, 0xc0, 0x4c, 0x0f, 0xa1, 0x01, 0x3c, 0xce, 0x2d, 0xc7, 0x2c, 0x20, 0xfe, 0x6c, + 0xed, 0xd9, 0xd8, 0x74, 0xa3, 0x3b, 0x12, 0xb7, 0x23, 0x32, 0xfd, 0x42, 0x8b, 0x97, 0x7d, 0xbf, + 0xe6, 0xd1, 0x7d, 0xfe, 0xea, 0xb6, 0x65, 0xbd, 0xbe, 0x6d, 0x59, 0x7f, 0xdd, 0xb6, 0xac, 0x1f, + 0xee, 0x5a, 0x5b, 0xaf, 0xef, 0x5a, 0x5b, 0xbf, 0xdf, 0xb5, 0xb6, 0xbe, 0xbe, 0x5c, 0x99, 0x4f, + 0x79, 0x82, 0xf4, 0x2b, 0xe8, 0xf3, 0xb1, 0xde, 0x9c, 0x9b, 0x67, 0x77, 0xba, 0x7c, 0x78, 0xf5, + 0xc0, 0xba, 0xd9, 0xd5, 0xaa, 0xcb, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x11, 0xe8, 0xaf, 0xfb, + 0x9b, 0x07, 0x00, 0x00, } func (m *Batch) Marshal() (dAtA []byte, err error) { @@ -860,6 +967,78 @@ func (m *DataResult) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *DataRequestIDHeights) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *DataRequestIDHeights) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *DataRequestIDHeights) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.DataRequestIdHeights) > 0 { + for iNdEx := len(m.DataRequestIdHeights) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.DataRequestIdHeights[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintBatching(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *DataRequestIDHeight) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *DataRequestIDHeight) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *DataRequestIDHeight) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.DataRequestHeight != 0 { + i = encodeVarintBatching(dAtA, i, uint64(m.DataRequestHeight)) + i-- + dAtA[i] = 0x10 + } + if len(m.DataRequestId) > 0 { + i -= len(m.DataRequestId) + copy(dAtA[i:], m.DataRequestId) + i = encodeVarintBatching(dAtA, i, uint64(len(m.DataRequestId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func (m *Params) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -880,8 +1059,8 @@ func (m *Params) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.MaxDataResultsToCheckForPrune != 0 { - i = encodeVarintBatching(dAtA, i, uint64(m.MaxDataResultsToCheckForPrune)) + if m.MaxLegacyDataResultPrunePerBlock != 0 { + i = encodeVarintBatching(dAtA, i, uint64(m.MaxLegacyDataResultPrunePerBlock)) i-- dAtA[i] = 0x18 } @@ -1048,6 +1227,37 @@ func (m *DataResult) Size() (n int) { return n } +func (m *DataRequestIDHeights) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.DataRequestIdHeights) > 0 { + for _, e := range m.DataRequestIdHeights { + l = e.Size() + n += 1 + l + sovBatching(uint64(l)) + } + } + return n +} + +func (m *DataRequestIDHeight) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.DataRequestId) + if l > 0 { + n += 1 + l + sovBatching(uint64(l)) + } + if m.DataRequestHeight != 0 { + n += 1 + sovBatching(uint64(m.DataRequestHeight)) + } + return n +} + func (m *Params) Size() (n int) { if m == nil { return 0 @@ -1060,8 +1270,8 @@ func (m *Params) Size() (n int) { if m.MaxBatchPrunePerBlock != 0 { n += 1 + sovBatching(uint64(m.MaxBatchPrunePerBlock)) } - if m.MaxDataResultsToCheckForPrune != 0 { - n += 1 + sovBatching(uint64(m.MaxDataResultsToCheckForPrune)) + if m.MaxLegacyDataResultPrunePerBlock != 0 { + n += 1 + sovBatching(uint64(m.MaxLegacyDataResultPrunePerBlock)) } return n } @@ -2037,6 +2247,191 @@ func (m *DataResult) Unmarshal(dAtA []byte) error { } return nil } +func (m *DataRequestIDHeights) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBatching + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: DataRequestIDHeights: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: DataRequestIDHeights: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DataRequestIdHeights", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBatching + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthBatching + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthBatching + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DataRequestIdHeights = append(m.DataRequestIdHeights, DataRequestIDHeight{}) + if err := m.DataRequestIdHeights[len(m.DataRequestIdHeights)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipBatching(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthBatching + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *DataRequestIDHeight) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBatching + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: DataRequestIDHeight: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: DataRequestIDHeight: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DataRequestId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBatching + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthBatching + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthBatching + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DataRequestId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field DataRequestHeight", wireType) + } + m.DataRequestHeight = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBatching + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.DataRequestHeight |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipBatching(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthBatching + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *Params) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -2106,9 +2501,9 @@ func (m *Params) Unmarshal(dAtA []byte) error { } case 3: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field MaxDataResultsToCheckForPrune", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field MaxLegacyDataResultPrunePerBlock", wireType) } - m.MaxDataResultsToCheckForPrune = 0 + m.MaxLegacyDataResultPrunePerBlock = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowBatching @@ -2118,7 +2513,7 @@ func (m *Params) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.MaxDataResultsToCheckForPrune |= uint64(b&0x7F) << shift + m.MaxLegacyDataResultPrunePerBlock |= uint64(b&0x7F) << shift if b < 0x80 { break } diff --git a/x/batching/types/genesis.go b/x/batching/types/genesis.go index cae91cc8..0d388dd1 100644 --- a/x/batching/types/genesis.go +++ b/x/batching/types/genesis.go @@ -16,22 +16,32 @@ func NewGenesisState( batches []Batch, batchData []BatchData, dataResults []GenesisDataResult, + legacyDataResults []GenesisDataResult, batchAssignments []BatchAssignment, params Params, + hasPruningCaughtUp bool, + batchNumberAtUpgrade uint64, ) GenesisState { return GenesisState{ - CurrentBatchNumber: curBatchNum, - Batches: batches, - BatchData: batchData, - DataResults: dataResults, - BatchAssignments: batchAssignments, - Params: params, + CurrentBatchNumber: curBatchNum, + Batches: batches, + BatchData: batchData, + DataResults: dataResults, + LegacyDataResults: legacyDataResults, + BatchAssignments: batchAssignments, + Params: params, + HasPruningCaughtUp: hasPruningCaughtUp, + BatchNumberAtUpgrade: batchNumberAtUpgrade, } } // DefaultGenesisState creates a default GenesisState object. func DefaultGenesisState() *GenesisState { - state := NewGenesisState(collections.DefaultSequenceStart, nil, nil, nil, nil, DefaultParams()) + state := NewGenesisState( + collections.DefaultSequenceStart, + nil, nil, nil, nil, nil, + DefaultParams(), true, 0, + ) return &state } diff --git a/x/batching/types/genesis.pb.go b/x/batching/types/genesis.pb.go index 1c0170d4..68a54bd4 100644 --- a/x/batching/types/genesis.pb.go +++ b/x/batching/types/genesis.pb.go @@ -27,12 +27,15 @@ const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package type GenesisState struct { // current_batch_number is the batch number of the most recently- // created batch. - CurrentBatchNumber uint64 `protobuf:"varint,1,opt,name=current_batch_number,json=currentBatchNumber,proto3" json:"current_batch_number,omitempty"` - Batches []Batch `protobuf:"bytes,2,rep,name=batches,proto3" json:"batches"` - BatchData []BatchData `protobuf:"bytes,3,rep,name=batch_data,json=batchData,proto3" json:"batch_data"` - DataResults []GenesisDataResult `protobuf:"bytes,4,rep,name=data_results,json=dataResults,proto3" json:"data_results"` - BatchAssignments []BatchAssignment `protobuf:"bytes,5,rep,name=batch_assignments,json=batchAssignments,proto3" json:"batch_assignments"` - Params Params `protobuf:"bytes,6,opt,name=params,proto3" json:"params"` + CurrentBatchNumber uint64 `protobuf:"varint,1,opt,name=current_batch_number,json=currentBatchNumber,proto3" json:"current_batch_number,omitempty"` + Batches []Batch `protobuf:"bytes,2,rep,name=batches,proto3" json:"batches"` + BatchData []BatchData `protobuf:"bytes,3,rep,name=batch_data,json=batchData,proto3" json:"batch_data"` + DataResults []GenesisDataResult `protobuf:"bytes,4,rep,name=data_results,json=dataResults,proto3" json:"data_results"` + BatchAssignments []BatchAssignment `protobuf:"bytes,5,rep,name=batch_assignments,json=batchAssignments,proto3" json:"batch_assignments"` + Params Params `protobuf:"bytes,6,opt,name=params,proto3" json:"params"` + LegacyDataResults []GenesisDataResult `protobuf:"bytes,7,rep,name=legacy_data_results,json=legacyDataResults,proto3" json:"legacy_data_results"` + HasPruningCaughtUp bool `protobuf:"varint,8,opt,name=has_pruning_caught_up,json=hasPruningCaughtUp,proto3" json:"has_pruning_caught_up,omitempty"` + BatchNumberAtUpgrade uint64 `protobuf:"varint,9,opt,name=batch_number_at_upgrade,json=batchNumberAtUpgrade,proto3" json:"batch_number_at_upgrade,omitempty"` } func (m *GenesisState) Reset() { *m = GenesisState{} } @@ -110,6 +113,27 @@ func (m *GenesisState) GetParams() Params { return Params{} } +func (m *GenesisState) GetLegacyDataResults() []GenesisDataResult { + if m != nil { + return m.LegacyDataResults + } + return nil +} + +func (m *GenesisState) GetHasPruningCaughtUp() bool { + if m != nil { + return m.HasPruningCaughtUp + } + return false +} + +func (m *GenesisState) GetBatchNumberAtUpgrade() uint64 { + if m != nil { + return m.BatchNumberAtUpgrade + } + return 0 +} + // BatchAssignment represents a batch assignment for genesis export // and import. type BatchAssignment struct { @@ -306,42 +330,47 @@ func init() { } var fileDescriptor_eccca5d98d3cb479 = []byte{ - // 554 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x94, 0xc1, 0x6e, 0xd3, 0x4c, - 0x10, 0xc7, 0xe3, 0x24, 0x5f, 0x3e, 0xb2, 0x09, 0x0a, 0x5e, 0x8a, 0x64, 0x55, 0x60, 0xdc, 0x80, - 0x2a, 0x23, 0x81, 0x4d, 0xdb, 0x23, 0x5c, 0x88, 0xa8, 0x28, 0x07, 0x10, 0xa4, 0x08, 0x04, 0x42, - 0xb2, 0xd6, 0xf6, 0xca, 0xb6, 0x94, 0xd8, 0x61, 0x77, 0x1d, 0xe8, 0x3b, 0x70, 0xe0, 0x51, 0x78, - 0x8c, 0x72, 0xeb, 0x91, 0x13, 0x42, 0xc9, 0x8b, 0x20, 0x8f, 0x77, 0x9d, 0x04, 0x92, 0xc0, 0x2d, - 0x99, 0xf9, 0xcf, 0x6f, 0xd6, 0xff, 0x99, 0x5d, 0x74, 0x8b, 0xd3, 0x90, 0x04, 0x31, 0x49, 0x52, - 0xd7, 0x27, 0x22, 0x88, 0x93, 0x34, 0x72, 0xa7, 0x07, 0x6e, 0x44, 0x53, 0xca, 0x13, 0xee, 0x4c, - 0x58, 0x26, 0x32, 0x7c, 0xad, 0x12, 0x39, 0x4a, 0xe4, 0x4c, 0x0f, 0x76, 0x77, 0xa2, 0x2c, 0xca, - 0x40, 0xe1, 0x16, 0xbf, 0x4a, 0xf1, 0xee, 0xed, 0xf5, 0xc4, 0xaa, 0x10, 0x54, 0xfd, 0xaf, 0x0d, - 0xd4, 0x7d, 0x52, 0x36, 0x39, 0x15, 0x44, 0x50, 0x7c, 0x1f, 0xed, 0x04, 0x39, 0x63, 0x34, 0x15, - 0x1e, 0x48, 0xbd, 0x34, 0x1f, 0xfb, 0x94, 0x19, 0x9a, 0xa5, 0xd9, 0xcd, 0x21, 0x96, 0xb9, 0x41, - 0x91, 0x7a, 0x0e, 0x19, 0xfc, 0x10, 0xfd, 0x0f, 0x4a, 0xca, 0x8d, 0xba, 0xd5, 0xb0, 0x3b, 0x87, - 0xd7, 0x9d, 0xb5, 0xe7, 0x74, 0xa0, 0x68, 0xd0, 0x3c, 0xff, 0x71, 0xb3, 0x36, 0x54, 0x25, 0xf8, - 0x18, 0xa1, 0xb2, 0x4f, 0x48, 0x04, 0x31, 0x1a, 0x00, 0xb0, 0xb6, 0x01, 0x1e, 0x13, 0x41, 0x24, - 0xa4, 0xed, 0xab, 0x00, 0x7e, 0x89, 0xba, 0x05, 0xc0, 0x63, 0x94, 0xe7, 0x23, 0xc1, 0x8d, 0x26, - 0x80, 0xec, 0x0d, 0x20, 0xf9, 0xc5, 0x45, 0xe5, 0x10, 0x0a, 0x24, 0xb0, 0x13, 0x56, 0x11, 0x8e, - 0xdf, 0x22, 0xbd, 0x3c, 0x19, 0xe1, 0x3c, 0x89, 0xd2, 0x31, 0x4d, 0x05, 0x37, 0xfe, 0x03, 0xee, - 0xfe, 0xb6, 0x03, 0x3e, 0xaa, 0xe4, 0x92, 0x7a, 0xc5, 0x5f, 0x0d, 0x73, 0xfc, 0x00, 0xb5, 0x26, - 0x84, 0x91, 0x31, 0x37, 0x5a, 0x96, 0x66, 0x77, 0x0e, 0x6f, 0x6c, 0xe0, 0xbd, 0x00, 0x91, 0xc4, - 0xc8, 0x92, 0xfe, 0x67, 0x0d, 0xf5, 0x7e, 0x6b, 0x84, 0xf7, 0x50, 0x77, 0xcd, 0xb4, 0x3a, 0xfe, - 0xd2, 0x98, 0xf6, 0x51, 0x4f, 0x3a, 0xf4, 0x21, 0xa7, 0x5c, 0x78, 0x49, 0x68, 0xd4, 0x2d, 0xcd, - 0x6e, 0x0f, 0x2f, 0x97, 0x1f, 0x0d, 0xd1, 0xa7, 0x21, 0x76, 0xd0, 0xd5, 0x15, 0x5d, 0x4c, 0x93, - 0x28, 0x16, 0x46, 0x03, 0x88, 0xfa, 0x92, 0xf6, 0x04, 0x12, 0xfd, 0x6f, 0x75, 0xd4, 0xae, 0x06, - 0xf3, 0x2f, 0x07, 0xf1, 0xab, 0x06, 0x85, 0xcf, 0x1e, 0x4d, 0x05, 0x4b, 0x60, 0x77, 0x0a, 0x27, - 0xee, 0x6e, 0x70, 0x62, 0x31, 0xaa, 0x57, 0x8c, 0xd2, 0xe3, 0xb2, 0x46, 0x1a, 0xa3, 0x2f, 0xa6, - 0x26, 0x13, 0xf8, 0x3d, 0xd2, 0xa7, 0x64, 0x94, 0x84, 0x44, 0x64, 0xac, 0xea, 0x50, 0x2e, 0xd7, - 0x9d, 0x0d, 0x1d, 0x5e, 0x2b, 0xbd, 0x6a, 0x70, 0xa6, 0xc6, 0x57, 0x91, 0x14, 0xfd, 0x0d, 0x2a, - 0x47, 0xea, 0x15, 0xfe, 0x13, 0x91, 0x33, 0xaa, 0x16, 0x6e, 0xeb, 0x62, 0x9c, 0x56, 0x6a, 0x49, - 0xee, 0xf9, 0xab, 0xe1, 0xfe, 0x47, 0xa4, 0xff, 0xb1, 0x9a, 0xd8, 0x50, 0xf7, 0x2b, 0x04, 0x37, - 0x2f, 0xa9, 0xbb, 0x13, 0xe2, 0x13, 0xd4, 0x59, 0x72, 0x52, 0x3a, 0xb8, 0xf7, 0x57, 0x07, 0x65, - 0x77, 0xb4, 0xb0, 0x6d, 0xf0, 0xec, 0x7c, 0x66, 0x6a, 0x17, 0x33, 0x53, 0xfb, 0x39, 0x33, 0xb5, - 0x2f, 0x73, 0xb3, 0x76, 0x31, 0x37, 0x6b, 0xdf, 0xe7, 0x66, 0xed, 0xdd, 0x51, 0x94, 0x88, 0x38, - 0xf7, 0x9d, 0x20, 0x1b, 0xbb, 0x05, 0x18, 0x9e, 0x8d, 0x20, 0x1b, 0xc1, 0x9f, 0x7b, 0xe5, 0xfb, - 0xf2, 0x69, 0xf1, 0xc2, 0x88, 0xb3, 0x09, 0xe5, 0x7e, 0x0b, 0x54, 0x47, 0xbf, 0x02, 0x00, 0x00, - 0xff, 0xff, 0xd3, 0x84, 0x10, 0x33, 0xd6, 0x04, 0x00, 0x00, + // 634 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x54, 0x41, 0x6f, 0xd3, 0x4c, + 0x10, 0x8d, 0x9b, 0x7c, 0x69, 0xb2, 0xc9, 0xa7, 0x90, 0x6d, 0x2a, 0xac, 0x0a, 0x8c, 0x1b, 0x50, + 0x65, 0x24, 0x70, 0x48, 0x2b, 0x4e, 0x70, 0x69, 0x68, 0x45, 0x39, 0x80, 0x4a, 0x4a, 0x41, 0x20, + 0x84, 0xb5, 0xb6, 0x57, 0xb6, 0xa5, 0xc4, 0x36, 0xbb, 0xeb, 0x40, 0xfe, 0x03, 0x07, 0x7e, 0x56, + 0xb9, 0xf5, 0xc8, 0x09, 0xa1, 0xe4, 0x5f, 0x70, 0x42, 0xde, 0x5d, 0x3b, 0x09, 0x24, 0x01, 0x71, + 0xb3, 0x67, 0xde, 0xbc, 0xd9, 0x79, 0xfb, 0x76, 0xc0, 0x4d, 0x8a, 0x5d, 0xe4, 0xf8, 0x28, 0x08, + 0x3b, 0x36, 0x62, 0x8e, 0x1f, 0x84, 0x5e, 0x67, 0xd4, 0xed, 0x78, 0x38, 0xc4, 0x34, 0xa0, 0x66, + 0x4c, 0x22, 0x16, 0xc1, 0xed, 0x1c, 0x64, 0x66, 0x20, 0x73, 0xd4, 0xdd, 0x69, 0x79, 0x91, 0x17, + 0x71, 0x44, 0x27, 0xfd, 0x12, 0xe0, 0x9d, 0x5b, 0xcb, 0x19, 0xf3, 0x42, 0x8e, 0x6a, 0xff, 0x28, + 0x81, 0xfa, 0x63, 0xd1, 0xe4, 0x8c, 0x21, 0x86, 0xe1, 0x3d, 0xd0, 0x72, 0x12, 0x42, 0x70, 0xc8, + 0x2c, 0x0e, 0xb5, 0xc2, 0x64, 0x68, 0x63, 0xa2, 0x2a, 0xba, 0x62, 0x94, 0xfa, 0x50, 0xe6, 0x7a, + 0x69, 0xea, 0x19, 0xcf, 0xc0, 0x87, 0x60, 0x93, 0x23, 0x31, 0x55, 0x37, 0xf4, 0xa2, 0x51, 0xdb, + 0xbf, 0x66, 0x2e, 0x3d, 0xa7, 0xc9, 0x8b, 0x7a, 0xa5, 0x8b, 0x6f, 0x37, 0x0a, 0xfd, 0xac, 0x04, + 0x1e, 0x03, 0x20, 0xfa, 0xb8, 0x88, 0x21, 0xb5, 0xc8, 0x09, 0xf4, 0x75, 0x04, 0x47, 0x88, 0x21, + 0x49, 0x52, 0xb5, 0xb3, 0x00, 0x7c, 0x0e, 0xea, 0x29, 0x81, 0x45, 0x30, 0x4d, 0x06, 0x8c, 0xaa, + 0x25, 0x4e, 0x64, 0xac, 0x20, 0x92, 0x13, 0xa7, 0x95, 0x7d, 0x5e, 0x20, 0x09, 0x6b, 0x6e, 0x1e, + 0xa1, 0xf0, 0x35, 0x68, 0x8a, 0x93, 0x21, 0x4a, 0x03, 0x2f, 0x1c, 0xe2, 0x90, 0x51, 0xf5, 0x3f, + 0xce, 0xbb, 0xb7, 0xee, 0x80, 0x87, 0x39, 0x5c, 0xb2, 0x5e, 0xb1, 0x17, 0xc3, 0x14, 0x3e, 0x00, + 0xe5, 0x18, 0x11, 0x34, 0xa4, 0x6a, 0x59, 0x57, 0x8c, 0xda, 0xfe, 0xf5, 0x15, 0x7c, 0xa7, 0x1c, + 0x24, 0x69, 0x64, 0x09, 0x7c, 0x07, 0xb6, 0x06, 0xd8, 0x43, 0xce, 0xd8, 0x5a, 0x98, 0x78, 0xf3, + 0x9f, 0x26, 0x6e, 0x0a, 0xaa, 0xa3, 0xb9, 0xb9, 0xbb, 0x60, 0xdb, 0x47, 0xd4, 0x8a, 0x49, 0x12, + 0x06, 0xa1, 0x67, 0x39, 0x28, 0xf1, 0x7c, 0x66, 0x25, 0xb1, 0x5a, 0xd1, 0x15, 0xa3, 0xd2, 0x87, + 0x3e, 0xa2, 0xa7, 0x22, 0xf7, 0x88, 0xa7, 0xce, 0x63, 0x78, 0x1f, 0x5c, 0x9d, 0x37, 0x8b, 0x85, + 0x52, 0xbc, 0x47, 0x90, 0x8b, 0xd5, 0x2a, 0xf7, 0x4d, 0xcb, 0x9e, 0x19, 0xe6, 0x90, 0x9d, 0x8b, + 0x5c, 0xfb, 0x93, 0x02, 0x1a, 0xbf, 0x48, 0x06, 0x77, 0x41, 0x7d, 0x89, 0xef, 0x6a, 0x73, 0xf5, + 0x70, 0x0f, 0x34, 0xe4, 0xe4, 0xef, 0x13, 0x4c, 0x99, 0x15, 0xb8, 0xea, 0x86, 0xae, 0x18, 0xd5, + 0xfe, 0xff, 0xe2, 0xfa, 0x78, 0xf4, 0x89, 0x0b, 0x4d, 0xb0, 0xb5, 0x80, 0xf3, 0x71, 0xe0, 0xf9, + 0x4c, 0x2d, 0x72, 0xc6, 0xe6, 0x1c, 0xf6, 0x84, 0x27, 0xda, 0x5f, 0x36, 0x40, 0x35, 0xb7, 0xd8, + 0xdf, 0x1c, 0xc4, 0xce, 0x1b, 0xa4, 0xca, 0x59, 0x38, 0x64, 0x24, 0xe0, 0xaf, 0x20, 0xbd, 0xd3, + 0x3b, 0x2b, 0x6e, 0x62, 0x26, 0xf5, 0x0b, 0x82, 0xf1, 0xb1, 0xa8, 0xc9, 0x6e, 0x63, 0xe6, 0x3f, + 0x99, 0x80, 0x6f, 0x41, 0x73, 0x84, 0x06, 0x81, 0x8b, 0x58, 0x44, 0xf2, 0x0e, 0xe2, 0x99, 0xdc, + 0x5e, 0xd1, 0xe1, 0x65, 0x86, 0xcf, 0x1a, 0x8c, 0x33, 0x23, 0xe6, 0x4c, 0x19, 0xfb, 0x2b, 0x20, + 0xcc, 0x69, 0xa5, 0xfa, 0x23, 0x96, 0x10, 0x9c, 0x3d, 0x9d, 0xb5, 0x16, 0x3f, 0xcb, 0xd1, 0x92, + 0xb9, 0x61, 0x2f, 0x86, 0xdb, 0x1f, 0x40, 0xf3, 0x37, 0xcb, 0x41, 0x35, 0xdb, 0x14, 0x2e, 0x57, + 0xb3, 0x92, 0x6d, 0x01, 0x17, 0x9e, 0x80, 0xda, 0x9c, 0x92, 0x52, 0xc1, 0xdd, 0x3f, 0x2a, 0x28, + 0xbb, 0x83, 0x99, 0x6c, 0xbd, 0xa7, 0x17, 0x13, 0x4d, 0xb9, 0x9c, 0x68, 0xca, 0xf7, 0x89, 0xa6, + 0x7c, 0x9e, 0x6a, 0x85, 0xcb, 0xa9, 0x56, 0xf8, 0x3a, 0xd5, 0x0a, 0x6f, 0x0e, 0xbc, 0x80, 0xf9, + 0x89, 0x6d, 0x3a, 0xd1, 0xb0, 0x93, 0x12, 0xf3, 0x05, 0xe8, 0x44, 0x03, 0xfe, 0x73, 0x57, 0x6c, + 0xca, 0x8f, 0xb3, 0x5d, 0xc9, 0xc6, 0x31, 0xa6, 0x76, 0x99, 0xa3, 0x0e, 0x7e, 0x06, 0x00, 0x00, + 0xff, 0xff, 0x2b, 0x95, 0x5d, 0x3b, 0xa0, 0x05, 0x00, 0x00, } func (m *GenesisState) Marshal() (dAtA []byte, err error) { @@ -364,6 +393,35 @@ func (m *GenesisState) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.BatchNumberAtUpgrade != 0 { + i = encodeVarintGenesis(dAtA, i, uint64(m.BatchNumberAtUpgrade)) + i-- + dAtA[i] = 0x48 + } + if m.HasPruningCaughtUp { + i-- + if m.HasPruningCaughtUp { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x40 + } + if len(m.LegacyDataResults) > 0 { + for iNdEx := len(m.LegacyDataResults) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.LegacyDataResults[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenesis(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x3a + } + } { size, err := m.Params.MarshalToSizedBuffer(dAtA[:i]) if err != nil { @@ -633,6 +691,18 @@ func (m *GenesisState) Size() (n int) { } l = m.Params.Size() n += 1 + l + sovGenesis(uint64(l)) + if len(m.LegacyDataResults) > 0 { + for _, e := range m.LegacyDataResults { + l = e.Size() + n += 1 + l + sovGenesis(uint64(l)) + } + } + if m.HasPruningCaughtUp { + n += 2 + } + if m.BatchNumberAtUpgrade != 0 { + n += 1 + sovGenesis(uint64(m.BatchNumberAtUpgrade)) + } return n } @@ -918,6 +988,79 @@ func (m *GenesisState) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field LegacyDataResults", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenesis + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenesis + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenesis + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.LegacyDataResults = append(m.LegacyDataResults, GenesisDataResult{}) + if err := m.LegacyDataResults[len(m.LegacyDataResults)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 8: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field HasPruningCaughtUp", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenesis + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.HasPruningCaughtUp = bool(v != 0) + case 9: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field BatchNumberAtUpgrade", wireType) + } + m.BatchNumberAtUpgrade = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenesis + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.BatchNumberAtUpgrade |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipGenesis(dAtA[iNdEx:]) diff --git a/x/batching/types/keys.go b/x/batching/types/keys.go index ea464fd6..f97a6146 100644 --- a/x/batching/types/keys.go +++ b/x/batching/types/keys.go @@ -11,7 +11,7 @@ const ( ) var ( - DataResultsPrefix = collections.NewPrefix(0) + LegacyDataResultsPrefix = collections.NewPrefix(0) BatchAssignmentsPrefix = collections.NewPrefix(1) CurrentBatchNumberKey = collections.NewPrefix(2) BatchesKeyPrefix = collections.NewPrefix(3) @@ -20,4 +20,8 @@ var ( DataResultTreeEntriesKeyPrefix = collections.NewPrefix(6) BatchSignaturesKeyPrefix = collections.NewPrefix(7) ParamsKey = collections.NewPrefix(8) + DataResultsPrefix = collections.NewPrefix(9) + BatchDataResultsPrefix = collections.NewPrefix(10) + BatchNumberAtUpgradeKey = collections.NewPrefix(11) + HasPruningCaughtUpKey = collections.NewPrefix(12) ) diff --git a/x/batching/types/params.go b/x/batching/types/params.go index 5e3e4fb2..217ba765 100644 --- a/x/batching/types/params.go +++ b/x/batching/types/params.go @@ -5,17 +5,17 @@ import ( ) const ( - DefaultNumBatchesToKeep = 10000 - DefaultMaxBatchPrunePerBlock = 100 - DefaultMaxDataResultsToCheckForPrune = 100 + DefaultNumBatchesToKeep = 10000 + DefaultMaxBatchPrunePerBlock = 100 + DefaultMaxLegacyDataResultPrunePerBlock = 1000 ) // DefaultParams returns default batching module parameters. func DefaultParams() Params { return Params{ - NumBatchesToKeep: DefaultNumBatchesToKeep, - MaxBatchPrunePerBlock: DefaultMaxBatchPrunePerBlock, - MaxDataResultsToCheckForPrune: DefaultMaxDataResultsToCheckForPrune, + NumBatchesToKeep: DefaultNumBatchesToKeep, + MaxBatchPrunePerBlock: DefaultMaxBatchPrunePerBlock, + MaxLegacyDataResultPrunePerBlock: DefaultMaxLegacyDataResultPrunePerBlock, } } diff --git a/x/tally/keeper/endblock_test.go b/x/tally/keeper/endblock_test.go index 7060fd53..ebade10a 100644 --- a/x/tally/keeper/endblock_test.go +++ b/x/tally/keeper/endblock_test.go @@ -135,7 +135,7 @@ func TestEndBlock(t *testing.T) { dataResults, err := f.batchingKeeper.GetDataResults(f.Context(), false) require.NoError(t, err) - require.Contains(t, dataResults, *dataResult) + require.Contains(t, dataResults, dataResult) }) } } @@ -180,7 +180,7 @@ func TestEndBlock_UpdateMaxResultSize(t *testing.T) { dataResults, err := f.batchingKeeper.GetDataResults(f.Context(), false) require.NoError(t, err) - require.Contains(t, dataResults, *dataResult) + require.Contains(t, dataResults, dataResult) // Ensure the new DR gets a unique ID f.AddBlock() @@ -214,7 +214,7 @@ func TestEndBlock_UpdateMaxResultSize(t *testing.T) { dataResultsAfter, err := f.batchingKeeper.GetDataResults(f.Context(), false) require.NoError(t, err) - require.Contains(t, dataResultsAfter, *dataResultAfter) + require.Contains(t, dataResultsAfter, dataResultAfter) } func TestEndBlock_ChunkedContractQuery(t *testing.T) { @@ -341,7 +341,7 @@ func TestEndBlock_PausedContract(t *testing.T) { f.pauseContract(t) - var noRevealsResult *batchingtypes.DataResult + var noRevealsResult batchingtypes.DataResult // Ensure the DR without commitments and the DR without reveals are timed out for i := range defaultCommitTimeoutBlocks { @@ -416,6 +416,6 @@ func TestTallyTestItems(t *testing.T) { require.Equal(t, testItems[i].ExpectedExitCode, dataResult.ExitCode) require.Equal(t, testItems[i].ExpectedGasUsed.String(), dataResult.GasUsed.String()) - require.Contains(t, dataResults, *dataResult) + require.Contains(t, dataResults, dataResult) } } From c4cbf350dd5c486d77dd509c69ce93b11dc91493 Mon Sep 17 00:00:00 2001 From: hacheigriega Date: Wed, 7 Jan 2026 15:58:48 -0500 Subject: [PATCH 3/8] refactor(x/batching): simplify batch pruning Simplify batch pruning logic by fixing numBatchesToKeep. --- proto/sedachain/batching/v1/batching.proto | 7 +- testutil/contract_msgs.go | 2 +- x/batching/keeper/endblock.go | 19 +-- x/batching/keeper/keeper.go | 2 +- x/batching/keeper/pruning.go | 37 +++-- x/batching/keeper/pruning_test.go | 53 ++++--- x/batching/types/batching.pb.go | 169 ++++++++------------- x/batching/types/data_result.go | 2 +- x/batching/types/genesis_test.go | 1 - x/batching/types/params.go | 9 -- x/tally/keeper/gas_meter.go | 2 + 11 files changed, 142 insertions(+), 161 deletions(-) diff --git a/proto/sedachain/batching/v1/batching.proto b/proto/sedachain/batching/v1/batching.proto index 3b413bf7..a9f82718 100644 --- a/proto/sedachain/batching/v1/batching.proto +++ b/proto/sedachain/batching/v1/batching.proto @@ -102,13 +102,10 @@ message DataRequestIDHeight { // Params defines the parameters for the batching module. message Params { - // NumBatchesToKeep is the number of batches to keep in the state without - // pruning. - uint64 num_batches_to_keep = 1; // MaxBatchPrunePerBlock is the maximum number of batches to prune per // block. - uint64 max_batch_prune_per_block = 2; + uint64 max_batch_prune_per_block = 1; // MaxLegacyDataResultPrunePerBlock is the maximum number of legacy data // results to be checked for pruning per block. - uint64 max_legacy_data_result_prune_per_block = 3; + uint64 max_legacy_data_result_prune_per_block = 2; } diff --git a/testutil/contract_msgs.go b/testutil/contract_msgs.go index 381cf922..65bdb9d5 100644 --- a/testutil/contract_msgs.go +++ b/testutil/contract_msgs.go @@ -19,7 +19,7 @@ func CommitMsg(drID, commitment, stakerPubKey, proof string, gasUsed uint64) []b } func RevealMsg(drID, reveal, stakerPubKey, proof string, proxyPubKeys []string, exitCode byte, drHeight, gasUsed uint64) []byte { - quotedObjects := []string{} + quotedObjects := make([]string, 0, len(proxyPubKeys)) for _, obj := range proxyPubKeys { quotedObjects = append(quotedObjects, fmt.Sprintf("%q", obj)) } diff --git a/x/batching/keeper/endblock.go b/x/batching/keeper/endblock.go index ca2ba491..598c72c8 100644 --- a/x/batching/keeper/endblock.go +++ b/x/batching/keeper/endblock.go @@ -18,6 +18,10 @@ import ( "github.com/sedaprotocol/seda-chain/x/batching/types" ) +// NumBatchesToKeep is the number of batches to keep in the state without pruning. +// This value must be at least 4 to avoid interruption of batch signing logic. +var NumBatchesToKeep uint64 = 10000 + func (k Keeper) EndBlock(ctx sdk.Context) error { params, err := k.GetParams(ctx) if err != nil { @@ -48,16 +52,9 @@ func (k Keeper) EndBlock(ctx sdk.Context) error { return err } - // Try pruning a batch. - // If there has been an upgrade (batchNumAtUpgrade is not 0), - // then prune only if the batch was created after the upgrade. - batchNumToPrune := newBatchNum - params.NumBatchesToKeep - if newBatchNum >= params.NumBatchesToKeep && - (batchNumAtUpgrade == 0 || batchNumToPrune > batchNumAtUpgrade) { - err = k.TryPruneBatch(ctx, batchNumToPrune) - if err != nil { - return err - } + err = k.BasicPruneBatch(ctx, newBatchNum, NumBatchesToKeep, batchNumAtUpgrade) + if err != nil { + return err } } } else { @@ -74,7 +71,7 @@ func (k Keeper) EndBlock(ctx sdk.Context) error { // have been pruned. // Note this operation does not prune data results, which will be pruned // separately in the else clause. - lastPrunedBatchNum, err := k.BatchPruneBatches(ctx, params.NumBatchesToKeep, params.MaxBatchPrunePerBlock, batchNumAtUpgrade) + lastPrunedBatchNum, err := k.BatchPruneBatches(ctx, NumBatchesToKeep, params.MaxBatchPrunePerBlock, batchNumAtUpgrade) if err != nil { telemetry.SetGauge(1, types.TelemetryKeyBatchingPruningFail) k.Logger(ctx).Error("error while pruning batches", "err", err) diff --git a/x/batching/keeper/keeper.go b/x/batching/keeper/keeper.go index 28b8052f..b2d0f9d1 100644 --- a/x/batching/keeper/keeper.go +++ b/x/batching/keeper/keeper.go @@ -43,7 +43,7 @@ type Keeper struct { // have corresponding items in batchDataResults. dataResults collections.Map[collections.Triple[bool, string, uint64], types.DataResult] // batchDataResults maps batch number to a list of data request ID - posted height - // pairs to support simple pruning of data results. + // pairs to support basic pruning of data results. batchDataResults collections.Map[uint64, types.DataRequestIDHeights] // legacyDataResults is the older version of dataResults. The items in this // collection do not have corresponding items in batchDataResults. diff --git a/x/batching/keeper/pruning.go b/x/batching/keeper/pruning.go index 95a5f7fd..ae33b3d2 100644 --- a/x/batching/keeper/pruning.go +++ b/x/batching/keeper/pruning.go @@ -31,9 +31,24 @@ func (k Keeper) HasPruningCaughtUp(ctx sdk.Context) (bool, error) { return k.hasPruningCaughtUp.Get(ctx) } -// TryPruneBatch attempts to prune the given batch and all of its associated data. -func (k Keeper) TryPruneBatch(ctx sdk.Context, batchNum uint64) error { - batch, err := k.GetBatchByBatchNumber(ctx, batchNum) +// BasicPruneBatch prunes a batch at newBatchNum - numBatchesToKeep and all of its +// associated data. It returns without error if there is not enough batches or if +// the batch was created before the upgrade. +func (k Keeper) BasicPruneBatch(ctx sdk.Context, newBatchNum, numBatchesToKeep, batchNumAtUpgrade uint64) error { + // Do not prune until we have sufficient number of batches. + if newBatchNum < numBatchesToKeep { + return nil + } + + batchNumToPrune := newBatchNum - numBatchesToKeep + + // If there has been an upgrade (i.e., batchNumAtUpgrade is not 0), + // then prune only if the batch was created after the upgrade. + if batchNumAtUpgrade != 0 && batchNumToPrune <= batchNumAtUpgrade { + return nil + } + + batch, err := k.GetBatchByBatchNumber(ctx, batchNumToPrune) if err != nil { return err } @@ -43,12 +58,12 @@ func (k Keeper) TryPruneBatch(ctx sdk.Context, batchNum uint64) error { if err != nil { return err } - err = k.dataResultTreeEntries.Remove(ctx, batchNum) + err = k.dataResultTreeEntries.Remove(ctx, batchNumToPrune) if err != nil { return err } - valRng := new(collections.Range[collections.Pair[uint64, []byte]]).Prefix(collections.PairPrefix[uint64, []byte](batchNum)) + valRng := new(collections.Range[collections.Pair[uint64, []byte]]).Prefix(collections.PairPrefix[uint64, []byte](batchNumToPrune)) err = k.validatorTreeEntries.Clear(ctx, valRng) if err != nil { return err @@ -58,10 +73,10 @@ func (k Keeper) TryPruneBatch(ctx sdk.Context, batchNum uint64) error { return err } - dataResults, err := k.GetBatchDataResults(ctx, batchNum) + dataResults, err := k.GetBatchDataResults(ctx, batchNumToPrune) if err != nil { if errors.Is(err, collections.ErrNotFound) { - k.Logger(ctx).Info("cannot prune batch because schema change has not been applied", "batch_num", batchNum) + k.Logger(ctx).Info("cannot prune batch because schema change has not been applied", "batch_num", batchNumToPrune) return nil } return err @@ -76,12 +91,12 @@ func (k Keeper) TryPruneBatch(ctx sdk.Context, batchNum uint64) error { return err } } - err = k.RemoveBatchDataResults(ctx, batchNum) + err = k.RemoveBatchDataResults(ctx, batchNumToPrune) if err != nil { return err } - k.Logger(ctx).Info("single pruned batch", "batch_num", batchNum) + k.Logger(ctx).Info("pruned a batch (basic strategy)", "batch_num", batchNumToPrune) return nil } @@ -136,7 +151,7 @@ func (k Keeper) BatchPruneBatches(ctx sdk.Context, numBatchesToKeep, maxBatchPru if err != nil { return 0, err } - k.Logger(ctx).Info("pruned batch", "batch_num", batchNum) + k.Logger(ctx).Info("pruned a batch (batch pruning strategy)", "batch_num", batchNum) lastPrunedBatchNum = batchNum @@ -147,7 +162,7 @@ func (k Keeper) BatchPruneBatches(ctx sdk.Context, numBatchesToKeep, maxBatchPru } if firstKey == nil { - k.Logger(ctx).Info("no batches to prune") + k.Logger(ctx).Info("no batches to prune (batch pruning strategy)") // This means all batches up to batch number rngEnd - 1 have been pruned. // Note we subtract 1 because rngEnd is exclusive. return rngEnd - 1, nil diff --git a/x/batching/keeper/pruning_test.go b/x/batching/keeper/pruning_test.go index a237e7a7..cf14da26 100644 --- a/x/batching/keeper/pruning_test.go +++ b/x/batching/keeper/pruning_test.go @@ -8,6 +8,7 @@ import ( "cosmossdk.io/collections" sedatypes "github.com/sedaprotocol/seda-chain/types" + "github.com/sedaprotocol/seda-chain/x/batching/keeper" "github.com/sedaprotocol/seda-chain/x/batching/types" pubkeytypes "github.com/sedaprotocol/seda-chain/x/pubkey/types" ) @@ -114,10 +115,14 @@ func TestLegacyDataResultPruning(t *testing.T) { err = f.batchingKeeper.SetHasPruningCaughtUp(f.Context(), false) require.NoError(t, err) - err = f.batchingKeeper.SetParams(f.Context(), types.Params{ - MaxLegacyDataResultPrunePerBlock: 101, - NumBatchesToKeep: 10, - }) + // Adjust the global variable for the test. + original := keeper.NumBatchesToKeep + defer func() { + keeper.NumBatchesToKeep = original + }() + keeper.NumBatchesToKeep = 10 + + err = f.batchingKeeper.SetParams(f.Context(), types.Params{MaxLegacyDataResultPrunePerBlock: 101}) require.NoError(t, err) // Create 10 data results for each of 100 batches @@ -169,8 +174,8 @@ func TestLegacyDataResultPruning(t *testing.T) { } } -// TestSimplePruning tests simple pruning with batch pruning disabled. -func TestSimplePruning(t *testing.T) { +// TestBasicPruning tests basic pruning with batch pruning disabled. +func TestBasicPruning(t *testing.T) { f := initFixture(t) f.addBatchSigningValidators(t, 10) @@ -181,10 +186,14 @@ func TestSimplePruning(t *testing.T) { }) require.NoError(t, err) - params := types.Params{ - NumBatchesToKeep: 15, - MaxBatchPrunePerBlock: 0, // disable batch pruning - } + // Adjust the global variable for the test. + original := keeper.NumBatchesToKeep + defer func() { + keeper.NumBatchesToKeep = original + }() + keeper.NumBatchesToKeep = 15 + + params := types.Params{MaxBatchPrunePerBlock: 0} // Disable batch pruning err = f.batchingKeeper.SetParams(f.Context(), params) require.NoError(t, err) @@ -235,7 +244,7 @@ func TestSimplePruning(t *testing.T) { } } -func TestNoSimplePruningUntilNumBatchesToKeepIsReached(t *testing.T) { +func TestNoBasicPruningUntilNumBatchesToKeepIsReached(t *testing.T) { f := initFixture(t) f.addBatchSigningValidators(t, 10) @@ -246,10 +255,14 @@ func TestNoSimplePruningUntilNumBatchesToKeepIsReached(t *testing.T) { }) require.NoError(t, err) - err = f.batchingKeeper.SetParams(f.Context(), types.Params{ - NumBatchesToKeep: 11, - MaxBatchPrunePerBlock: 0, - }) + // Adjust the global variable for the test. + original := keeper.NumBatchesToKeep + defer func() { + keeper.NumBatchesToKeep = original + }() + keeper.NumBatchesToKeep = 11 + + err = f.batchingKeeper.SetParams(f.Context(), types.Params{MaxBatchPrunePerBlock: 0}) require.NoError(t, err) // Create 10 batches with random associated data. @@ -311,8 +324,14 @@ func TestPruningMockedUpgrade(t *testing.T) { }) require.NoError(t, err) + // Adjust the global variable for the test. + original := keeper.NumBatchesToKeep + defer func() { + keeper.NumBatchesToKeep = original + }() + keeper.NumBatchesToKeep = 10 + err = f.batchingKeeper.SetParams(f.Context(), types.Params{ - NumBatchesToKeep: 10, MaxBatchPrunePerBlock: 15, MaxLegacyDataResultPrunePerBlock: 80, }) @@ -390,7 +409,7 @@ func TestPruningMockedUpgrade(t *testing.T) { } // Block 40 - 43: - // - Batch creation at every block but number of batches stays at 10 with simple pruning. + // - Batch creation at every block but number of batches stays at 10 with basic pruning. // - HasPruningCaughtUp is now True and legacy data results pruning is in effect. for i := range 4 { f.BatchingEndBlock(t, 10) diff --git a/x/batching/types/batching.pb.go b/x/batching/types/batching.pb.go index c8f80d4d..cfdfc375 100644 --- a/x/batching/types/batching.pb.go +++ b/x/batching/types/batching.pb.go @@ -535,15 +535,12 @@ func (m *DataRequestIDHeight) GetDataRequestHeight() uint64 { // Params defines the parameters for the batching module. type Params struct { - // NumBatchesToKeep is the number of batches to keep in the state without - // pruning. - NumBatchesToKeep uint64 `protobuf:"varint,1,opt,name=num_batches_to_keep,json=numBatchesToKeep,proto3" json:"num_batches_to_keep,omitempty"` // MaxBatchPrunePerBlock is the maximum number of batches to prune per // block. - MaxBatchPrunePerBlock uint64 `protobuf:"varint,2,opt,name=max_batch_prune_per_block,json=maxBatchPrunePerBlock,proto3" json:"max_batch_prune_per_block,omitempty"` + MaxBatchPrunePerBlock uint64 `protobuf:"varint,1,opt,name=max_batch_prune_per_block,json=maxBatchPrunePerBlock,proto3" json:"max_batch_prune_per_block,omitempty"` // MaxLegacyDataResultPrunePerBlock is the maximum number of legacy data // results to be checked for pruning per block. - MaxLegacyDataResultPrunePerBlock uint64 `protobuf:"varint,3,opt,name=max_legacy_data_result_prune_per_block,json=maxLegacyDataResultPrunePerBlock,proto3" json:"max_legacy_data_result_prune_per_block,omitempty"` + MaxLegacyDataResultPrunePerBlock uint64 `protobuf:"varint,2,opt,name=max_legacy_data_result_prune_per_block,json=maxLegacyDataResultPrunePerBlock,proto3" json:"max_legacy_data_result_prune_per_block,omitempty"` } func (m *Params) Reset() { *m = Params{} } @@ -579,13 +576,6 @@ func (m *Params) XXX_DiscardUnknown() { var xxx_messageInfo_Params proto.InternalMessageInfo -func (m *Params) GetNumBatchesToKeep() uint64 { - if m != nil { - return m.NumBatchesToKeep - } - return 0 -} - func (m *Params) GetMaxBatchPrunePerBlock() uint64 { if m != nil { return m.MaxBatchPrunePerBlock @@ -616,69 +606,67 @@ func init() { } var fileDescriptor_5b2a028024867de2 = []byte{ - // 980 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x55, 0xcb, 0x6e, 0x23, 0x45, - 0x17, 0x4e, 0x3b, 0xce, 0xc5, 0xc7, 0x76, 0x92, 0xa9, 0x24, 0xff, 0xdf, 0x99, 0x85, 0xdb, 0x58, - 0xcc, 0xc8, 0x0c, 0x8a, 0x4d, 0x26, 0xe2, 0x22, 0xc1, 0x86, 0x66, 0x90, 0x88, 0x60, 0x46, 0x56, - 0x11, 0x66, 0xc1, 0x82, 0x56, 0xa5, 0xab, 0xd4, 0x6e, 0xd9, 0xdd, 0xd5, 0x54, 0x55, 0x1b, 0x7b, - 0xc5, 0x2b, 0xf0, 0x02, 0x6c, 0x78, 0x06, 0xde, 0x80, 0x05, 0xb3, 0x1c, 0xb1, 0x42, 0x20, 0xb5, - 0x50, 0xb2, 0xf3, 0x23, 0xb0, 0x42, 0x5d, 0xd5, 0x6e, 0x5f, 0xc8, 0x96, 0x55, 0x57, 0x7d, 0xdf, - 0x77, 0x4e, 0x9d, 0xea, 0x73, 0x29, 0x78, 0x53, 0x32, 0x4a, 0xfc, 0x21, 0x09, 0xe3, 0xfe, 0x0d, - 0x51, 0xfe, 0x30, 0x8c, 0x83, 0xfe, 0xe4, 0xa2, 0x5c, 0xf7, 0x12, 0xc1, 0x15, 0x47, 0xa7, 0xa5, - 0xaa, 0x57, 0x32, 0x93, 0x8b, 0x87, 0x67, 0x3e, 0x97, 0x11, 0x97, 0x9e, 0x16, 0xf5, 0xcd, 0xc6, - 0x58, 0x3c, 0x3c, 0x09, 0x78, 0xc0, 0x0d, 0x9e, 0xaf, 0x0c, 0xda, 0xf9, 0xb1, 0x02, 0x3b, 0x6e, - 0xee, 0x00, 0xbd, 0x01, 0x0d, 0xed, 0xc9, 0x8b, 0xd3, 0xe8, 0x86, 0x09, 0xdb, 0x6a, 0x5b, 0xdd, - 0x2a, 0xae, 0x6b, 0xec, 0x85, 0x86, 0xb4, 0x64, 0xcc, 0xfd, 0x91, 0x37, 0x64, 0x61, 0x30, 0x54, - 0x76, 0xa5, 0x6d, 0x75, 0xb7, 0x71, 0x5d, 0x63, 0x9f, 0x69, 0x08, 0xbd, 0x0f, 0xb6, 0x9f, 0x0a, - 0xc1, 0x62, 0xe5, 0x51, 0xa2, 0x88, 0x27, 0x98, 0x4c, 0xc7, 0xca, 0x13, 0x9c, 0x2b, 0x7b, 0xbb, - 0x6d, 0x75, 0x6b, 0xf8, 0xb4, 0xe0, 0x9f, 0x11, 0x45, 0xb0, 0x66, 0x31, 0xe7, 0x0a, 0x75, 0xe1, - 0xe8, 0x5f, 0x06, 0x55, 0x6d, 0x70, 0x40, 0xd7, 0x95, 0x8f, 0xe0, 0x60, 0x42, 0xc6, 0x21, 0x25, - 0x8a, 0x0b, 0xa3, 0xdb, 0xd1, 0xba, 0x66, 0x89, 0x6a, 0xd9, 0x19, 0xec, 0x9b, 0xfb, 0x84, 0xd4, - 0xde, 0x6d, 0x5b, 0xdd, 0x06, 0xde, 0xd3, 0xfb, 0x2b, 0x8a, 0xde, 0x82, 0xa3, 0x44, 0xf0, 0x49, - 0x18, 0x07, 0x5e, 0xc4, 0x14, 0xc9, 0xfd, 0xdb, 0x7b, 0x5a, 0x72, 0x58, 0xe0, 0xcf, 0x0b, 0xb8, - 0x73, 0x01, 0xa7, 0xcb, 0x40, 0xaf, 0x05, 0x63, 0x9f, 0xc6, 0x4a, 0x84, 0x4c, 0x22, 0x1b, 0xf6, - 0x98, 0x59, 0xda, 0x56, 0x7b, 0x3b, 0xf7, 0x5e, 0x6c, 0x3b, 0xbf, 0x5a, 0x80, 0x5e, 0x2e, 0x42, - 0x59, 0x98, 0xcc, 0xd0, 0x37, 0xf0, 0x60, 0x19, 0x36, 0xa1, 0x54, 0x30, 0x29, 0xf5, 0x4f, 0x6e, - 0xb8, 0x17, 0x7f, 0x67, 0xce, 0x79, 0x10, 0xaa, 0x61, 0x7a, 0xd3, 0xf3, 0x79, 0x54, 0xe4, 0xad, - 0xf8, 0x9c, 0x4b, 0x3a, 0xea, 0xab, 0x59, 0xc2, 0x64, 0xef, 0x25, 0x19, 0x7f, 0x6c, 0x0c, 0xf1, - 0x51, 0xe9, 0xab, 0x40, 0xd0, 0x3b, 0x70, 0x32, 0xe1, 0x2a, 0xbf, 0x53, 0xc2, 0xbf, 0x63, 0xc2, - 0x4b, 0x98, 0xf0, 0x59, 0x6c, 0x92, 0xd4, 0xc4, 0xc8, 0x70, 0x83, 0x9c, 0x1a, 0x18, 0x06, 0x39, - 0x50, 0x67, 0x6a, 0x58, 0xc6, 0xb2, 0xad, 0xff, 0x00, 0x30, 0x35, 0x2c, 0x5c, 0x76, 0x7e, 0xb2, - 0xe0, 0x50, 0x17, 0xc7, 0x97, 0x61, 0x10, 0x13, 0x95, 0x0a, 0x26, 0xff, 0xf3, 0x6b, 0xf4, 0xe1, - 0x58, 0x32, 0x3f, 0x79, 0xfa, 0xee, 0x7b, 0xa3, 0x0b, 0x4f, 0x2e, 0xce, 0xd5, 0xb7, 0x68, 0x60, - 0x54, 0x52, 0x65, 0x44, 0x9d, 0x3f, 0xab, 0x00, 0xcb, 0x14, 0xa1, 0xff, 0x41, 0x25, 0xa4, 0x3a, - 0xa0, 0x9a, 0xbb, 0x3b, 0xcf, 0x9c, 0x4a, 0x48, 0x71, 0x25, 0xa4, 0xa8, 0x05, 0x3b, 0x54, 0xe4, - 0xb5, 0x50, 0xd1, 0x54, 0x6d, 0x9e, 0x39, 0x06, 0xc0, 0x55, 0x2a, 0xae, 0x28, 0xfa, 0x10, 0x0e, - 0xa9, 0xf0, 0xd6, 0xca, 0x3b, 0xff, 0x21, 0x55, 0xf7, 0x78, 0x9e, 0x39, 0x9b, 0x14, 0x6e, 0x52, - 0xe1, 0xae, 0x54, 0xfd, 0x23, 0xd8, 0x9b, 0x30, 0x21, 0x43, 0x1e, 0x9b, 0x9a, 0x75, 0xeb, 0xf3, - 0xcc, 0x59, 0x40, 0x78, 0xb1, 0x40, 0x97, 0x1b, 0xfd, 0xb3, 0xa3, 0x0f, 0x38, 0x9a, 0x67, 0xce, - 0x1a, 0xbe, 0xde, 0x51, 0x1f, 0xc1, 0xa1, 0x21, 0x55, 0x18, 0x31, 0xa9, 0x48, 0x94, 0xe8, 0x72, - 0x2e, 0x02, 0xdb, 0xa0, 0xf0, 0x81, 0x06, 0xae, 0x17, 0x7b, 0xf4, 0x04, 0x6a, 0x6c, 0x1a, 0x2a, - 0xcf, 0xe7, 0x94, 0xe9, 0x1a, 0x6f, 0xba, 0xcd, 0x79, 0xe6, 0x2c, 0x41, 0xbc, 0x9f, 0x2f, 0x3f, - 0xe1, 0x94, 0xa1, 0x17, 0xb0, 0x1f, 0x10, 0xe9, 0xa5, 0x92, 0x51, 0x7b, 0x5f, 0x5f, 0xe3, 0xf2, - 0x8f, 0xcc, 0x39, 0x35, 0xf9, 0x93, 0x74, 0xd4, 0x0b, 0x79, 0x3f, 0x22, 0x6a, 0xd8, 0xbb, 0x8a, - 0xd5, 0x3c, 0x73, 0x4a, 0xf1, 0x6f, 0x3f, 0x9f, 0x43, 0x31, 0x6a, 0xae, 0x62, 0x85, 0xf7, 0x02, - 0x22, 0xbf, 0x92, 0x8c, 0xa2, 0x0e, 0xec, 0x9a, 0x6e, 0xb6, 0x6b, 0xba, 0x3e, 0x60, 0x9e, 0x39, - 0x05, 0x82, 0x8b, 0x6f, 0x7e, 0xbb, 0x84, 0xcc, 0x6e, 0x88, 0x3f, 0x2a, 0x8b, 0x09, 0xf4, 0xd1, - 0xfa, 0x76, 0x1b, 0x14, 0x3e, 0x28, 0x80, 0x45, 0xb1, 0x5c, 0x42, 0x23, 0x9f, 0x83, 0x5e, 0x42, - 0x66, 0x63, 0x4e, 0xa8, 0x5d, 0xd7, 0xa6, 0xfa, 0x87, 0xae, 0xe2, 0xb8, 0x9e, 0xef, 0x06, 0x66, - 0x83, 0xde, 0x86, 0x9a, 0xcf, 0x63, 0xc9, 0x62, 0x99, 0x4a, 0xbb, 0xd1, 0xb6, 0xba, 0xfb, 0xe6, - 0x97, 0x94, 0x20, 0x5e, 0x2e, 0x3b, 0xdf, 0xc3, 0x89, 0x29, 0xae, 0x6f, 0x53, 0x26, 0xd5, 0xd5, - 0x33, 0x93, 0x14, 0x89, 0x02, 0xf8, 0x7f, 0x31, 0xae, 0x34, 0xe1, 0x85, 0xb4, 0x48, 0x9e, 0x19, - 0x07, 0xf5, 0xa7, 0x4f, 0x7a, 0xf7, 0x4e, 0xe8, 0xde, 0x3d, 0xde, 0xdc, 0xea, 0xab, 0xcc, 0xd9, - 0xc2, 0x27, 0x74, 0x85, 0xa2, 0xc5, 0x41, 0x9d, 0x08, 0x8e, 0xef, 0x31, 0x41, 0x8f, 0xe1, 0x70, - 0xe3, 0x7c, 0x53, 0xf3, 0xb8, 0xb9, 0xe6, 0x05, 0xf5, 0xe0, 0x78, 0x4d, 0xb7, 0x32, 0xb9, 0xab, - 0xf8, 0xc1, 0x8a, 0xd6, 0xf8, 0xed, 0xfc, 0x62, 0xc1, 0xee, 0x80, 0x08, 0x12, 0x49, 0x74, 0x0e, - 0xc7, 0x71, 0x1a, 0x79, 0x3a, 0x78, 0x26, 0x3d, 0xc5, 0xbd, 0x11, 0x63, 0x49, 0xf1, 0x2e, 0x1c, - 0xc5, 0x69, 0xe4, 0x1a, 0xe6, 0x9a, 0x7f, 0xce, 0x58, 0x82, 0x3e, 0x80, 0xb3, 0x88, 0x4c, 0x8d, - 0xdc, 0x4b, 0x44, 0x1a, 0xb3, 0x7c, 0x04, 0x99, 0xb6, 0x29, 0xce, 0x3b, 0x8d, 0xc8, 0x54, 0x1b, - 0x0d, 0x72, 0x7a, 0xc0, 0x4c, 0x0f, 0xa1, 0x01, 0x3c, 0xce, 0x2d, 0xc7, 0x2c, 0x20, 0xfe, 0x6c, - 0xed, 0xd9, 0xd8, 0x74, 0xa3, 0x3b, 0x12, 0xb7, 0x23, 0x32, 0xfd, 0x42, 0x8b, 0x97, 0x7d, 0xbf, - 0xe6, 0xd1, 0x7d, 0xfe, 0xea, 0xb6, 0x65, 0xbd, 0xbe, 0x6d, 0x59, 0x7f, 0xdd, 0xb6, 0xac, 0x1f, - 0xee, 0x5a, 0x5b, 0xaf, 0xef, 0x5a, 0x5b, 0xbf, 0xdf, 0xb5, 0xb6, 0xbe, 0xbe, 0x5c, 0x99, 0x4f, - 0x79, 0x82, 0xf4, 0x2b, 0xe8, 0xf3, 0xb1, 0xde, 0x9c, 0x9b, 0x67, 0x77, 0xba, 0x7c, 0x78, 0xf5, - 0xc0, 0xba, 0xd9, 0xd5, 0xaa, 0xcb, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x11, 0xe8, 0xaf, 0xfb, - 0x9b, 0x07, 0x00, 0x00, + // 951 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x55, 0xcd, 0x8e, 0xe3, 0x44, + 0x10, 0x1e, 0x67, 0x32, 0x3f, 0xa9, 0x24, 0x33, 0xb3, 0x3d, 0x13, 0xf0, 0xee, 0x21, 0x0e, 0x11, + 0xbb, 0x0a, 0x8b, 0x26, 0x21, 0x1b, 0xf1, 0x23, 0xc1, 0x85, 0xb0, 0x48, 0x44, 0x62, 0x57, 0x51, + 0xb3, 0xec, 0x81, 0x03, 0x56, 0xc7, 0xdd, 0x72, 0xac, 0xc4, 0x6e, 0xd3, 0xdd, 0x09, 0xc9, 0x89, + 0x57, 0xe0, 0xc0, 0x95, 0x0b, 0xcf, 0xc0, 0x3b, 0xb0, 0xc7, 0x15, 0x27, 0x04, 0x92, 0x85, 0x66, + 0x6e, 0x79, 0x04, 0x4e, 0xc8, 0xdd, 0x8e, 0xf3, 0xc3, 0x5c, 0x39, 0xb9, 0xeb, 0xfb, 0xbe, 0x2a, + 0x57, 0x75, 0x57, 0x57, 0xc3, 0xdb, 0x92, 0x51, 0xe2, 0x8d, 0x49, 0x10, 0x75, 0x46, 0x44, 0x79, + 0xe3, 0x20, 0xf2, 0x3b, 0xf3, 0x6e, 0xbe, 0x6e, 0xc7, 0x82, 0x2b, 0x8e, 0x6a, 0xb9, 0xaa, 0x9d, + 0x33, 0xf3, 0xee, 0x83, 0xfb, 0x1e, 0x97, 0x21, 0x97, 0xae, 0x16, 0x75, 0x8c, 0x61, 0x3c, 0x1e, + 0x5c, 0xf9, 0xdc, 0xe7, 0x06, 0x4f, 0x57, 0x06, 0x6d, 0xfe, 0x5c, 0x80, 0xa3, 0x7e, 0x1a, 0x00, + 0xbd, 0x05, 0x15, 0x1d, 0xc9, 0x8d, 0x66, 0xe1, 0x88, 0x09, 0xdb, 0x6a, 0x58, 0xad, 0x22, 0x2e, + 0x6b, 0xec, 0xb9, 0x86, 0xb4, 0x64, 0xca, 0xbd, 0x89, 0x3b, 0x66, 0x81, 0x3f, 0x56, 0x76, 0xa1, + 0x61, 0xb5, 0x0e, 0x71, 0x59, 0x63, 0x5f, 0x68, 0x08, 0x7d, 0x08, 0xb6, 0x37, 0x13, 0x82, 0x45, + 0xca, 0xa5, 0x44, 0x11, 0x57, 0x30, 0x39, 0x9b, 0x2a, 0x57, 0x70, 0xae, 0xec, 0xc3, 0x86, 0xd5, + 0x2a, 0xe1, 0x5a, 0xc6, 0x3f, 0x25, 0x8a, 0x60, 0xcd, 0x62, 0xce, 0x15, 0x6a, 0xc1, 0xc5, 0x7f, + 0x1c, 0x8a, 0xda, 0xe1, 0x8c, 0xee, 0x2a, 0x1f, 0xc2, 0xd9, 0x9c, 0x4c, 0x03, 0x4a, 0x14, 0x17, + 0x46, 0x77, 0xa4, 0x75, 0xd5, 0x1c, 0xd5, 0xb2, 0xfb, 0x70, 0x6a, 0xea, 0x09, 0xa8, 0x7d, 0xdc, + 0xb0, 0x5a, 0x15, 0x7c, 0xa2, 0xed, 0x01, 0x45, 0xef, 0xc0, 0x45, 0x2c, 0xf8, 0x3c, 0x88, 0x7c, + 0x37, 0x64, 0x8a, 0xa4, 0xf1, 0xed, 0x13, 0x2d, 0x39, 0xcf, 0xf0, 0x67, 0x19, 0xdc, 0xec, 0x42, + 0x6d, 0x93, 0xe8, 0x0b, 0xc1, 0xd8, 0xe7, 0x91, 0x12, 0x01, 0x93, 0xc8, 0x86, 0x13, 0x66, 0x96, + 0xb6, 0xd5, 0x38, 0x4c, 0xa3, 0x67, 0x66, 0xf3, 0x37, 0x0b, 0xd0, 0xcb, 0x75, 0x2a, 0x6b, 0x97, + 0x25, 0xfa, 0x16, 0xee, 0x6d, 0xd2, 0x26, 0x94, 0x0a, 0x26, 0xa5, 0xde, 0xe4, 0x4a, 0xbf, 0xfb, + 0x4f, 0xe2, 0x5c, 0xfb, 0x81, 0x1a, 0xcf, 0x46, 0x6d, 0x8f, 0x87, 0xd9, 0xb9, 0x65, 0x9f, 0x6b, + 0x49, 0x27, 0x1d, 0xb5, 0x8c, 0x99, 0x6c, 0xbf, 0x24, 0xd3, 0x4f, 0x8d, 0x23, 0xbe, 0xc8, 0x63, + 0x65, 0x08, 0x7a, 0x0f, 0xae, 0xe6, 0x5c, 0xa5, 0x35, 0xc5, 0xfc, 0x7b, 0x26, 0xdc, 0x98, 0x09, + 0x8f, 0x45, 0xe6, 0x90, 0xaa, 0x18, 0x19, 0x6e, 0x98, 0x52, 0x43, 0xc3, 0x20, 0x07, 0xca, 0x4c, + 0x8d, 0xf3, 0x5c, 0x0e, 0xf5, 0x0e, 0x00, 0x53, 0xe3, 0x2c, 0x64, 0xf3, 0x17, 0x0b, 0xce, 0x75, + 0x73, 0x7c, 0x15, 0xf8, 0x11, 0x51, 0x33, 0xc1, 0xe4, 0xff, 0x5e, 0x46, 0x07, 0x2e, 0x25, 0xf3, + 0xe2, 0x27, 0xef, 0x7f, 0x30, 0xe9, 0xba, 0x72, 0xfd, 0x5f, 0x5d, 0x45, 0x05, 0xa3, 0x9c, 0xca, + 0x33, 0x6a, 0xfe, 0x55, 0x04, 0xd8, 0x1c, 0x11, 0x7a, 0x03, 0x0a, 0x01, 0xd5, 0x09, 0x95, 0xfa, + 0xc7, 0xab, 0xc4, 0x29, 0x04, 0x14, 0x17, 0x02, 0x8a, 0xea, 0x70, 0x44, 0x45, 0xda, 0x0b, 0x05, + 0x4d, 0x95, 0x56, 0x89, 0x63, 0x00, 0x5c, 0xa4, 0x62, 0x40, 0xd1, 0xc7, 0x70, 0x4e, 0x85, 0xbb, + 0xd3, 0xde, 0xe9, 0x86, 0x14, 0xfb, 0x97, 0xab, 0xc4, 0xd9, 0xa7, 0x70, 0x95, 0x8a, 0xfe, 0x56, + 0xd7, 0x3f, 0x84, 0x93, 0x39, 0x13, 0x32, 0xe0, 0x91, 0xe9, 0xd9, 0x7e, 0x79, 0x95, 0x38, 0x6b, + 0x08, 0xaf, 0x17, 0xa8, 0xb7, 0x77, 0x7f, 0x8e, 0xf4, 0x0f, 0x2e, 0x56, 0x89, 0xb3, 0x83, 0xef, + 0xde, 0xa8, 0x4f, 0xe0, 0xdc, 0x90, 0x2a, 0x08, 0x99, 0x54, 0x24, 0x8c, 0x75, 0x3b, 0x67, 0x89, + 0xed, 0x51, 0xf8, 0x4c, 0x03, 0x2f, 0xd6, 0x36, 0x7a, 0x0c, 0x25, 0xb6, 0x08, 0x94, 0xeb, 0x71, + 0xca, 0x74, 0x8f, 0x57, 0xfb, 0xd5, 0x55, 0xe2, 0x6c, 0x40, 0x7c, 0x9a, 0x2e, 0x3f, 0xe3, 0x94, + 0xa1, 0xe7, 0x70, 0xea, 0x13, 0xe9, 0xce, 0x24, 0xa3, 0xf6, 0xa9, 0x2e, 0xa3, 0xf7, 0x67, 0xe2, + 0xd4, 0xcc, 0xf9, 0x49, 0x3a, 0x69, 0x07, 0xbc, 0x13, 0x12, 0x35, 0x6e, 0x0f, 0x22, 0xb5, 0x4a, + 0x9c, 0x5c, 0xfc, 0xfb, 0xaf, 0xd7, 0x90, 0x8d, 0x9a, 0x41, 0xa4, 0xf0, 0x89, 0x4f, 0xe4, 0xd7, + 0x92, 0x51, 0xd4, 0x84, 0x63, 0x73, 0x9b, 0xed, 0x92, 0xee, 0x0f, 0x58, 0x25, 0x4e, 0x86, 0xe0, + 0xec, 0x9b, 0x56, 0x17, 0x93, 0xe5, 0x88, 0x78, 0x93, 0xbc, 0x99, 0x40, 0xff, 0x5a, 0x57, 0xb7, + 0x47, 0xe1, 0xb3, 0x0c, 0x58, 0x37, 0x4b, 0x0f, 0x2a, 0xe9, 0x1c, 0x74, 0x63, 0xb2, 0x9c, 0x72, + 0x42, 0xed, 0xb2, 0x76, 0xd5, 0x1b, 0xba, 0x8d, 0xe3, 0x72, 0x6a, 0x0d, 0x8d, 0x81, 0xde, 0x85, + 0x92, 0xc7, 0x23, 0xc9, 0x22, 0x39, 0x93, 0x76, 0xa5, 0x61, 0xb5, 0x4e, 0xcd, 0x96, 0xe4, 0x20, + 0xde, 0x2c, 0x9b, 0x3f, 0xc0, 0x95, 0x69, 0xae, 0xef, 0x66, 0x4c, 0xaa, 0xc1, 0x53, 0x73, 0x28, + 0x12, 0xf9, 0xf0, 0x66, 0x36, 0xae, 0x34, 0xe1, 0x06, 0x34, 0x3b, 0x3c, 0x33, 0x0e, 0xca, 0x4f, + 0x1e, 0xb7, 0xef, 0x9c, 0xd0, 0xed, 0x3b, 0xa2, 0xf5, 0x8b, 0xaf, 0x12, 0xe7, 0x00, 0x5f, 0xd1, + 0x2d, 0x8a, 0x66, 0x3f, 0x6a, 0x86, 0x70, 0x79, 0x87, 0x0b, 0x7a, 0x04, 0xe7, 0x7b, 0xff, 0x37, + 0x3d, 0x8f, 0xab, 0x3b, 0x51, 0x50, 0x1b, 0x2e, 0x77, 0x74, 0x5b, 0x93, 0xbb, 0x88, 0xef, 0x6d, + 0x69, 0x4d, 0xdc, 0xe6, 0x4f, 0x16, 0x1c, 0x0f, 0x89, 0x20, 0xa1, 0x44, 0x1f, 0xc1, 0xfd, 0x90, + 0x2c, 0x5c, 0x33, 0x44, 0x63, 0x31, 0x8b, 0x58, 0x3a, 0x53, 0xcc, 0x3d, 0xc8, 0x5e, 0x87, 0x5a, + 0x48, 0x16, 0x7a, 0x40, 0x0c, 0x53, 0x7a, 0xc8, 0xcc, 0xa5, 0x40, 0x43, 0x78, 0x94, 0x7a, 0x4e, + 0x99, 0x4f, 0xbc, 0xe5, 0xce, 0x3b, 0xb0, 0x1f, 0xc6, 0xe4, 0xd1, 0x08, 0xc9, 0xe2, 0x4b, 0x2d, + 0xde, 0x5c, 0xe4, 0x9d, 0x88, 0xfd, 0x67, 0xaf, 0x6e, 0xea, 0xd6, 0xeb, 0x9b, 0xba, 0xf5, 0xf7, + 0x4d, 0xdd, 0xfa, 0xf1, 0xb6, 0x7e, 0xf0, 0xfa, 0xb6, 0x7e, 0xf0, 0xc7, 0x6d, 0xfd, 0xe0, 0x9b, + 0xde, 0xd6, 0xc0, 0x49, 0x77, 0x5c, 0x3f, 0x6b, 0x1e, 0x9f, 0x6a, 0xe3, 0xda, 0xbc, 0xa3, 0x8b, + 0xcd, 0x4b, 0xaa, 0x27, 0xd0, 0xe8, 0x58, 0xab, 0x7a, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0x9d, + 0x9f, 0x0c, 0x09, 0x6c, 0x07, 0x00, 0x00, } func (m *Batch) Marshal() (dAtA []byte, err error) { @@ -1062,16 +1050,11 @@ func (m *Params) MarshalToSizedBuffer(dAtA []byte) (int, error) { if m.MaxLegacyDataResultPrunePerBlock != 0 { i = encodeVarintBatching(dAtA, i, uint64(m.MaxLegacyDataResultPrunePerBlock)) i-- - dAtA[i] = 0x18 + dAtA[i] = 0x10 } if m.MaxBatchPrunePerBlock != 0 { i = encodeVarintBatching(dAtA, i, uint64(m.MaxBatchPrunePerBlock)) i-- - dAtA[i] = 0x10 - } - if m.NumBatchesToKeep != 0 { - i = encodeVarintBatching(dAtA, i, uint64(m.NumBatchesToKeep)) - i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil @@ -1264,9 +1247,6 @@ func (m *Params) Size() (n int) { } var l int _ = l - if m.NumBatchesToKeep != 0 { - n += 1 + sovBatching(uint64(m.NumBatchesToKeep)) - } if m.MaxBatchPrunePerBlock != 0 { n += 1 + sovBatching(uint64(m.MaxBatchPrunePerBlock)) } @@ -2462,25 +2442,6 @@ func (m *Params) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field NumBatchesToKeep", wireType) - } - m.NumBatchesToKeep = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowBatching - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.NumBatchesToKeep |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field MaxBatchPrunePerBlock", wireType) } @@ -2499,7 +2460,7 @@ func (m *Params) Unmarshal(dAtA []byte) error { break } } - case 3: + case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field MaxLegacyDataResultPrunePerBlock", wireType) } diff --git a/x/batching/types/data_result.go b/x/batching/types/data_result.go index e2c768bd..87110f9c 100644 --- a/x/batching/types/data_result.go +++ b/x/batching/types/data_result.go @@ -64,7 +64,7 @@ func (dr *DataResult) TryHash() (string, error) { sedaPayloadHash := hasher.Sum(nil) hasher.Reset() - var allBytes []byte + allBytes := make([]byte, 0, len(versionHash)+len(drIDBytes)+1+1+len(resultHash)+len(blockHeightBytes)+len(blockTimestampBytes)+len(gasUsedBytes)+len(paybackAddrHash)+len(sedaPayloadHash)) allBytes = append(allBytes, versionHash...) allBytes = append(allBytes, drIDBytes...) allBytes = append(allBytes, consensusByte) diff --git a/x/batching/types/genesis_test.go b/x/batching/types/genesis_test.go index 5b6823d2..07f231a1 100644 --- a/x/batching/types/genesis_test.go +++ b/x/batching/types/genesis_test.go @@ -13,7 +13,6 @@ import ( var validGenesisJSON = []byte(`{ "params": { - "num_batches_to_keep": 12, "max_batch_prune_per_block": 5 }, "current_batch_number": "5", diff --git a/x/batching/types/params.go b/x/batching/types/params.go index 217ba765..c7822b9e 100644 --- a/x/batching/types/params.go +++ b/x/batching/types/params.go @@ -1,11 +1,6 @@ package types -import ( - sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" -) - const ( - DefaultNumBatchesToKeep = 10000 DefaultMaxBatchPrunePerBlock = 100 DefaultMaxLegacyDataResultPrunePerBlock = 1000 ) @@ -13,7 +8,6 @@ const ( // DefaultParams returns default batching module parameters. func DefaultParams() Params { return Params{ - NumBatchesToKeep: DefaultNumBatchesToKeep, MaxBatchPrunePerBlock: DefaultMaxBatchPrunePerBlock, MaxLegacyDataResultPrunePerBlock: DefaultMaxLegacyDataResultPrunePerBlock, } @@ -21,8 +15,5 @@ func DefaultParams() Params { // ValidateBasic performs basic validation on batching module parameters. func (p *Params) Validate() error { - if p.NumBatchesToKeep <= 3 { - return sdkerrors.ErrInvalidRequest.Wrapf("num batches to keep must be greater than 3: %d", p.NumBatchesToKeep) - } return nil } diff --git a/x/tally/keeper/gas_meter.go b/x/tally/keeper/gas_meter.go index 38ada84d..2d3c2d2a 100644 --- a/x/tally/keeper/gas_meter.go +++ b/x/tally/keeper/gas_meter.go @@ -18,7 +18,9 @@ import ( // sent to the core contract based on the given gas meter. It takes the ID and // the height of the request for event emission. func (k Keeper) DistributionsFromGasMeter(ctx sdk.Context, reqID string, reqHeight uint64, gasMeter *types.GasMeter, burnRatio math.LegacyDec) []types.Distribution { + //nolint:prealloc // TODO: To be addressed in the future dists := []types.Distribution{} + //nolint:prealloc // TODO: To be addressed in the future attrs := []sdk.Attribute{ sdk.NewAttribute(types.AttributeDataRequestID, reqID), sdk.NewAttribute(types.AttributeDataRequestHeight, strconv.FormatUint(reqHeight, 10)), From 87bb33afd9bcff7d824082bf42d2ef25e786726f Mon Sep 17 00:00:00 2001 From: Hyoung-yoon Kim Date: Mon, 12 Jan 2026 13:57:06 -0500 Subject: [PATCH 4/8] refactor(x/batching): update in hasPruningCaughtUp logic Before this commit, hasPruningCaughtUp was switched to true only when BatchPruneBatches has pruned all batched up to batchNumberAtUpgrade. This condition could be blocked for a long time by a big numBatchToKeep. So we add an alternative condition under which hasCaughtUp is switched to true: If all batches up to (currentBatchNum - numBatchesToKeep) have been pruned. BatchPruneBatches is updated accordingly. BasicPruneBatch now prunes a batch as long as it exists, whether it was created before or after the upgrade. --- x/batching/keeper/benchmark_endblock_test.go | 5 +- x/batching/keeper/endblock.go | 21 ++-- x/batching/keeper/pruning.go | 110 +++++++++---------- x/batching/keeper/pruning_test.go | 61 ++++++---- 4 files changed, 100 insertions(+), 97 deletions(-) diff --git a/x/batching/keeper/benchmark_endblock_test.go b/x/batching/keeper/benchmark_endblock_test.go index 911b6da0..0a2d0a42 100644 --- a/x/batching/keeper/benchmark_endblock_test.go +++ b/x/batching/keeper/benchmark_endblock_test.go @@ -14,7 +14,6 @@ func BenchmarkBatchPruning(b *testing.B) { numBatchesToKeep := uint64(1000) maxBatchPrunePerBlock := uint64(100) - var lastBatchNum uint64 for range numBatches { f.AddBlock() @@ -22,12 +21,12 @@ func BenchmarkBatchPruning(b *testing.B) { require.NoError(b, err) batch, dataEntries, valEntries, err := f.batchingKeeper.ConstructBatch(f.Context()) require.NoError(b, err) - lastBatchNum, err = f.batchingKeeper.SetNewBatch(f.Context(), batch, dataEntries, valEntries) + _, err = f.batchingKeeper.SetNewBatch(f.Context(), batch, dataEntries, valEntries) require.NoError(b, err) } for b.Loop() { - _, err := f.batchingKeeper.BatchPruneBatches(f.Context(), numBatchesToKeep, maxBatchPrunePerBlock, lastBatchNum) + _, err := f.batchingKeeper.BatchPruneBatches(f.Context(), numBatchesToKeep, maxBatchPrunePerBlock) require.NoError(b, err) } } diff --git a/x/batching/keeper/endblock.go b/x/batching/keeper/endblock.go index 598c72c8..4cfad242 100644 --- a/x/batching/keeper/endblock.go +++ b/x/batching/keeper/endblock.go @@ -27,10 +27,6 @@ func (k Keeper) EndBlock(ctx sdk.Context) error { if err != nil { return err } - batchNumAtUpgrade, err := k.GetBatchNumberAtUpgrade(ctx) - if err != nil { - return err - } // Since we're only using the secp256k1 key for batching, we only // need to check if the secp256k1 proving scheme is activated. @@ -52,9 +48,11 @@ func (k Keeper) EndBlock(ctx sdk.Context) error { return err } - err = k.BasicPruneBatch(ctx, newBatchNum, NumBatchesToKeep, batchNumAtUpgrade) - if err != nil { - return err + if newBatchNum >= NumBatchesToKeep { + err = k.BasicPruneBatch(ctx, newBatchNum-NumBatchesToKeep) + if err != nil { + return err + } } } } else { @@ -66,19 +64,14 @@ func (k Keeper) EndBlock(ctx sdk.Context) error { return err } if !hasCaughtUp { - // Batch prune MaxBatchPrunePerBlock batches and switch HasPruningCaughtUp - // to true if all batches up to the batch number at the time of the upgrade - // have been pruned. - // Note this operation does not prune data results, which will be pruned - // separately in the else clause. - lastPrunedBatchNum, err := k.BatchPruneBatches(ctx, NumBatchesToKeep, params.MaxBatchPrunePerBlock, batchNumAtUpgrade) + newHasCaughtUp, err := k.BatchPruneBatches(ctx, NumBatchesToKeep, params.MaxBatchPrunePerBlock) if err != nil { telemetry.SetGauge(1, types.TelemetryKeyBatchingPruningFail) k.Logger(ctx).Error("error while pruning batches", "err", err) return nil } - if lastPrunedBatchNum >= batchNumAtUpgrade { + if newHasCaughtUp { err = k.SetHasPruningCaughtUp(ctx, true) if err != nil { return err diff --git a/x/batching/keeper/pruning.go b/x/batching/keeper/pruning.go index ae33b3d2..3847d129 100644 --- a/x/batching/keeper/pruning.go +++ b/x/batching/keeper/pruning.go @@ -31,25 +31,20 @@ func (k Keeper) HasPruningCaughtUp(ctx sdk.Context) (bool, error) { return k.hasPruningCaughtUp.Get(ctx) } -// BasicPruneBatch prunes a batch at newBatchNum - numBatchesToKeep and all of its -// associated data. It returns without error if there is not enough batches or if -// the batch was created before the upgrade. -func (k Keeper) BasicPruneBatch(ctx sdk.Context, newBatchNum, numBatchesToKeep, batchNumAtUpgrade uint64) error { - // Do not prune until we have sufficient number of batches. - if newBatchNum < numBatchesToKeep { - return nil - } - - batchNumToPrune := newBatchNum - numBatchesToKeep - - // If there has been an upgrade (i.e., batchNumAtUpgrade is not 0), - // then prune only if the batch was created after the upgrade. - if batchNumAtUpgrade != 0 && batchNumToPrune <= batchNumAtUpgrade { - return nil - } - - batch, err := k.GetBatchByBatchNumber(ctx, batchNumToPrune) +// BasicPruneBatch prunes a given batch and its associated tree entries and +// signatures. It also prunes associated data results if the batchDataResults +// mapping is available. It returns without error if the batch does not exist +// (must have been pruned by the batch pruning strategy). +func (k Keeper) BasicPruneBatch(ctx sdk.Context, batchNumber uint64) error { + k.Logger(ctx).Info("[basic pruning strategy] pruning a batch", "batch_num", batchNumber) + + batch, err := k.GetBatchByBatchNumber(ctx, batchNumber) if err != nil { + if errors.Is(err, collections.ErrNotFound) { + // Batch may have been pruned already. + k.Logger(ctx).Info("[basic pruning strategy] batch not found", "batch_num", batchNumber) + return nil + } return err } batchHeight := batch.BlockHeight @@ -58,12 +53,12 @@ func (k Keeper) BasicPruneBatch(ctx sdk.Context, newBatchNum, numBatchesToKeep, if err != nil { return err } - err = k.dataResultTreeEntries.Remove(ctx, batchNumToPrune) + err = k.dataResultTreeEntries.Remove(ctx, batchNumber) if err != nil { return err } - valRng := new(collections.Range[collections.Pair[uint64, []byte]]).Prefix(collections.PairPrefix[uint64, []byte](batchNumToPrune)) + valRng := new(collections.Range[collections.Pair[uint64, []byte]]).Prefix(collections.PairPrefix[uint64, []byte](batchNumber)) err = k.validatorTreeEntries.Clear(ctx, valRng) if err != nil { return err @@ -73,10 +68,12 @@ func (k Keeper) BasicPruneBatch(ctx sdk.Context, newBatchNum, numBatchesToKeep, return err } - dataResults, err := k.GetBatchDataResults(ctx, batchNumToPrune) + dataResults, err := k.GetBatchDataResults(ctx, batchNumber) if err != nil { if errors.Is(err, collections.ErrNotFound) { - k.Logger(ctx).Info("cannot prune batch because schema change has not been applied", "batch_num", batchNumToPrune) + // Batches created before the upgrade do not have batchDataResults + // mapping, so we resort to PruneLegacyDataResults(). + k.Logger(ctx).Info("[basic pruning strategy] skip pruning data results", "batch_num", batchNumber) return nil } return err @@ -91,54 +88,56 @@ func (k Keeper) BasicPruneBatch(ctx sdk.Context, newBatchNum, numBatchesToKeep, return err } } - err = k.RemoveBatchDataResults(ctx, batchNumToPrune) + err = k.RemoveBatchDataResults(ctx, batchNumber) if err != nil { return err } - k.Logger(ctx).Info("pruned a batch (basic strategy)", "batch_num", batchNumToPrune) return nil } -// BatchPruneBatches prunes batches and their associated data, except for data -// results, in batches based on the module parameters NumBatchesToKeep and -// MaxBatchPrunePerBlock. -// It returns the batch number of the last batch that has been confirmed to have -// been pruned. -func (k Keeper) BatchPruneBatches(ctx sdk.Context, numBatchesToKeep, maxBatchPrunePerBlock, batchNumAtUpgrade uint64) (uint64, error) { +// BatchPruneBatches prunes up to MaxBatchPrunePerBlock batches and their associated +// data except for data results. +// It returns a boolean hasCaughtUp if either of the following conditions is met: +// (i) All batches up to batchNumberAtUpgrade have been pruned. +// (ii) All batches up to (currentBatchNum - numBatchesToKeep) have been pruned. +func (k Keeper) BatchPruneBatches(ctx sdk.Context, numBatchesToKeep, maxBatchPrunePerBlock uint64) (bool, error) { if maxBatchPrunePerBlock == 0 { - k.Logger(ctx).Info("skip batch pruning", "max_batch_prune_per_block", maxBatchPrunePerBlock) - return 0, nil + k.Logger(ctx).Info("[batch pruning strategy] disabled") + return false, nil + } + + batchNumAtUpgrade, err := k.GetBatchNumberAtUpgrade(ctx) + if err != nil { + return false, nil + } + if batchNumAtUpgrade == 0 { + k.Logger(ctx).Info("[batch pruning strategy] skipped due to lack of upgrade") + return false, nil } - // Prune up to, but not including, current batch number minus numBatchesToKeep. currentBatchNum, err := k.GetCurrentBatchNum(ctx) if err != nil { - return 0, err + return false, err } if currentBatchNum <= numBatchesToKeep { - k.Logger(ctx).Info("skip batch pruning", "current_batch_num", currentBatchNum, "num_batches_to_keep", numBatchesToKeep) - return 0, nil + k.Logger(ctx).Info("[batch pruning strategy] skipped", "current_batch_num", currentBatchNum, "num_batches_to_keep", numBatchesToKeep) + return false, nil } rngEnd := min(currentBatchNum-numBatchesToKeep, batchNumAtUpgrade+1) rng := new(collections.Range[uint64]).EndExclusive(rngEnd) iter, err := k.batches.Indexes.Number.Iterate(ctx, rng) if err != nil { - return 0, err + return false, err } defer iter.Close() - var firstKey *collections.Pair[uint64, int64] - var pruneCount uint64 - var lastPrunedBatchNum uint64 + var pruneCount, lastPrunedBatchNum uint64 for ; iter.Valid(); iter.Next() { fullKey, err := iter.FullKey() if err != nil { - return 0, err - } - if firstKey == nil { - firstKey = &fullKey + return false, err } batchNum, batchHeight := fullKey.K1(), fullKey.K2() @@ -149,7 +148,7 @@ func (k Keeper) BatchPruneBatches(ctx sdk.Context, numBatchesToKeep, maxBatchPru err = k.batches.Remove(ctx, batchHeight) if err != nil { - return 0, err + return false, err } k.Logger(ctx).Info("pruned a batch (batch pruning strategy)", "batch_num", batchNum) @@ -161,31 +160,30 @@ func (k Keeper) BatchPruneBatches(ctx sdk.Context, numBatchesToKeep, maxBatchPru } } - if firstKey == nil { - k.Logger(ctx).Info("no batches to prune (batch pruning strategy)") - // This means all batches up to batch number rngEnd - 1 have been pruned. - // Note we subtract 1 because rngEnd is exclusive. - return rngEnd - 1, nil + if pruneCount == 0 { + k.Logger(ctx).Info("[batch pruning strategy] no batches to prune") + return true, nil } - dataRng := new(collections.Range[uint64]).EndExclusive(firstKey.K1() + pruneCount) + dataRng := new(collections.Range[uint64]).EndExclusive(lastPrunedBatchNum + 1) err = k.dataResultTreeEntries.Clear(ctx, dataRng) if err != nil { - return 0, err + return false, err } valRng := new(collections.Range[collections.Pair[uint64, []byte]]). - EndExclusive(collections.PairPrefix[uint64, []byte](firstKey.K1() + pruneCount)) + EndExclusive(collections.PairPrefix[uint64, []byte](lastPrunedBatchNum + 1)) err = k.validatorTreeEntries.Clear(ctx, valRng) if err != nil { - return 0, err + return false, err } err = k.batchSignatures.Clear(ctx, valRng) if err != nil { - return 0, err + return false, err } - return lastPrunedBatchNum, nil + k.Logger(ctx).Info("[batch pruning strategy] pruned batches", "count", pruneCount) + return pruneCount < maxBatchPrunePerBlock, nil } func (k Keeper) PruneLegacyDataResults(ctx sdk.Context, maxDataResultPrunePerBlock uint64) error { diff --git a/x/batching/keeper/pruning_test.go b/x/batching/keeper/pruning_test.go index cf14da26..16ed4981 100644 --- a/x/batching/keeper/pruning_test.go +++ b/x/batching/keeper/pruning_test.go @@ -21,13 +21,17 @@ func TestBatchPruneBatches(t *testing.T) { numBatchesToKeep := uint64(75) maxBatchPrunePerBlock := uint64(150) + err := f.batchingKeeper.SetParams(f.Context(), types.Params{ + MaxBatchPrunePerBlock: maxBatchPrunePerBlock, + }) + require.NoError(t, err) + // Should prune nothing. - lastRemovedBatchNum, err := f.batchingKeeper.BatchPruneBatches(f.Context(), numBatchesToKeep, maxBatchPrunePerBlock, 0) + hasCaughtUp, err := f.batchingKeeper.BatchPruneBatches(f.Context(), numBatchesToKeep, maxBatchPrunePerBlock) require.NoError(t, err) - require.Equal(t, uint64(0), lastRemovedBatchNum) + require.False(t, hasCaughtUp) // Create 300 batches with random associated data. - var lastBatchNum uint64 for range 300 { f.AddBlock() @@ -35,7 +39,7 @@ func TestBatchPruneBatches(t *testing.T) { require.NoError(t, err) batch, dataEntries, valEntries, err := f.batchingKeeper.ConstructBatch(f.Context()) require.NoError(t, err) - lastBatchNum, err = f.batchingKeeper.SetNewBatch(f.Context(), batch, dataEntries, valEntries) + _, err = f.batchingKeeper.SetNewBatch(f.Context(), batch, dataEntries, valEntries) require.NoError(t, err) err = f.batchingKeeper.SetBatchSigSecp256k1(f.Context(), batch.BatchNumber, valEntries[0].ValidatorAddress, generateRandomBytes(64)) require.NoError(t, err) @@ -46,10 +50,15 @@ func TestBatchPruneBatches(t *testing.T) { require.Equal(t, 300, len(batches)) // Suppose an upgrade happens here and sets batchNumberAtUpgrade. + err = f.batchingKeeper.SetBatchNumberAtUpgrade(f.Context()) + require.NoError(t, err) + err = f.batchingKeeper.SetHasPruningCaughtUp(f.Context(), false) + require.NoError(t, err) + // Should prune first 150 batches (0-149) - lastRemovedBatchNum, err = f.batchingKeeper.BatchPruneBatches(f.Context(), numBatchesToKeep, maxBatchPrunePerBlock, lastBatchNum) + hasCaughtUp, err = f.batchingKeeper.BatchPruneBatches(f.Context(), numBatchesToKeep, maxBatchPrunePerBlock) require.NoError(t, err) - require.Equal(t, uint64(149), lastRemovedBatchNum) + require.False(t, hasCaughtUp) batches, err = f.batchingKeeper.GetAllBatches(f.Context()) require.NoError(t, err) @@ -65,9 +74,9 @@ func TestBatchPruneBatches(t *testing.T) { } // Should prune second 75 batches (150-224) - lastRemovedBatchNum, err = f.batchingKeeper.BatchPruneBatches(f.Context(), numBatchesToKeep, maxBatchPrunePerBlock, lastBatchNum) + hasCaughtUp, err = f.batchingKeeper.BatchPruneBatches(f.Context(), numBatchesToKeep, maxBatchPrunePerBlock) require.NoError(t, err) - require.Equal(t, uint64(224), lastRemovedBatchNum) + require.True(t, hasCaughtUp) batches, err = f.batchingKeeper.GetAllBatches(f.Context()) require.NoError(t, err) @@ -83,9 +92,9 @@ func TestBatchPruneBatches(t *testing.T) { } // Should prune nothing - lastRemovedBatchNum, err = f.batchingKeeper.BatchPruneBatches(f.Context(), numBatchesToKeep, maxBatchPrunePerBlock, lastBatchNum) + hasCaughtUp, err = f.batchingKeeper.BatchPruneBatches(f.Context(), numBatchesToKeep, maxBatchPrunePerBlock) require.NoError(t, err) - require.Equal(t, uint64(224), lastRemovedBatchNum) + require.True(t, hasCaughtUp) batches, err = f.batchingKeeper.GetAllBatches(f.Context()) require.NoError(t, err) @@ -364,19 +373,21 @@ func TestPruningMockedUpgrade(t *testing.T) { // Block 31: // - Creates 31st batch Batch 30 + // - Basic pruning prunes Batch 20 // - Batch prunes Batches 0-14 f.BatchingEndBlock(t, 10) batches, err := f.batchingKeeper.GetAllBatches(f.Context()) require.NoError(t, err) - require.Equal(t, 16, len(batches)) + require.Equal(t, 15, len(batches)) require.Equal(t, uint64(15), batches[0].BatchNumber) require.Equal(t, uint64(30), batches[len(batches)-1].BatchNumber) - for i := 0; i <= 14; i++ { - f.checkNoBatchData(t, uint64(i)) - } - for i := 15; i <= 30; i++ { - f.checkBatchData(t, uint64(i), false) + for i := 0; i <= 30; i++ { + if i <= 14 || i == 20 { + f.checkNoBatchData(t, uint64(i)) + } else { + f.checkBatchData(t, uint64(i), false) + } } f.checkNumLegacyDataResults(t, 300) @@ -385,8 +396,11 @@ func TestPruningMockedUpgrade(t *testing.T) { require.False(t, hasCaughtUp) // Block 32~39: - // - Creates 32nd batch Batch 31 - // - Batch pruning in effect but limited by NumBatchesToKeep + // - Block 32: + // - Creates 32nd batch Batch 31 + // - Basic pruning prunes Batch 21 + // - Batch pruning prunes Batches 15-21 (20 & 21 already pruned by basic pruning) + // - Legacy data result pruning starts next block for i := range 8 { f.BatchingEndBlock(t, 10) @@ -401,16 +415,15 @@ func TestPruningMockedUpgrade(t *testing.T) { for j := 22 + i; j <= 31+i; j++ { f.checkBatchData(t, uint64(j), false) } - f.checkNumLegacyDataResults(t, 300) + f.checkNumLegacyDataResults(t, max(300-80*i, 0)) hasCaughtUp, err = f.batchingKeeper.HasPruningCaughtUp(f.Context()) require.NoError(t, err) - require.False(t, hasCaughtUp) + require.True(t, hasCaughtUp) } // Block 40 - 43: // - Batch creation at every block but number of batches stays at 10 with basic pruning. - // - HasPruningCaughtUp is now True and legacy data results pruning is in effect. for i := range 4 { f.BatchingEndBlock(t, 10) @@ -425,7 +438,7 @@ func TestPruningMockedUpgrade(t *testing.T) { for j := 30 + i; j <= 39+i; j++ { f.checkBatchData(t, uint64(j), false) } - f.checkNumLegacyDataResults(t, 300-80*i) + f.checkNumLegacyDataResults(t, 0) hasCaughtUp, err = f.batchingKeeper.HasPruningCaughtUp(f.Context()) require.NoError(t, err) @@ -460,9 +473,9 @@ func (f *fixture) BatchingEndBlock(t *testing.T, numDataResults int) { func (f *fixture) checkNoBatchData(t *testing.T, batchNum uint64) { batch, err := f.batchingKeeper.GetBatchByBatchNumber(f.Context(), batchNum) - require.ErrorIs(t, err, collections.ErrNotFound) + require.ErrorIs(t, err, collections.ErrNotFound, "batchNum %d", batchNum) dataEntries, err := f.batchingKeeper.GetDataResultTreeEntries(f.Context(), batchNum) - require.ErrorIs(t, err, collections.ErrNotFound) + require.ErrorIs(t, err, collections.ErrNotFound, "batchNum %d", batchNum) valEntries, _ := f.batchingKeeper.GetValidatorTreeEntries(f.Context(), batchNum) // require.ErrorIs(t, err, collections.ErrNotFound) // this function does not error even if there are no entries. sigs, _ := f.batchingKeeper.GetBatchSignatures(f.Context(), batchNum) From 49f371f9007de7044f0eb283d188c94cc4acb864 Mon Sep 17 00:00:00 2001 From: Hyoung-yoon Kim Date: Mon, 12 Jan 2026 15:02:27 -0500 Subject: [PATCH 5/8] fix(x/batching): create map from batch number to data results --- x/batching/keeper/endblock.go | 13 ++++++++ x/batching/keeper/keeper.go | 6 ++-- x/batching/keeper/pruning_test.go | 52 +++++++++++++++++++++++++++++-- 3 files changed, 67 insertions(+), 4 deletions(-) diff --git a/x/batching/keeper/endblock.go b/x/batching/keeper/endblock.go index 4cfad242..818d320c 100644 --- a/x/batching/keeper/endblock.go +++ b/x/batching/keeper/endblock.go @@ -170,6 +170,7 @@ func (k Keeper) ConstructDataResultTree(ctx sdk.Context, newBatchNum uint64) (ty entries := make([][]byte, len(dataResults)) treeEntries := make([][]byte, len(dataResults)) + dataRequestIDHeights := make([]types.DataRequestIDHeight, len(dataResults)) for i, res := range dataResults { resID, err := hex.DecodeString(res.Id) if err != nil { @@ -182,6 +183,18 @@ func (k Keeper) ConstructDataResultTree(ctx sdk.Context, newBatchNum uint64) (ty if err != nil { return types.DataResultTreeEntries{}, nil, err } + + dataRequestIDHeights[i] = types.DataRequestIDHeight{ + DataRequestId: res.DrId, + DataRequestHeight: res.DrBlockHeight, + } + } + + err = k.SetBatchDataResults(ctx, newBatchNum, types.DataRequestIDHeights{ + DataRequestIdHeights: dataRequestIDHeights, + }) + if err != nil { + return types.DataResultTreeEntries{}, nil, err } return types.DataResultTreeEntries{Entries: entries}, utils.RootFromEntries(treeEntries), nil diff --git a/x/batching/keeper/keeper.go b/x/batching/keeper/keeper.go index b2d0f9d1..9228ac15 100644 --- a/x/batching/keeper/keeper.go +++ b/x/batching/keeper/keeper.go @@ -48,8 +48,10 @@ type Keeper struct { // legacyDataResults is the older version of dataResults. The items in this // collection do not have corresponding items in batchDataResults. legacyDataResults collections.Map[collections.Triple[bool, string, uint64], types.DataResult] - // hasPruningCaughtUp indicates that all batches up to batchNumberAtUpgrade have - // been pruned by batch pruning. + // hasPruningCaughtUp is switched to true when either of the following conditions + // is met: + // (i) All batches up to batchNumberAtUpgrade have been pruned. + // (ii) All batches up to (currentBatchNum - numBatchesToKeep) have been pruned. hasPruningCaughtUp collections.Item[bool] // batchNumberAtUpgrade is the batch number of the latest batch at upgrade time // except when its value is 0, in which case there was no upgrade. diff --git a/x/batching/keeper/pruning_test.go b/x/batching/keeper/pruning_test.go index 16ed4981..42b167b1 100644 --- a/x/batching/keeper/pruning_test.go +++ b/x/batching/keeper/pruning_test.go @@ -13,6 +13,50 @@ import ( pubkeytypes "github.com/sedaprotocol/seda-chain/x/pubkey/types" ) +func TestBasicPruneBatch(t *testing.T) { + f := initFixture(t) + + f.addBatchSigningValidators(t, 1) + + err := f.pubKeyKeeper.SetProvingScheme(f.Context(), pubkeytypes.ProvingScheme{ + Index: uint32(sedatypes.SEDAKeyIndexSecp256k1), + IsActivated: true, + }) + require.NoError(t, err) + + // Adjust the global variable for the test. + original := keeper.NumBatchesToKeep + defer func() { + keeper.NumBatchesToKeep = original + }() + keeper.NumBatchesToKeep = 5 + + err = f.batchingKeeper.SetParams(f.Context(), types.Params{ + MaxBatchPrunePerBlock: 15, + MaxLegacyDataResultPrunePerBlock: 80, + }) + require.NoError(t, err) + + // 5 batching endblocks create 5 batches. + for i := range 5 { + if i == 1 { + // Create Batch 2 with a validator change only without new data results. + f.addBatchSigningValidators(t, 1) + f.BatchingEndBlock(t, 0) + } else { + f.BatchingEndBlock(t, 5) + } + } + + // Creates batch 6 and prunes batch 1 + f.BatchingEndBlock(t, 5) + + // Creates Batch 7 (with no data results) and prunes Batch 2, + // which does not have associated data results. + f.addBatchSigningValidators(t, 1) + f.BatchingEndBlock(t, 0) +} + func TestBatchPruneBatches(t *testing.T) { f := initFixture(t) @@ -375,7 +419,12 @@ func TestPruningMockedUpgrade(t *testing.T) { // - Creates 31st batch Batch 30 // - Basic pruning prunes Batch 20 // - Batch prunes Batches 0-14 - f.BatchingEndBlock(t, 10) + f.BatchingEndBlock(t, 12) // 12 data results + + // Check that batchDataResults collection was used. + drIDHeights, err := f.batchingKeeper.GetBatchDataResults(f.Context(), 30) + require.NoError(t, err) + require.Equal(t, 12, len(drIDHeights.DataRequestIdHeights)) batches, err := f.batchingKeeper.GetAllBatches(f.Context()) require.NoError(t, err) @@ -452,7 +501,6 @@ func TestPruningMockedUpgrade(t *testing.T) { // Block 45 without batch creation f.BatchingEndBlock(t, 5) f.checkNumLegacyDataResults(t, 0) - } // BatchingEndBlock adds a given number of data results to the store and executes From 652e6c47d3a746fde2814a62eca1be2b2dd4113f Mon Sep 17 00:00:00 2001 From: Hyoung-yoon Kim Date: Fri, 9 Jan 2026 05:46:03 -0500 Subject: [PATCH 6/8] chore(x/batching): set version to 2 and add migrator from version 1 --- app/upgrades/mainnet/v1.0.7/constants.go | 54 ------------------------ x/batching/keeper/migration.go | 35 +++++++++++++++ x/batching/keeper/migration_test.go | 49 +++++++++++++++++++++ x/batching/keeper/pruning.go | 19 ++++++--- x/batching/module.go | 7 ++- 5 files changed, 102 insertions(+), 62 deletions(-) delete mode 100644 app/upgrades/mainnet/v1.0.7/constants.go create mode 100644 x/batching/keeper/migration.go create mode 100644 x/batching/keeper/migration_test.go diff --git a/app/upgrades/mainnet/v1.0.7/constants.go b/app/upgrades/mainnet/v1.0.7/constants.go deleted file mode 100644 index 60840df8..00000000 --- a/app/upgrades/mainnet/v1.0.7/constants.go +++ /dev/null @@ -1,54 +0,0 @@ -package v1 - -import ( - "context" - - storetypes "cosmossdk.io/store/types" - upgradetypes "cosmossdk.io/x/upgrade/types" - - sdk "github.com/cosmos/cosmos-sdk/types" - "github.com/cosmos/cosmos-sdk/types/module" - - "github.com/sedaprotocol/seda-chain/app/keepers" - "github.com/sedaprotocol/seda-chain/app/upgrades" -) - -const ( - UpgradeName = "v" // TODO Update name and register this handler. -) - -var Upgrade = upgrades.Upgrade{ - UpgradeName: UpgradeName, - CreateUpgradeHandler: CreateUpgradeHandler, - StoreUpgrades: storetypes.StoreUpgrades{ - Added: []string{}, - Deleted: []string{}, - }, -} - -func CreateUpgradeHandler( - mm upgrades.ModuleManager, - configurator module.Configurator, - keepers *keepers.AppKeepers, -) upgradetypes.UpgradeHandler { - return func(context context.Context, _ upgradetypes.Plan, fromVM module.VersionMap) (module.VersionMap, error) { - ctx := sdk.UnwrapSDKContext(context) - - // Run module migrations. - migrations, err := mm.RunMigrations(ctx, configurator, fromVM) - if err != nil { - return nil, err - } - - err = keepers.BatchingKeeper.SetBatchNumberAtUpgrade(ctx) - if err != nil { - return nil, err - } - err = keepers.BatchingKeeper.SetHasPruningCaughtUp(ctx, false) - if err != nil { - return nil, err - } - - return migrations, nil - } -} diff --git a/x/batching/keeper/migration.go b/x/batching/keeper/migration.go new file mode 100644 index 00000000..2f5c1c8e --- /dev/null +++ b/x/batching/keeper/migration.go @@ -0,0 +1,35 @@ +package keeper + +import ( + sdk "github.com/cosmos/cosmos-sdk/types" + + "github.com/sedaprotocol/seda-chain/x/batching/types" +) + +// Migrator is a struct for handling in-place store migrations. +type Migrator struct { + keeper Keeper +} + +// NewMigrator returns a new Migrator. +func NewMigrator(keeper Keeper) Migrator { + return Migrator{keeper: keeper} +} + +// Migrate1to2 migrates from version 1 to 2. +func (m Migrator) Migrate1to2(ctx sdk.Context) error { + // Initialize new states. + err := m.keeper.SetBatchNumberAtUpgrade(ctx) + if err != nil { + return err + } + err = m.keeper.SetHasPruningCaughtUp(ctx, false) + if err != nil { + return err + } + err = m.keeper.SetParams(ctx, types.DefaultParams()) + if err != nil { + return err + } + return nil +} diff --git a/x/batching/keeper/migration_test.go b/x/batching/keeper/migration_test.go new file mode 100644 index 00000000..700d041a --- /dev/null +++ b/x/batching/keeper/migration_test.go @@ -0,0 +1,49 @@ +package keeper_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/sedaprotocol/seda-chain/x/batching/keeper" + "github.com/sedaprotocol/seda-chain/x/batching/types" +) + +// TestMigration tests the migration of unbatched legacy data results to the new collection. +func TestMigration(t *testing.T) { + f := initFixture(t) + + // Create 5 batches with 10 data results each before mock upgrade. + // We simulate the chain before the upgrade by using legacy functions. + for range 5 { + f.AddBlock() + + dataResults := generateDataResults(t, 10) + for _, dataResult := range dataResults { + err := f.batchingKeeper.LegacySetDataResultForBatching(f.Context(), dataResult) + require.NoError(t, err) + } + batch, dataEntries, valEntries, err := f.batchingKeeper.LegacyConstructBatch(f.Context()) + require.NoError(t, err) + _, err = f.batchingKeeper.SetNewBatch(f.Context(), batch, dataEntries, valEntries) + require.NoError(t, err) + } + + // Execute the migration. + migrator := keeper.NewMigrator(f.batchingKeeper) + err := migrator.Migrate1to2(f.Context()) + require.NoError(t, err) + + // Check that batchNumberAtUpgrade and hasPruningCaughtUp have been set. + batchNumberAtUpgrade, err := f.batchingKeeper.GetBatchNumberAtUpgrade(f.Context()) + require.NoError(t, err) + require.Equal(t, uint64(4), batchNumberAtUpgrade) + + hasPruningCaughtUp, err := f.batchingKeeper.HasPruningCaughtUp(f.Context()) + require.NoError(t, err) + require.False(t, hasPruningCaughtUp) + + params, err := f.batchingKeeper.GetParams(f.Context()) + require.NoError(t, err) + require.Equal(t, types.DefaultParams(), params) +} diff --git a/x/batching/keeper/pruning.go b/x/batching/keeper/pruning.go index 3847d129..63937a3c 100644 --- a/x/batching/keeper/pruning.go +++ b/x/batching/keeper/pruning.go @@ -16,7 +16,12 @@ func (k Keeper) SetBatchNumberAtUpgrade(ctx sdk.Context) error { if err != nil { return err } - return k.batchNumberAtUpgrade.Set(ctx, currentBatchNum-1) + err = k.batchNumberAtUpgrade.Set(ctx, currentBatchNum-1) + if err != nil { + return err + } + k.Logger(ctx).Info("set batch number at upgrade", "batch_number", currentBatchNum-1) + return nil } func (k Keeper) GetBatchNumberAtUpgrade(ctx sdk.Context) (uint64, error) { @@ -150,7 +155,7 @@ func (k Keeper) BatchPruneBatches(ctx sdk.Context, numBatchesToKeep, maxBatchPru if err != nil { return false, err } - k.Logger(ctx).Info("pruned a batch (batch pruning strategy)", "batch_num", batchNum) + k.Logger(ctx).Debug("[batch pruning strategy] pruned a batch", "batch_num", batchNum) lastPrunedBatchNum = batchNum @@ -188,7 +193,7 @@ func (k Keeper) BatchPruneBatches(ctx sdk.Context, numBatchesToKeep, maxBatchPru func (k Keeper) PruneLegacyDataResults(ctx sdk.Context, maxDataResultPrunePerBlock uint64) error { if maxDataResultPrunePerBlock == 0 { - k.Logger(ctx).Info("skip legacy data result pruning", "max_data_results_to_check_for_prune", maxDataResultPrunePerBlock) + k.Logger(ctx).Info("[legacy data result pruning] disabled") return nil } @@ -198,7 +203,7 @@ func (k Keeper) PruneLegacyDataResults(ctx sdk.Context, maxDataResultPrunePerBlo } defer iter.Close() - var numPruned uint64 + var pruneCount uint64 for ; iter.Valid(); iter.Next() { kv, err := iter.KeyValue() if err != nil { @@ -214,12 +219,12 @@ func (k Keeper) PruneLegacyDataResults(ctx sdk.Context, maxDataResultPrunePerBlo return err } - numPruned++ - if numPruned == maxDataResultPrunePerBlock { + pruneCount++ + if pruneCount == maxDataResultPrunePerBlock { break } } - k.Logger(ctx).Info("pruned legacy data results", "num_pruned", numPruned) + k.Logger(ctx).Info("[legacy data result pruning] pruned legacy data results", "count", pruneCount) return nil } diff --git a/x/batching/module.go b/x/batching/module.go index f36b1f8c..5b1b88c1 100644 --- a/x/batching/module.go +++ b/x/batching/module.go @@ -116,6 +116,11 @@ func NewAppModule(cdc codec.Codec, keeper keeper.Keeper) AppModule { // RegisterServices registers a gRPC query service to respond to the module-specific gRPC queries func (am AppModule) RegisterServices(cfg module.Configurator) { types.RegisterQueryServer(cfg.QueryServer(), keeper.Querier{Keeper: am.keeper}) + + m := keeper.NewMigrator(am.keeper) + if err := cfg.RegisterMigration(types.ModuleName, 1, m.Migrate1to2); err != nil { + panic(errorsmod.Wrapf(err, "failed to migrate x/%s from version 1 to 2", types.ModuleName)) + } } // RegisterInvariants registers the invariants of the module. If an invariant deviates from its predicted value, the InvariantRegistry triggers appropriate logic (most often the chain will be halted) @@ -135,7 +140,7 @@ func (am AppModule) ExportGenesis(ctx sdk.Context, cdc codec.JSONCodec) json.Raw } // ConsensusVersion is a sequence number for state-breaking change of the module. It should be incremented on each consensus-breaking change introduced by the module. To avoid wrong/empty versions, the initial version should be set to 1 -func (AppModule) ConsensusVersion() uint64 { return 1 } +func (AppModule) ConsensusVersion() uint64 { return 2 } // BeginBlock contains the logic that is automatically triggered at the beginning of each block func (am AppModule) BeginBlock(_ context.Context) error { From 820865294f37ab95fc61f8396faab56b8c44c6d2 Mon Sep 17 00:00:00 2001 From: Hyoung-yoon Kim Date: Mon, 12 Jan 2026 05:51:52 -0500 Subject: [PATCH 7/8] fix(x/batching): proper registration of MsgUpdateParams --- proto/sedachain/batching/v1/tx.proto | 28 ++++++++++++++++++++++++++++ x/batching/keeper/msg_server.go | 5 +++++ x/batching/module.go | 1 + x/batching/types/codec.go | 1 + 4 files changed, 35 insertions(+) create mode 100644 proto/sedachain/batching/v1/tx.proto diff --git a/proto/sedachain/batching/v1/tx.proto b/proto/sedachain/batching/v1/tx.proto new file mode 100644 index 00000000..179dca32 --- /dev/null +++ b/proto/sedachain/batching/v1/tx.proto @@ -0,0 +1,28 @@ +syntax = "proto3"; +package sedachain.batching.v1; + +import "cosmos/msg/v1/msg.proto"; +import "gogoproto/gogo.proto"; +import "cosmos_proto/cosmos.proto"; +import "sedachain/batching/v1/batching.proto"; + +option go_package = "github.com/sedaprotocol/seda-chain/x/batching/types"; + +// Msg service defines the batching tx gRPC methods. +service Msg { + // The UpdateParams method updates the module's parameters. + rpc UpdateParams(MsgUpdateParams) returns (MsgUpdateParamsResponse); +} + +// The request message for the UpdateParams method. +message MsgUpdateParams { + option (cosmos.msg.v1.signer) = "authority"; + + // Authority is the address that controls the module (defaults to x/gov unless + // overwritten). + string authority = 1 [ (cosmos_proto.scalar) = "cosmos.AddressString" ]; + Params params = 2 [ (gogoproto.nullable) = false ]; +} + +// The response message for the UpdateParams method. +message MsgUpdateParamsResponse {} diff --git a/x/batching/keeper/msg_server.go b/x/batching/keeper/msg_server.go index 05154f01..d4f50fd5 100644 --- a/x/batching/keeper/msg_server.go +++ b/x/batching/keeper/msg_server.go @@ -15,6 +15,11 @@ type msgServer struct { var _ types.MsgServer = msgServer{} +// NewMsgServerImpl returns an implementation of the MsgServer interface. +func NewMsgServerImpl(keeper Keeper) types.MsgServer { + return &msgServer{Keeper: keeper} +} + // UpdateParams updates the module parameters. func (m msgServer) UpdateParams(goCtx context.Context, req *types.MsgUpdateParams) (*types.MsgUpdateParamsResponse, error) { ctx := sdk.UnwrapSDKContext(goCtx) diff --git a/x/batching/module.go b/x/batching/module.go index 5b1b88c1..8a4bb037 100644 --- a/x/batching/module.go +++ b/x/batching/module.go @@ -116,6 +116,7 @@ func NewAppModule(cdc codec.Codec, keeper keeper.Keeper) AppModule { // RegisterServices registers a gRPC query service to respond to the module-specific gRPC queries func (am AppModule) RegisterServices(cfg module.Configurator) { types.RegisterQueryServer(cfg.QueryServer(), keeper.Querier{Keeper: am.keeper}) + types.RegisterMsgServer(cfg.MsgServer(), keeper.NewMsgServerImpl(am.keeper)) m := keeper.NewMigrator(am.keeper) if err := cfg.RegisterMigration(types.ModuleName, 1, m.Migrate1to2); err != nil { diff --git a/x/batching/types/codec.go b/x/batching/types/codec.go index 6467df48..2d949d0d 100644 --- a/x/batching/types/codec.go +++ b/x/batching/types/codec.go @@ -19,6 +19,7 @@ var ( func RegisterInterfaces(registry cdctypes.InterfaceRegistry) { registry.RegisterImplementations((*sdk.Msg)(nil), &BatchDoubleSign{}, + &MsgUpdateParams{}, ) registry.RegisterImplementations( From 7b1305a14d5f95927c05d5b6994b273e426368d4 Mon Sep 17 00:00:00 2001 From: Hyoung-yoon Kim Date: Wed, 8 Apr 2026 10:51:16 -0400 Subject: [PATCH 8/8] chore: address lint warnings --- app/app.go | 1 - app/utils/seda_keys.go | 2 +- x/batching/types/data_result.go | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/app.go b/app/app.go index 2b84b90a..ce62f6a0 100644 --- a/app/app.go +++ b/app/app.go @@ -139,7 +139,6 @@ import ( "github.com/sedaprotocol/seda-chain/app/keepers" appparams "github.com/sedaprotocol/seda-chain/app/params" "github.com/sedaprotocol/seda-chain/app/utils" - // Used in cosmos-sdk when registering the route for swagger docs. _ "github.com/sedaprotocol/seda-chain/client/docs/statik" "github.com/sedaprotocol/seda-chain/cmd/sedad/gentx" diff --git a/app/utils/seda_keys.go b/app/utils/seda_keys.go index 1ff6fe32..74377e9b 100644 --- a/app/utils/seda_keys.go +++ b/app/utils/seda_keys.go @@ -39,7 +39,7 @@ const ( func ReadSEDAKeyEncryptionKeyFromEnv() string { keyFile := os.Getenv(SEDAKeyEncryptionKeyFile) if keyFile != "" && cmtos.FileExists(keyFile) { - keyBytes, err := os.ReadFile(keyFile) + keyBytes, err := os.ReadFile(filepath.Clean(keyFile)) if err == nil { return string(keyBytes) } diff --git a/x/batching/types/data_result.go b/x/batching/types/data_result.go index 87110f9c..48e46b70 100644 --- a/x/batching/types/data_result.go +++ b/x/batching/types/data_result.go @@ -32,6 +32,7 @@ func (dr *DataResult) TryHash() (string, error) { blockTimestampBytes := make([]byte, 8) binary.BigEndian.PutUint64(blockTimestampBytes, dr.BlockTimestamp) + //nolint:gosec // G115: Exit code is guaranteed to fit in a byte. exitCodeByte := byte(dr.ExitCode) hasher.Reset()