From 199f0f14ec1ccb54fbdc24214d25301015094115 Mon Sep 17 00:00:00 2001 From: Vadim Macagon Date: Wed, 22 Jul 2020 21:55:13 +0700 Subject: [PATCH 1/5] Add debug find-coin-transfers cmd to loom CLI --- cmd/loom/dbg/cmds.go | 44 +++++++ cmd/loom/dbg/coin_transfers.go | 224 +++++++++++++++++++++++++++++++++ 2 files changed, 268 insertions(+) create mode 100644 cmd/loom/dbg/coin_transfers.go diff --git a/cmd/loom/dbg/cmds.go b/cmd/loom/dbg/cmds.go index cadf3fb0de..b556c78f3f 100644 --- a/cmd/loom/dbg/cmds.go +++ b/cmd/loom/dbg/cmds.go @@ -259,6 +259,49 @@ func newDeleteAppHeightCommand() *cobra.Command { return cmd } +func newFindCoinTransfersCommand() *cobra.Command { + var contracts []string + var startHeight, endHeight int + var recipient, dataPath string + cmd := &cobra.Command{ + Use: "find-coin-transfers", + Short: "Searches the block store for confirmed coin transfer txs", + Example: "find-coin-transfers --data path/to/chaindata --contract default:0xbeef --recipient default:0xfeed", + RunE: func(cmd *cobra.Command, args []string) error { + if len(contracts) == 0 { + return errors.New("At least one coin contract address must be specified") + } + coinContracts := make([]loom.Address, len(contracts)) + for i := range contracts { + var err error + coinContracts[i], err = loom.ParseAddress(contracts[i]) + if err != nil { + return errors.Wrap(err, "failed to parse coin contract address") + } + } + + var recipientAdrr loom.Address + var err error + if recipient != "" { + recipientAdrr, err = loom.ParseAddress(recipient) + if err != nil { + return errors.Wrap(err, "failed to parse recipient address") + } + } + + return findCoinTransfers(coinContracts, dataPath, int64(startHeight), int64(endHeight), &recipientAdrr) + }, + } + cmdFlags := cmd.Flags() + cmdFlags.StringVar(&dataPath, "data", "", "Path to chaindata directory where txs should be read from") + cmdFlags.StringSliceVar(&contracts, "contract", []string{}, "Comma separated list of coin contract addresses") + cmdFlags.IntVar(&startHeight, "start-height", 1, "Block height to start searching from") + cmdFlags.IntVar(&endHeight, "end-height", 0, "Block height to stop searching at (defaults to latest block)") + cmdFlags.StringVar(&recipient, "recipient", "", "Only match transfers to a specific account (optional)") + cmd.MarkFlagRequired("contract") + return cmd +} + // NewDebugCommand creates a new instance of the top-level debug command func NewDebugCommand() *cobra.Command { cmd := &cobra.Command{ @@ -272,6 +315,7 @@ func NewDebugCommand() *cobra.Command { newSetAppHeightCommand(), newGetAppHeightCommand(), newDeleteAppHeightCommand(), + newFindCoinTransfersCommand(), ) return cmd } diff --git a/cmd/loom/dbg/coin_transfers.go b/cmd/loom/dbg/coin_transfers.go new file mode 100644 index 0000000000..90ebf543fa --- /dev/null +++ b/cmd/loom/dbg/coin_transfers.go @@ -0,0 +1,224 @@ +package dbg + +import ( + "encoding/json" + "fmt" + "math/big" + "path" + "time" + + "github.com/gogo/protobuf/proto" + loom "github.com/loomnetwork/go-loom" + ctypes "github.com/loomnetwork/go-loom/builtin/types/coin" + "github.com/loomnetwork/go-loom/plugin" + "github.com/loomnetwork/go-loom/types" + "github.com/loomnetwork/loomchain/auth" + "github.com/loomnetwork/loomchain/eth/utils" + "github.com/loomnetwork/loomchain/vm" + "github.com/pkg/errors" + "github.com/syndtr/goleveldb/leveldb/opt" + abci "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/blockchain" + dbm "github.com/tendermint/tendermint/libs/db" + "github.com/tendermint/tendermint/state/txindex/kv" + tmtypes "github.com/tendermint/tendermint/types" +) + +func findCoinTransfers( + coinContracts []loom.Address, chaindataPath string, startHeight, endHeight int64, recipient *loom.Address, +) error { + var err error + blockStoreDB, err := dbm.NewGoLevelDBWithOpts( + "blockstore", path.Join(chaindataPath, "data"), + &opt.Options{ + ReadOnly: true, + }, + ) + if err != nil { + return errors.New("failed to load block store") + } + defer blockStoreDB.Close() + + txIndexDB, err := dbm.NewGoLevelDBWithOpts( + "tx_index", path.Join(chaindataPath, "data"), + &opt.Options{ + ReadOnly: true, + }, + ) + if err != nil { + return errors.New("failed to load tx index store") + } + defer txIndexDB.Close() + + blockStore := blockchain.NewBlockStore(blockStoreDB) + if startHeight == 0 { + startHeight = 1 + } + if endHeight == 0 { + endHeight = blockchain.LoadBlockStoreStateJSON(blockStoreDB).Height + } + + txIndexer := kv.NewTxIndex(txIndexDB) + + fmt.Printf("Searching from block %v to block %v...\n", startHeight, endHeight) + + var recipientAddr string + if recipient != nil { + recipientAddr = recipient.String() + } + + var totalTxCount, matchingTxCount int + for h := startHeight; h <= endHeight; h++ { + block := blockStore.LoadBlock(h) + if block == nil { + return fmt.Errorf("missing block at height %v", h) + } + if len(block.Data.Txs) > 0 { + for _, tx := range block.Data.Txs { + txr, err := txIndexer.Get(tx.Hash()) + if err != nil { + return err + } + if txr != nil { // means no result was found + // Skip failed txs since they don't modify state, only look at calls to Go contracts + if txr.Result.Code != abci.CodeTypeOK || txr.Result.Info != utils.CallPlugin { + continue + } + } + if info, err := decodeCoinTransferTx(tx, coinContracts); err == nil { + if recipient != nil && info.Sender != recipientAddr { + continue + } + info.Time = block.Header.Time + info.Height = block.Header.Height + output, err := json.MarshalIndent(info, "", " ") + if err != nil { + return err + } + fmt.Printf("%s,\n", string(output)) + matchingTxCount++ + } + } + totalTxCount += len(block.Data.Txs) + } + } + + fmt.Printf("Examined %v txs, found %v matches\n", totalTxCount, matchingTxCount) + return nil +} + +type coinTxInfo struct { + Time time.Time + Height int64 + TxHash string + Sender string + Amount string + Contract string + Method string +} + +func decodeCoinTransferTx(tx tmtypes.Tx, coinContracts []loom.Address) (coinTxInfo, error) { + var def coinTxInfo + var signedTx auth.SignedTx + if err := proto.Unmarshal(tx, &signedTx); err != nil { + return def, errors.Wrap(err, "failed to unmarshal SignedTx") + } + + var nonceTx auth.NonceTx + if err := proto.Unmarshal(signedTx.Inner, &nonceTx); err != nil { + return def, errors.Wrap(err, "failed to unmarshal NonceTx") + } + + var loomTx types.Transaction + if err := proto.Unmarshal(nonceTx.Inner, &loomTx); err != nil { + return def, errors.Wrap(err, "failed to unmarshal Transaction") + } + + var msgTx vm.MessageTx + if err := proto.Unmarshal(loomTx.Data, &msgTx); err != nil { + return def, errors.Wrap(err, "failed to unmarshal MessageTx") + } + + if msgTx.To == nil { + return def, errors.New("MessageTx.To not set") + } + + if msgTx.From == nil { + return def, errors.New("MessageTx.From not set") + } + + var coinAddr loom.Address + for i := range coinContracts { + if coinContracts[i].Compare(loom.UnmarshalAddressPB(msgTx.To)) == 0 { + coinAddr = coinContracts[i] + + } + } + if coinAddr.IsEmpty() { + return def, errors.New("not a coin contract call") + } + + if loomTx.Id != 2 { + return def, errors.New("not a CallTx") + } + + var callTx vm.CallTx + if err := proto.Unmarshal(msgTx.Data, &callTx); err != nil { + return def, errors.Wrap(err, "failed to unmarshal CallTx") + } + + var req plugin.Request + if err := proto.Unmarshal(callTx.Input, &req); err != nil { + return def, errors.Wrap(err, "failed to unmarshal Request") + } + + var methodCall plugin.ContractMethodCall + if err := proto.Unmarshal(req.Body, &methodCall); err != nil { + return def, errors.Wrap(err, "failed to unmarshal ContractMethodCall") + } + + if methodCall.Method == "Transfer" { + var args ctypes.TransferRequest + if err := proto.Unmarshal(methodCall.Args, &args); err != nil { + return def, errors.Wrap(err, "failed to unmarshal TransferRequest") + } + if args.To == nil { + return def, errors.New("TransferRequest.To not set") + } + + amount := new(big.Int) + if args.Amount != nil { + amount = args.Amount.Value.Int + } + + return coinTxInfo{ + TxHash: fmt.Sprintf("%X", tx.Hash()), + Sender: loom.UnmarshalAddressPB(args.To).String(), + Amount: amount.String(), + Contract: coinAddr.String(), + Method: "Transfer", + }, nil + } else if methodCall.Method == "TransferFrom" { + var args ctypes.TransferFromRequest + if err := proto.Unmarshal(methodCall.Args, &args); err != nil { + return def, errors.Wrap(err, "failed to unmarshal TransferFromRequest") + } + if args.From == nil || args.To == nil { + return def, errors.New("TransferFromRequest missing From or To") + } + + amount := new(big.Int) + if args.Amount != nil { + amount = args.Amount.Value.Int + } + + return coinTxInfo{ + TxHash: fmt.Sprintf("%X", tx.Hash()), + Sender: loom.UnmarshalAddressPB(args.To).String(), + Amount: amount.String(), + Contract: coinAddr.String(), + Method: "TransferFrom", + }, nil + } + return def, errors.New("not a coin transfer") +} From 85b947f08df3690bff929790cf477cce15a27669 Mon Sep 17 00:00:00 2001 From: Vadim Macagon Date: Thu, 23 Jul 2020 17:09:06 +0700 Subject: [PATCH 2/5] Fix handling of the optional recipient address --- cmd/loom/dbg/cmds.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/loom/dbg/cmds.go b/cmd/loom/dbg/cmds.go index b556c78f3f..2c6d64297f 100644 --- a/cmd/loom/dbg/cmds.go +++ b/cmd/loom/dbg/cmds.go @@ -280,16 +280,16 @@ func newFindCoinTransfersCommand() *cobra.Command { } } - var recipientAdrr loom.Address - var err error + var recipientAddr *loom.Address if recipient != "" { - recipientAdrr, err = loom.ParseAddress(recipient) + addr, err := loom.ParseAddress(recipient) if err != nil { return errors.Wrap(err, "failed to parse recipient address") } + recipientAddr = &addr } - return findCoinTransfers(coinContracts, dataPath, int64(startHeight), int64(endHeight), &recipientAdrr) + return findCoinTransfers(coinContracts, dataPath, int64(startHeight), int64(endHeight), recipientAddr) }, } cmdFlags := cmd.Flags() From 81ce46b53be8454908120f3984dd9cbd6a200ec3 Mon Sep 17 00:00:00 2001 From: Vadim Macagon Date: Thu, 23 Jul 2020 19:10:34 +0700 Subject: [PATCH 3/5] Skip missing blocks --- cmd/loom/dbg/coin_transfers.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/loom/dbg/coin_transfers.go b/cmd/loom/dbg/coin_transfers.go index 90ebf543fa..543267e44f 100644 --- a/cmd/loom/dbg/coin_transfers.go +++ b/cmd/loom/dbg/coin_transfers.go @@ -71,7 +71,8 @@ func findCoinTransfers( for h := startHeight; h <= endHeight; h++ { block := blockStore.LoadBlock(h) if block == nil { - return fmt.Errorf("missing block at height %v", h) + fmt.Printf("missing block at height %v\n", h) + continue } if len(block.Data.Txs) > 0 { for _, tx := range block.Data.Txs { From f1ef9b0dd8f7155d4d8c51793ee3a881fe2a4473 Mon Sep 17 00:00:00 2001 From: Vadim Macagon Date: Fri, 24 Jul 2020 15:42:42 +0700 Subject: [PATCH 4/5] Log recipient address and indicate when tx results are missing --- cmd/loom/dbg/coin_transfers.go | 53 +++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 20 deletions(-) diff --git a/cmd/loom/dbg/coin_transfers.go b/cmd/loom/dbg/coin_transfers.go index 543267e44f..afbdea6d03 100644 --- a/cmd/loom/dbg/coin_transfers.go +++ b/cmd/loom/dbg/coin_transfers.go @@ -75,19 +75,29 @@ func findCoinTransfers( continue } if len(block.Data.Txs) > 0 { - for _, tx := range block.Data.Txs { + for ti, tx := range block.Data.Txs { txr, err := txIndexer.Get(tx.Hash()) if err != nil { return err } if txr != nil { // means no result was found // Skip failed txs since they don't modify state, only look at calls to Go contracts - if txr.Result.Code != abci.CodeTypeOK || txr.Result.Info != utils.CallPlugin { + if txr.Result.Code != abci.CodeTypeOK { continue } + if txr.Result.Info != utils.CallPlugin { + if txr.Result.Info != "" { + continue + } else { + fmt.Printf("warning: unknown tx type at height %v, index %v\n", h, ti) + } + } + } else { + fmt.Printf("missing tx result at height %v, index %v\n", h, ti) + continue } if info, err := decodeCoinTransferTx(tx, coinContracts); err == nil { - if recipient != nil && info.Sender != recipientAddr { + if recipient != nil && info.Recipient != recipientAddr { continue } info.Time = block.Header.Time @@ -109,13 +119,14 @@ func findCoinTransfers( } type coinTxInfo struct { - Time time.Time - Height int64 - TxHash string - Sender string - Amount string - Contract string - Method string + Time time.Time + Height int64 + TxHash string + Sender string + Recipient string + Amount string + Contract string + Method string } func decodeCoinTransferTx(tx tmtypes.Tx, coinContracts []loom.Address) (coinTxInfo, error) { @@ -193,11 +204,12 @@ func decodeCoinTransferTx(tx tmtypes.Tx, coinContracts []loom.Address) (coinTxIn } return coinTxInfo{ - TxHash: fmt.Sprintf("%X", tx.Hash()), - Sender: loom.UnmarshalAddressPB(args.To).String(), - Amount: amount.String(), - Contract: coinAddr.String(), - Method: "Transfer", + TxHash: fmt.Sprintf("%X", tx.Hash()), + Sender: loom.UnmarshalAddressPB(msgTx.From).String(), + Recipient: loom.UnmarshalAddressPB(args.To).String(), + Amount: amount.String(), + Contract: coinAddr.String(), + Method: "Transfer", }, nil } else if methodCall.Method == "TransferFrom" { var args ctypes.TransferFromRequest @@ -214,11 +226,12 @@ func decodeCoinTransferTx(tx tmtypes.Tx, coinContracts []loom.Address) (coinTxIn } return coinTxInfo{ - TxHash: fmt.Sprintf("%X", tx.Hash()), - Sender: loom.UnmarshalAddressPB(args.To).String(), - Amount: amount.String(), - Contract: coinAddr.String(), - Method: "TransferFrom", + TxHash: fmt.Sprintf("%X", tx.Hash()), + Sender: loom.UnmarshalAddressPB(args.From).String(), + Recipient: loom.UnmarshalAddressPB(args.To).String(), + Amount: amount.String(), + Contract: coinAddr.String(), + Method: "TransferFrom", }, nil } return def, errors.New("not a coin transfer") From 54c4f3c5469b57bfd0259670d9fa306e04a9d4df Mon Sep 17 00:00:00 2001 From: Vadim Macagon Date: Fri, 24 Jul 2020 16:48:51 +0700 Subject: [PATCH 5/5] Load tx results from state DB instead of tx index DB The tx index DB may be missing some old data due to past migrations. --- cmd/loom/dbg/coin_transfers.go | 52 ++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/cmd/loom/dbg/coin_transfers.go b/cmd/loom/dbg/coin_transfers.go index afbdea6d03..060906601f 100644 --- a/cmd/loom/dbg/coin_transfers.go +++ b/cmd/loom/dbg/coin_transfers.go @@ -20,7 +20,9 @@ import ( abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/blockchain" dbm "github.com/tendermint/tendermint/libs/db" - "github.com/tendermint/tendermint/state/txindex/kv" + + //"github.com/tendermint/tendermint/state/txindex/kv" + sm "github.com/tendermint/tendermint/state" tmtypes "github.com/tendermint/tendermint/types" ) @@ -39,16 +41,28 @@ func findCoinTransfers( } defer blockStoreDB.Close() - txIndexDB, err := dbm.NewGoLevelDBWithOpts( - "tx_index", path.Join(chaindataPath, "data"), + stateDB, err := dbm.NewGoLevelDBWithOpts( + "state", path.Join(chaindataPath, "data"), &opt.Options{ ReadOnly: true, }, ) if err != nil { - return errors.New("failed to load tx index store") + return errors.New("failed to load state store") } - defer txIndexDB.Close() + defer stateDB.Close() + /* + txIndexDB, err := dbm.NewGoLevelDBWithOpts( + "tx_index", path.Join(chaindataPath, "data"), + &opt.Options{ + ReadOnly: true, + }, + ) + if err != nil { + return errors.New("failed to load tx index store") + } + defer txIndexDB.Close() + */ blockStore := blockchain.NewBlockStore(blockStoreDB) if startHeight == 0 { @@ -58,7 +72,7 @@ func findCoinTransfers( endHeight = blockchain.LoadBlockStoreStateJSON(blockStoreDB).Height } - txIndexer := kv.NewTxIndex(txIndexDB) + //txIndexer := kv.NewTxIndex(txIndexDB) fmt.Printf("Searching from block %v to block %v...\n", startHeight, endHeight) @@ -75,26 +89,34 @@ func findCoinTransfers( continue } if len(block.Data.Txs) > 0 { + blockResults, err := sm.LoadABCIResponses(stateDB, h) + if err != nil { + fmt.Println("failed to load block results, err:", err) + continue + } for ti, tx := range block.Data.Txs { - txr, err := txIndexer.Get(tx.Hash()) - if err != nil { - return err - } + /* + txr, err := txIndexer.Get(tx.Hash()) + if err != nil { + return err + } + */ + txr := blockResults.DeliverTx[ti] if txr != nil { // means no result was found // Skip failed txs since they don't modify state, only look at calls to Go contracts - if txr.Result.Code != abci.CodeTypeOK { + if txr.Code != abci.CodeTypeOK { continue } - if txr.Result.Info != utils.CallPlugin { - if txr.Result.Info != "" { + if txr.Info != utils.CallPlugin { + if txr.Info != "" { continue } else { fmt.Printf("warning: unknown tx type at height %v, index %v\n", h, ti) } } } else { - fmt.Printf("missing tx result at height %v, index %v\n", h, ti) - continue + // probably means the tx was successful, but no other output was saved for it + fmt.Printf("warning: missing tx result at height %v, index %v\n", h, ti) } if info, err := decodeCoinTransferTx(tx, coinContracts); err == nil { if recipient != nil && info.Recipient != recipientAddr {