diff --git a/.github/ISSUE_TEMPLATE/ask-a-question.md b/.github/ISSUE_TEMPLATE/ask-a-question.md index 1bb285b1..f2017630 100644 --- a/.github/ISSUE_TEMPLATE/ask-a-question.md +++ b/.github/ISSUE_TEMPLATE/ask-a-question.md @@ -1,9 +1,39 @@ --- name: Ask a question -about: Something is unclear -title: '' +about: Something is unclear or needs clarification +title: '[Question] ' labels: 'type:docs' assignees: '' --- -If you are not 100% sure if something is a bug or asking a question that leads to improving the documentation. For general questions please use [Discord](https://discord.gg/cGKSsRVCGm) or [Telegram](https://t.me/TronOfficialDevelopersGroupEn). + + + +## Question + + + +## Context + + + +**What are you trying to achieve?** + + +**What have you tried so far?** + + +**Relevant documentation or code** + + +## Additional Information + + diff --git a/.github/ISSUE_TEMPLATE/report-a-bug.md b/.github/ISSUE_TEMPLATE/report-a-bug.md index c5c62d06..197a2637 100644 --- a/.github/ISSUE_TEMPLATE/report-a-bug.md +++ b/.github/ISSUE_TEMPLATE/report-a-bug.md @@ -1,7 +1,7 @@ --- name: Report a bug about: Create a report to help us improve -title: '' +title: '[Bug] ' labels: 'type:bug' assignees: '' @@ -9,42 +9,51 @@ assignees: '' - + -#### Software Versions - - +## Expected behaviour -#### Expected behaviour +## Actual behaviour -#### Actual behaviour - + +## Frequency -#### Frequency +- [ ] Always (100%) +- [ ] Frequently (>50%) +- [ ] Sometimes (10-50%) +- [ ] Rarely (<10%) + +## Steps to reproduce the behaviour + + +1. +2. +3. + +## Logs and Error Messages + + + +``` +[Paste error messages, stack traces, or relevant logs here] +``` -#### Steps to reproduce the behaviour +## Additional Context -1. [Step 1] -2. [Step 2] -3. [Step ...] + -#### Backtrace +**Screenshots** + -```` -[backtrace] -```` +**Related Issues** + -When submitting logs: please submit them as text and not screenshots. +**Possible Solution** + diff --git a/.github/ISSUE_TEMPLATE/request-a-feature.md b/.github/ISSUE_TEMPLATE/request-a-feature.md index 261f1088..ba1ff76e 100644 --- a/.github/ISSUE_TEMPLATE/request-a-feature.md +++ b/.github/ISSUE_TEMPLATE/request-a-feature.md @@ -1,28 +1,86 @@ --- name: Request a feature about: Suggest an idea for this project -title: '' +title: '[Feature] ' labels: 'type:feature' assignees: '' --- -# Background -# Rationale +## Background -Why should this feature exist? + -What are the use-cases? +## Problem Statement -# Specification + -# Test Specification +## Rationale -# Scope Of Impact +**Why should this feature exist?** + +**What are the use cases?** + -# Implementation +1. +2. +3. -Do you have ideas regarding the implementation of this feature? +## Proposed Solution -Are you willing to implement this feature? + + +### Specification + + + +## Test Specification + + + +**Test Scenarios** + +1. +2. +3. + +**Performance Considerations** + + +## Scope Of Impact + + + +## Implementation + +**Do you have ideas regarding the implementation?** + + +**Are you willing to implement this feature?** + +- [ ] Yes, I can implement this feature +- [ ] I can help with implementation +- [ ] I need help with implementation +- [ ] I'm just suggesting the idea + +**Estimated Complexity** + +- [ ] Low (minor changes) +- [ ] Medium (moderate changes) +- [ ] High (significant changes) +- [ ] Unknown + +## Alternatives Considered + + + +## Additional Context + + + +**Related Issues/PRs** + + +**References** + diff --git a/.github/workflows/docker-s3-deploy.yml b/.github/workflows/docker-s3-deploy.yml index dbca6b11..7eab1969 100644 --- a/.github/workflows/docker-s3-deploy.yml +++ b/.github/workflows/docker-s3-deploy.yml @@ -36,26 +36,47 @@ jobs: exit 1 fi - - name: Import GPG key - uses: crazy-max/ghaction-import-gpg@v6 - with: - gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} - passphrase: ${{ secrets.GPG_PASSPHRASE }} + - name: Install cosign + uses: sigstore/cosign-installer@v3 - - name: Sign tron-docker.zip with GPG + - name: Sign tron-docker.zip with Sigstore cosign (keyless) + # Keyless signing using GitHub OIDC — no private keys to manage or leak. + # The signature is bound to this workflow's identity (repo, ref, commit SHA). + # Verification: cosign verify-blob --certificate tron-docker.zip.cert \ + # --signature tron-docker.zip.sig \ + # --certificate-identity-regexp "https://github.com/tronprotocol/tron-docker" \ + # --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \ + # tron-docker.zip run: | - gpg --detach-sign --armor tron-docker.zip - # This creates tron-docker.zip.asc (ASCII-armored signature) + cosign sign-blob tron-docker.zip \ + --yes \ + --output-signature tron-docker.zip.sig \ + --output-certificate tron-docker.zip.cert - - name: Configure AWS Credentials + - name: Configure AWS Credentials (OIDC — no long-lived keys) uses: aws-actions/configure-aws-credentials@v4 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ github.event.inputs.aws-region }} # Use input for region + # Uses GitHub OIDC provider to assume an IAM role with short-lived credentials. + # No static keys needed — credentials expire after the workflow run. + # Prerequisites: + # 1. Create an IAM OIDC identity provider for token.actions.githubusercontent.com + # 2. Create an IAM role with trust policy allowing this repo: + # "Condition": { + # "StringEquals": { + # "token.actions.githubusercontent.com:aud": "sts.amazonaws.com", + # "token.actions.githubusercontent.com:sub": "repo:tronprotocol/tron-docker:ref:refs/heads/main" + # } + # } + # 3. Attach S3 put-object policy scoped to the target bucket only + # 4. Delete the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY secrets from GitHub + role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} + aws-region: ${{ github.event.inputs.aws-region }} - name: Upload tron-docker.zip to S3 + env: + BUCKET_NAME: ${{ github.event.inputs.bucket-name }} + VERSION: ${{ github.event.inputs.version }} run: | - zip -r publish.zip tron-docker.zip tron-docker.zip.asc - aws s3 cp publish.zip s3://${{ github.event.inputs.bucket-name }}/package/publish-latest.zip - aws s3 cp publish.zip s3://${{ github.event.inputs.bucket-name }}/package/publish-${{github.event.inputs.version}}.zip + zip -r publish.zip tron-docker.zip tron-docker.zip.sig tron-docker.zip.cert + aws s3 cp publish.zip "s3://${BUCKET_NAME}/package/publish-latest.zip" + aws s3 cp publish.zip "s3://${BUCKET_NAME}/package/publish-${VERSION}.zip" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4e348419..0a2a59ab 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,14 +6,14 @@ Here are some guidelines to get started quickly and easily: - [Contributing](#contributing) - [Before contribution](#before-contribution) - [Ask a question](#ask-a-question) - - [Reporting an issue](#reporting-an-issue) + - [Report a bug](#report-a-bug) - [Request a feature](#request-a-feature) - [Contribute to tron-docker](#contribute-to-tron-docker) - [Key branches](#key-branches) - [Submitting code steps](#submitting-code-steps) - - [Fork then make changes](#fork-then-make-changes) + - [Fork and make changes](#fork-and-make-changes) - [Linting](#linting) - - [Push code](#push-code) + - [Submit a pull request](#submit-a-pull-request) - [Commit messages](#commit-messages) - [Branch naming conventions](#branch-naming-conventions) - [Pull request best practise](#pull-request-best-practise) @@ -22,23 +22,20 @@ Here are some guidelines to get started quickly and easily: ## Before contribution -Before making any modifications to tron-docker, it's advisable to discuss your issues or suggestions with the community. They may already be working on similar topics or have them in their roadmap, which can avoid duplicating efforts. +If you have any question about tron-docker, please search [existing issues](https://github.com/tronprotocol/tron-docker/issues?q=is%3Aissue%20state%3Aclosed%20OR%20state%3Aopen) to avoid duplicates. Your questions might already be under discussion or part of our roadmap. Checking first helps us streamline efforts and focus on new contributions. ### Ask a question -Feel free to ask any TRON related question to solve your doubt. Please click **Ask a question** in GitHub Issues, then follow the instructions thereafter. +Feel free to ask any tron-docker related question to solve your doubt. Please click **Ask a question** in GitHub Issues, using [Ask a question](.github/ISSUE_TEMPLATE/ask-a-question.md) template. ![image](images/github_issue_ask_a_question.png) -### Reporting an issue +### Report a bug -If you're about to raise an issue because you think you've found a problem or bug with tron-docker, please respect the following restrictions: - -- Please search for [existing issues](https://github.com/tronprotocol/tron-docker/issues?q=is%3Aissue%20state%3Aclosed%20OR%20state%3Aopen). Help us keep duplicate issues to a minimum by checking to see if someone has already reported your problem or requested your idea. -- Use the [Report a bug](.github/ISSUE_TEMPLATE/report-a-bug.md) template. +If you think you've found a bug with tron-docker, please click **Report a bug** in GitHub Issues, using [Report a bug](.github/ISSUE_TEMPLATE/report-a-bug.md) template. ### Request a feature -If you have any good feature suggestions for tron-docker, community would be very welcome to hear them. Please refer to our [Request a feature](.github/ISSUE_TEMPLATE/request-a-feature.md) template. +If you have any good feature suggestions for tron-docker, please click **Request a feature** in GitHub Issues, using [Request a feature](.github/ISSUE_TEMPLATE/request-a-feature.md) template. ## Contribute to tron-docker @@ -68,42 +65,57 @@ tron-docker only has `main`, `develop`, `release-*`, `feature-*`, and `hotfix-*` ### Submitting code steps -If you want to contribute codes to tron-docker, please follow the following steps. +If you want to contribute code to tron-docker, please follow the following steps. -#### Fork then make changes -Fork a new repository from tronprotocol/tron-docker to your personal code repository, and then edit the code in the your fork repository. -``` - git clone https://github.com/yourname/tron-docker.git - git remote add upstream https://github.com/tronprotocol/tron-docker.git ("upstream" refers to upstream projects repositories, namely tronprotocol's repositories, and can be named as you like it. We usually call it "upstream" for convenience) -``` +#### Fork and make changes -Before developing new features, please synchronize your fork repository with the upstream repository. -``` - git fetch upstream - git checkout develop - git merge upstream/develop --no-ff (Add --no-ff to turn off the default fast merge mode) -``` +* Fork the Repository -Pull a new branch from the **develop** branch of your repository for local development. Please refer to [Branch Naming Conventions](#Branch-Naming-Conventions). -``` - git checkout -b feature/branch_name develop -``` + Visit [tronprotocol/tron-docker](https://github.com/tronprotocol/tron-docker/) and click **Fork** to create a fork repository under your GitHub account. -Write and commit the new code when it is completed. Please refer to [Commit Messages](#Commit-Messages). -``` - git add . - git commit -m 'commit message' -``` -Commit the new branch to your personal remote repository. -``` - git push origin feature/branch_name -``` +* Setup Local Environment + + Clone your fork repository to local and add the official repository as **upstream**. + ``` + git clone https://github.com/yourname/tron-docker.git + + cd tron-docker + + git remote add upstream https://github.com/tronprotocol/tron-docker.git + ``` + +* Synchronize and Develop + + Before developing new features, fetch the latest changes from the upstream repository, create a local `develop` branch tracking the upstream's version, and push it to your fork to ensure all environments are aligned. + ``` + git fetch upstream + git checkout -b develop upstream/develop + git push origin develop + ``` + + Create a new branch for development. Please refer to [Branch Naming Conventions](#Branch-Naming-Conventions). + ``` + git checkout -b feature/branch_name develop + ``` + +* Commit and Push + + Write and commit the new code when it is completed. Please refer to [Commit Messages](#Commit-Messages). + ``` + git add . + git commit -m 'commit message' + ``` + + Push the new branch to your fork repository + ``` + git push origin feature/branch_name + ``` #### Linting tron-docker CI uses [pre-commit](https://pre-commit.com/) to lint all code within the repo. Add it to your local following the [installation](https://pre-commit.com/#installation). And ensure ShellCheck is also installed, refer to [installation](https://github.com/koalaman/shellcheck?tab=readme-ov-file#installing). Check [.pre-commit-config.yaml](.pre-commit-config-fix.yaml) for existing validators. -#### Push code +#### Submit a pull request Submit a pull request (PR) from your repository to `tronprotocol/tron-docker`. Please be sure to click on the link in the red box shown below. Select the base branch for tron-docker and the compare branch for your personal fork repository. diff --git a/README.md b/README.md index ca8eb78a..2e5b0d76 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,7 @@ To start all available features, or you want more customized operations, navigat - **Gradle Docker**: Automate Docker image builds and testing. Check the [gradle docker](./tools/docker/README.md) documentation. - **Toolkit**: Perform a set of database related operations. Follow the [Toolkit guidance](./tools/toolkit/README.md). - **Stress Test**: Execute the stress test. Follow the [stress test guidance](./tools/stress_test/README.md). + - **Slack SR Monitor**: Monitor Super Representatives and notify a Slack channel after every maintenance period. Follow the [Slack SR Monitor guidance](./tools/slack_sr_monitor/README.md). ## Troubleshooting If you encounter any difficulties, please refer to the [Issue Work Flow](https://tronprotocol.github.io/documentation-en/developers/issue-workflow/#issue-work-flow), then raise an issue on [GitHub](https://github.com/tronprotocol/tron-docker/issues). For general questions, please use [Discord](https://discord.gg/cGKSsRVCGm) or [Telegram](https://t.me/TronOfficialDevelopersGroupEn). diff --git a/build_trond.sh b/build_trond.sh index 6f8fae8e..183e3405 100755 --- a/build_trond.sh +++ b/build_trond.sh @@ -6,15 +6,30 @@ ARCH=$(uname -m) # Set Go version to 1.23.6 GO_VERSION="1.23.6" +GO_SHA256="" + +calculate_sha256() { + local file="$1" + if command -v sha256sum &> /dev/null; then + sha256sum "$file" | awk '{print $1}' + elif command -v shasum &> /dev/null; then + shasum -a 256 "$file" | awk '{print $1}' + else + echo "No SHA-256 checksum tool found (sha256sum or shasum)." + exit 1 + fi +} # Determine download URL and archive filename based on OS and ARCH if [[ "$OS" == "Linux" ]]; then if [[ "$ARCH" == "x86_64" ]]; then GO_ARCHIVE="go$GO_VERSION.linux-amd64.tar.gz" GO_URL="https://go.dev/dl/$GO_ARCHIVE" + GO_SHA256="9379441ea310de000f33a4dc767bd966e72ab2826270e038e78b2c53c2e7802d" elif [[ "$ARCH" == "arm64" || "$ARCH" == "aarch64" ]]; then GO_ARCHIVE="go$GO_VERSION.linux-arm64.tar.gz" GO_URL="https://go.dev/dl/$GO_ARCHIVE" + GO_SHA256="561c780e8f4a8955d32bf72e46af0b5ee5e0debe1e4633df9a03781878219202" else echo "Unsupported architecture: $ARCH" exit 1 @@ -23,9 +38,11 @@ elif [[ "$OS" == "Darwin" ]]; then if [[ "$ARCH" == "x86_64" ]]; then GO_ARCHIVE="go$GO_VERSION.darwin-amd64.tar.gz" GO_URL="https://go.dev/dl/$GO_ARCHIVE" + GO_SHA256="782da50ce8ec5e98fac2cd3cdc6a1d7130d093294fc310038f651444232a3fb0" elif [[ "$ARCH" == "arm64" ]]; then GO_ARCHIVE="go$GO_VERSION.darwin-arm64.tar.gz" GO_URL="https://go.dev/dl/$GO_ARCHIVE" + GO_SHA256="5cae2450a1708aeb0333237a155640d5562abaf195defebc4306054565536221" else echo "Unsupported architecture: $ARCH" exit 1 @@ -106,8 +123,20 @@ if [[ "$SYSTEM_GO" == false ]]; then echo "go/$GO_ARCHIVE already exists. Skipping download." else echo "Downloading Go from $GO_URL..." - curl -Lo "go/$GO_ARCHIVE" "$GO_URL" + curl -fL -o "go/$GO_ARCHIVE" "$GO_URL" + fi + + # Verify the downloaded archive checksum + echo "Verifying SHA-256 for go/$GO_ARCHIVE..." + ACTUAL_SHA256="$(calculate_sha256 "go/$GO_ARCHIVE")" + if [[ "$ACTUAL_SHA256" != "$GO_SHA256" ]]; then + echo "SHA-256 mismatch for go/$GO_ARCHIVE" + echo "Expected: $GO_SHA256" + echo "Actual: $ACTUAL_SHA256" + rm -f "go/$GO_ARCHIVE" + exit 1 fi + echo "SHA-256 verification passed." # Extract Golang to the go directory if [[ -d "go/bin" ]]; then diff --git a/conf/main_net_config.conf b/conf/main_net_config.conf index f0fb74c8..069b5128 100644 --- a/conf/main_net_config.conf +++ b/conf/main_net_config.conf @@ -1,85 +1,78 @@ +net { + # type is deprecated and has no effect. + # type = mainnet +} + storage { # Directory for storing persistent data - db.engine = "LEVELDB", + db.engine = "LEVELDB", // deprecated for arm, because arm only support "ROCKSDB". db.sync = false, db.directory = "database", - index.directory = "index", - transHistory.switch = "on", - # You can custom these 14 databases' configs: - # account, account-index, asset-issue, block, block-index, - # block_KDB, peers, properties, recent-block, trans, - # utxo, votes, witness, witness_schedule. - - # Otherwise, db configs will remain default and data will be stored in - # the path of "output-directory" or which is set by "-d" ("--output-directory"). + # Whether to write transaction result in transactionRetStore + transHistory.switch = "on", - # setting can impove leveldb performance .... start + # setting can improve leveldb performance .... start, deprecated for arm # node: if this will increase process fds,you may be check your ulimit if 'too many open files' error occurs # see https://github.com/tronprotocol/tips/blob/master/tip-343.md for detail - # if you find block sync has lower performance,you can try this settings - #default = { + # if you find block sync has lower performance, you can try this settings + # default = { # maxOpenFiles = 100 - #} - #defaultM = { + # } + # defaultM = { # maxOpenFiles = 500 - #} - #defaultL = { + # } + # defaultL = { # maxOpenFiles = 1000 - #} - # setting can impove leveldb performance .... end + # } + # setting can improve leveldb performance .... end, deprecated for arm - # Attention: name is a required field that must be set !!! + # You can customize the configuration for each database. Otherwise, the database settings will use + # their defaults, and data will be stored in the "output-directory" or in the directory specified + # by the "-d" or "--output-directory" option. Attention: name is a required field that must be set! + # In this configuration, the name and path properties take effect for both LevelDB and RocksDB storage engines, + # while the additional properties (such as createIfMissing, paranoidChecks, compressionType, etc.) only take effect when using LevelDB. properties = [ - // { - // name = "account", - // path = "storage_directory_test", - // createIfMissing = true, - // paranoidChecks = true, - // verifyChecksums = true, - // compressionType = 1, // compressed with snappy - // blockSize = 4096, // 4 KB = 4 * 1024 B - // writeBufferSize = 10485760, // 10 MB = 10 * 1024 * 1024 B - // cacheSize = 10485760, // 10 MB = 10 * 1024 * 1024 B - // maxOpenFiles = 100 - // }, - // { - // name = "account-index", - // path = "storage_directory_test", - // createIfMissing = true, - // paranoidChecks = true, - // verifyChecksums = true, - // compressionType = 1, // compressed with snappy - // blockSize = 4096, // 4 KB = 4 * 1024 B - // writeBufferSize = 10485760, // 10 MB = 10 * 1024 * 1024 B - // cacheSize = 10485760, // 10 MB = 10 * 1024 * 1024 B - // maxOpenFiles = 100 - // }, + # { + # name = "account", + # path = "storage_directory_test", + # createIfMissing = true, // deprecated for arm start + # paranoidChecks = true, + # verifyChecksums = true, + # compressionType = 1, // compressed with snappy + # blockSize = 4096, // 4 KB = 4 * 1024 B + # writeBufferSize = 10485760, // 10 MB = 10 * 1024 * 1024 B + # cacheSize = 10485760, // 10 MB = 10 * 1024 * 1024 B + # maxOpenFiles = 100 // deprecated for arm end + # }, + # { + # name = "account-index", + # path = "storage_directory_test", + # createIfMissing = true, + # paranoidChecks = true, + # verifyChecksums = true, + # compressionType = 1, // compressed with snappy + # blockSize = 4096, // 4 KB = 4 * 1024 B + # writeBufferSize = 10485760, // 10 MB = 10 * 1024 * 1024 B + # cacheSize = 10485760, // 10 MB = 10 * 1024 * 1024 B + # maxOpenFiles = 100 + # }, ] needToUpdateAsset = true - //dbsettings is needed when using rocksdb as the storage implement (db.engine="ROCKSDB"). - //we'd strongly recommend that do not modify it unless you know every item's meaning clearly. + # dbsettings is needed when using rocksdb as the storage implement (db.engine="ROCKSDB"). + # we'd strongly recommend that do not modify it unless you know every item's meaning clearly. dbSettings = { levelNumber = 7 - //compactThreads = 32 + # compactThreads = 32 blocksize = 64 // n * KB maxBytesForLevelBase = 256 // n * MB maxBytesForLevelMultiplier = 10 level0FileNumCompactionTrigger = 4 targetFileSizeBase = 256 // n * MB targetFileSizeMultiplier = 1 - } - - //backup settings when using rocks db as the storage implement (db.engine="ROCKSDB"). - //if you want to use the backup plugin, please confirm set the db.engine="ROCKSDB" above. - backup = { - enable = false // indicate whether enable the backup plugin - propPath = "prop.properties" // record which bak directory is valid - bak1path = "bak1/database" // you must set two backup directories to prevent application halt unexpected(e.g. kill -9). - bak2path = "bak2/database" - frequency = 10000 // indicate backup db once every 10000 blocks processed. + maxOpenFiles = 5000 } balance.history.lookup = false @@ -90,11 +83,14 @@ storage { # the estimated number of block transactions (default 1000, min 100, max 10000). # so the total number of cached transactions is 65536 * txCache.estimatedTransactions # txCache.estimatedTransactions = 1000 - # if true, transaction cache initialization will be faster. default false - # txCache.initOptimization = true - # data root setting, for check data, currently, only reward-vi is used. + # if true, transaction cache initialization will be faster. Default: false + txCache.initOptimization = true + + # The number of blocks flushed to db in each batch during node syncing. Default: 1 + # snapshot.maxFlushCount = 1 + # data root setting, for check data, currently, only reward-vi is used. # merkleRoot = { # reward-vi = 9debcb9924055500aaae98cdee10501c5c39d4daa75800a996f4bdda73dbccd8 // main-net, Sha256Hash, hexString # } @@ -130,18 +126,27 @@ node.backup { ] } +# Specify the algorithm for generating a public key from private key. To avoid forks, please do not modify it crypto { engine = "eckey" } -# prometheus metrics start -# node.metrics = { -# prometheus{ -# enable=true -# port="9527" -# } -# } -# prometheus metrics end +node.metrics = { + # prometheus metrics + prometheus { + enable = false + port = 9527 + } + + # influxdb metrics + storageEnable = false # Whether write metrics data into InfluxDb. Default: false. + influxdb { + ip = "" + port = 8086 + database = "" + metricsReportInterval = 10 + } +} node { # trust node for solidity node @@ -156,10 +161,7 @@ node { connection.timeout = 2 fetchBlock.timeout = 200 - - tcpNettyWorkThreadNum = 0 - - udpNettyWorkThreadNum = 1 + # syncFetchBatchNum = 2000 # Number of validate sign thread, default availableProcessors # validateSignThreadNum = 16 @@ -176,11 +178,23 @@ node { minParticipationRate = 15 + # allowShieldedTransactionApi = true + + # openPrintLog = true + + # If set to true, SR packs transactions into a block in descending order of fee; otherwise, it packs + # them based on their receive timestamp. Default: false + # openTransactionSort = false + + # The threshold for the number of broadcast transactions received from each peer every second, + # transactions exceeding this threshold will be discarded + # maxTps = 1000 + isOpenFullTcpDisconnect = false inactiveThreshold = 600 //seconds p2p { - version = 11111 # mainnet:11111; nile testnet:201910292 + version = 11111 # Mainnet:11111; Nile:201910292; Shasta:1 } active = [ @@ -240,17 +254,23 @@ node { # The maximum size of header list allowed to be received, default 8192 # maxHeaderListSize = + # The number of RST_STREAM frames allowed to be sent per connection per period for grpc, by default there is no limit. + # maxRstStream = + + # The number of seconds per period for grpc + # secondsPerWindow = + # Transactions can only be broadcast if the number of effective connections is reached. minEffectiveConnection = 1 - # The switch of the reflection service, effective for all gRPC services - # reflectionService = true + # The switch of the reflection service, effective for all gRPC services, used for grpcurl tool. Default: false + reflectionService = false } # number of solidity thread in the FullNode. # If accessing solidity rpc and http interface timeout, could increase the number of threads, # The default value is the number of cpu cores of the machine. - #solidity.threads = 8 + # solidity.threads = 8 # Limits the maximum percentage (default 75%) of producing block interval # to provide sufficient time to perform other operations e.g. broadcast block @@ -259,21 +279,24 @@ node { # Limits the maximum number (default 700) of transaction from network layer # netMaxTrxPerSecond = 700 - # Whether to enable the node detection function, default false + # Whether to enable the node detection function. Default: false # nodeDetectEnable = false - # use your ipv6 address for node discovery and tcp connection, default false + # use your ipv6 address for node discovery and tcp connection. Default: false # enableIpv6 = false - # if your node's highest block num is below than all your pees', try to acquire new connection. default false + # if your node's highest block num is below than all your pees', try to acquire new connection. Default: false # effectiveCheckEnable = false # Dynamic loading configuration function, disabled by default - # dynamicConfig = { - # enable = false - # Configuration file change check interval, default is 600 seconds - # checkInterval = 600 - # } + dynamicConfig = { + # enable = false + # checkInterval = 600 // Check interval of Configuration file's change, default is 600 seconds + } + + # Whether to continue broadcast transactions after at least maxUnsolidifiedBlocks are not solidified. Default: false + # unsolidifiedBlockCheck = false + # maxUnsolidifiedBlocks = 54 dns { # dns urls to get nodes, url format tree://{pubkey}@{domain}, default empty @@ -281,7 +304,7 @@ node { #"tree://AKMQMNAJJBL73LXWPXDI4I5ZWWIZ4AWO34DWQ636QOBBXNFXH3LQS@main.trondisco.net", ] - # enable or disable dns publish, default false + # enable or disable dns publish. Default: false # publish = false # dns domain to publish nodes, required if publish is true @@ -327,22 +350,20 @@ node { # awsHostZoneId = "your-host-zone-id" } - # open the history query APIs(http&GRPC) when node is a lite fullNode, - # like {getBlockByNum, getBlockByID, getTransactionByID...}. - # default: false. + # open the history query APIs(http&GRPC) when node is a lite FullNode, + # like {getBlockByNum, getBlockByID, getTransactionByID...}. Default: false. # note: above APIs may return null even if blocks and transactions actually are on the blockchain - # when opening on a lite fullnode. only open it if the consequences being clearly known + # when opening on a lite FullNode. only open it if the consequences being clearly known # openHistoryQueryWhenLiteFN = false jsonrpc { - # Note: If you turn on jsonrpc and run it for a while and then turn it off, you will not - # be able to get the data from eth_getLogs for that period of time. - - # httpFullNodeEnable = true + # Note: Before release_4.8.1, if you turn on jsonrpc and run it for a while and then turn it off, + # you will not be able to get the data from eth_getLogs for that period of time. Default: false + # httpFullNodeEnable = false # httpFullNodePort = 8545 - # httpSolidityEnable = true + # httpSolidityEnable = false # httpSolidityPort = 8555 - # httpPBFTEnable = true + # httpPBFTEnable = false # httpPBFTPort = 8565 # The maximum blocks range to retrieve logs for eth_getLogs, default value is 5000, @@ -352,27 +373,26 @@ node { # The maximum number of allowed topics within a topic criteria, default value is 1000, # should be > 0, otherwise means no limit. maxSubTopics = 1000 + # Allowed maximum number for blockFilter + maxBlockFilterNum = 50000 } - # Disabled api list, it will work for http, rpc and pbft, both fullnode and soliditynode, - # but not jsonrpc. - # Sample: The setting is case insensitive, GetNowBlock2 is equal to getnowblock2 - # - # disabledApi = [ - # "getaccount", - # "getnowblock2" - # ] + # Disabled api list, it will work for http, rpc and pbft, both FullNode and SolidityNode, + # but not jsonrpc. The setting is case insensitive, GetNowBlock2 is equal to getnowblock2 + disabledApi = [ + # "getaccount", + # "getnowblock2" + ] } ## rate limiter config rate.limiter = { - # Every api could be set a specific rate limit strategy. Three strategy are supported:GlobalPreemptibleAdapter、IPQPSRateLimiterAdapte、QpsRateLimiterAdapter - # GlobalPreemptibleAdapter: permit is the number of preemptible resource, every client must apply one resourse - # before do the request and release the resource after got the reponse automaticlly. permit should be a Integer. + # Every api could only set a specific rate limit strategy. Three blocking strategy are supported: + # GlobalPreemptibleAdapter: The number of preemptible resource or maximum concurrent requests globally. # QpsRateLimiterAdapter: qps is the average request count in one second supported by the server, it could be a Double or a Integer. # IPQPSRateLimiterAdapter: similar to the QpsRateLimiterAdapter, qps could be a Double or a Integer. - # If do not set, the "default strategy" is set.The "default startegy" is based on QpsRateLimiterAdapter, the qps is set as 10000. + # If not set, QpsRateLimiterAdapter with qps=1000 is the default strategy. # # Sample entries: # @@ -416,10 +436,16 @@ rate.limiter = { # }, ] + p2p = { + # syncBlockChain = 3.0 + # fetchInvData = 3.0 + # disconnect = 1.0 + } + # global qps, default 50000 - # global.qps = 50000 + global.qps = 50000 # IP-based global qps, default 10000 - # global.ip.qps = 10000 + global.ip.qps = 10000 } @@ -434,17 +460,17 @@ seed.node = { # ] ip.list = [ "3.225.171.164:18888", - "52.53.189.99:18888", - "18.196.99.16:18888", - "34.253.187.192:18888", + "52.8.46.215:18888", + "3.79.71.167:18888", + "108.128.110.16:18888", "18.133.82.227:18888", - "35.180.51.163:18888", - "54.252.224.209:18888", + "35.180.81.133:18888", + "13.210.151.5:18888", "18.231.27.82:18888", - "52.15.93.92:18888", - "34.220.77.106:18888", + "3.12.212.122:18888", + "52.24.128.7:18888", "15.207.144.3:18888", - "13.124.62.58:18888", + "3.39.38.55:18888", "54.151.226.240:18888", "35.174.93.198:18888", "18.210.241.149:18888", @@ -629,34 +655,33 @@ genesis.block = { } ] - timestamp = "0" #2017-8-26 12:00:00 + timestamp = "0" # Genesis block timestamp, milli seconds parentHash = "0xe58f33f9baf9305dc6f82b9f1934ea8f0ade2defb951258d50167028c780351f" } -// Optional.The default is empty. -// It is used when the witness account has set the witnessPermission. -// When it is not empty, the localWitnessAccountAddress represents the address of the witness account, -// and the localwitness is configured with the private key of the witnessPermissionAddress in the witness account. -// When it is empty,the localwitness is configured with the private key of the witness account. - -//localWitnessAccountAddress = +# Optional. The default is empty. It is used when the witness account has set the witnessPermission. +# When it is not empty, the localWitnessAccountAddress represents the address of the witness account, +# and the localwitness is configured with the private key of the witnessPermissionAddress in the witness account. +# When it is empty,the localwitness is configured with the private key of the witness account. +# localWitnessAccountAddress = localwitness = [ ] -#localwitnesskeystore = [ +# localwitnesskeystore = [ # "localwitnesskeystore.json" -#] +# ] block = { needSyncCheck = true - maintenanceTimeInterval = 21600000 - proposalExpireTime = 259200000 // 3 day: 259200000(ms) + maintenanceTimeInterval = 21600000 // 6 hours: 21600000(ms) + proposalExpireTime = 259200000 // default value: 3 days: 259200000(ms), Note: this value is controlled by committee proposal + # checkFrozenTime = 1 // for test only } # Transaction reference block, default is "solid", configure to "head" may cause TaPos error -# trx.reference.block = "solid" // head;solid; +trx.reference.block = "solid" // "head" or "solid" # This property sets the number of milliseconds after the creation of the transaction that is expired, default value is 60000. # trx.expiration.timeInMilliseconds = 60000 @@ -667,30 +692,77 @@ vm = { minTimeRatio = 0.0 maxTimeRatio = 5.0 saveInternalTx = false + # lruCacheSize = 500 + # vmTrace = false - # Indicates whether the node stores featured internal transactions, such as freeze, vote and so on + # Indicates whether the node stores featured internal transactions, such as freeze, vote and so on. Default: false. # saveFeaturedInternalTx = false - # Indicates whether the node stores the details of the internal transactions generated by the - # CANCELALLUNFREEZEV2 opcode, such as bandwidth/energy/tronpower cancel amount. + # Indicates whether the node stores the details of the internal transactions generated by the CANCELALLUNFREEZEV2 opcode, + # such as bandwidth/energy/tronpower cancel amount. Default: false. # saveCancelAllUnfreezeV2Details = false # In rare cases, transactions that will be within the specified maximum execution time (default 10(ms)) are re-executed and packaged # longRunningTime = 10 - # Indicates whether the node support estimate energy API. + # Indicates whether the node support estimate energy API. Default: false. # estimateEnergy = false - # Indicates the max retry time for executing transaction in estimating energy. + # Indicates the max retry time for executing transaction in estimating energy. Default 3. # estimateEnergyMaxRetry = 3 } +# These parameters are designed for private chain testing only and cannot be freely switched on or off in production systems. committee = { allowCreationOfContracts = 0 //mainnet:0 (reset by committee),test:1 allowAdaptiveEnergy = 0 //mainnet:0 (reset by committee),test:1 + # allowCreationOfContracts = 0 + # allowMultiSign = 0 + # allowAdaptiveEnergy = 0 + # allowDelegateResource = 0 + # allowSameTokenName = 0 + # allowTvmTransferTrc10 = 0 + # allowTvmConstantinople = 0 + # allowTvmSolidity059 = 0 + # forbidTransferToContract = 0 + # allowShieldedTRC20Transaction = 0 + # allowTvmIstanbul = 0 + # allowMarketTransaction = 0 + # allowProtoFilterNum = 0 + # allowAccountStateRoot = 0 + # changedDelegation = 0 + # allowPBFT = 0 + # pBFTExpireNum = 0 + # allowTransactionFeePool = 0 + # allowBlackHoleOptimization = 0 + # allowNewResourceModel = 0 + # allowReceiptsMerkleRoot = 0 + # allowTvmFreeze = 0 + # allowTvmVote = 0 + # unfreezeDelayDays = 0 + # allowTvmLondon = 0 + # allowTvmCompatibleEvm = 0 + # allowNewRewardAlgorithm = 0 + # allowAccountAssetOptimization = 0 + # allowAssetOptimization = 0 + # allowNewReward = 0 + # memoFee = 0 + # allowDelegateOptimization = 0 + # allowDynamicEnergy = 0 + # dynamicEnergyThreshold = 0 + # dynamicEnergyMaxFactor = 0 + # allowTvmShangHai = 0 + # allowOldRewardOpt = 0 + # allowEnergyAdjustment = 0 + # allowStrictMath = 0 + # allowTvmCancun = 0 + # allowTvmBlob = 0 + # consensusLogicOptimization = 0 + # allowOptimizedReturnValueOfChainId = 0 } event.subscribe = { + enable = false // enable event subscribe, replaces deprecated CLI flag --es native = { useNativeQueue = true // if true, use native message queue, else use event plugin. bindport = 5555 // bind port @@ -703,10 +775,10 @@ event.subscribe = { path = "" // absolute path of plugin server = "" // target server address to receive event triggers - // dbname|username|password, if you want to create indexes for collections when the collections - // are not exist, you can add version and set it to 2, as dbname|username|password|version - // if you use version 2 and one collection not exists, it will create index automaticaly; - // if you use version 2 and one collection exists, it will not create index, you must create index manually; + # dbname|username|password, if you want to create indexes for collections when the collections + # are not exist, you can add version and set it to 2, as dbname|username|password|version + # if you use version 2 and one collection not exists, it will create index automaticaly; + # if you use version 2 and one collection exists, it will not create index, you must create index manually; dbconfig = "" contractParse = true topics = [ @@ -714,17 +786,17 @@ event.subscribe = { triggerName = "block" // block trigger, the value can't be modified enable = false topic = "block" // plugin topic, the value could be modified - solidified = false // if set true, just need solidified block, default is false + solidified = false // if set true, just need solidified block. Default: false }, { triggerName = "transaction" enable = false topic = "transaction" solidified = false - ethCompatible = false // if set true, add transactionIndex, cumulativeEnergyUsed, preCumulativeLogCount, logList, energyUnitPrice, default is false + ethCompatible = false // if set true, add transactionIndex, cumulativeEnergyUsed, preCumulativeLogCount, logList, energyUnitPrice. Default: false }, { - triggerName = "contractevent" + triggerName = "contractevent" // contractevent represents contractlog data decoded by the ABI. enable = false topic = "contractevent" }, @@ -736,7 +808,7 @@ event.subscribe = { }, { triggerName = "solidity" // solidity block trigger(just include solidity block number and timestamp), the value can't be modified - enable = true // the default value is true + enable = true // Default: true topic = "solidity" }, { diff --git a/conf/main_net_config_open_metric.conf b/conf/main_net_config_open_metric.conf new file mode 100644 index 00000000..ee09f956 --- /dev/null +++ b/conf/main_net_config_open_metric.conf @@ -0,0 +1,8 @@ +include "main_net_config.conf" + +node.metrics = { + prometheus { + enable = true + port = 9527 + } +} diff --git a/conf/private_net_config_witness1.conf b/conf/private_net_config_witness1.conf index 22953e16..1e07306f 100644 --- a/conf/private_net_config_witness1.conf +++ b/conf/private_net_config_witness1.conf @@ -272,7 +272,10 @@ genesis.block = { } localwitness = [ - # address TPL66VK2gCXNCD7EJg9pgJRfqcRazjhUZY + # IMPORTANT: This is a DEMO private key for local testing ONLY + # Address: TPL66VK2gCXNCD7EJg9pgJRfqcRazjhUZY + # WARNING: Replace with your own generated key for any real deployment + # NEVER use this key on mainnet with real funds da146374a75310b9666e834ee4ad0866d6f4035967bfc76217c5a495fff9f0d0 # you must enable this value and the witness address are match. ] diff --git a/conf/private_net_config_witness2.conf b/conf/private_net_config_witness2.conf index b07d09d3..9f53d58f 100644 --- a/conf/private_net_config_witness2.conf +++ b/conf/private_net_config_witness2.conf @@ -274,7 +274,10 @@ genesis.block = { } localwitness = [ - # address TCjptjyjenNKB2Y6EwyVT43DQyUUorxKWi + # IMPORTANT: This is a DEMO private key for local testing ONLY + # Address: TCjptjyjenNKB2Y6EwyVT43DQyUUorxKWi + # WARNING: Replace with your own generated key for any real deployment + # NEVER use this key on mainnet with real funds 0ab0b4893c83102ed7be35eee6d50f081625ac75a07da6cb58b1ad2e9c18ce43 # you must enable this value and the witness address are match. ] diff --git a/conf/private_net_layout.toml b/conf/private_net_layout.toml index 11519b22..5a8706d5 100644 --- a/conf/private_net_layout.toml +++ b/conf/private_net_layout.toml @@ -1,8 +1,15 @@ +# SECURITY WARNING: This file contains sensitive configuration for remote node deployment. +# DO NOT commit actual credentials, SSH keys, or real hostnames to the repository. +# Use environment variables, .env files (added to .gitignore), or secure credential management systems. +# See: https://12factor.net/config +# +# Example configuration for private network layout: + # [[nodes]] -# node_ip = "192.168.1.1" # Remote node's IP +# node_ip = "127.0.0.1" # Remote node's IP # node_directory = "/path/to/direcotry" # Remote node's working direcotry for node # config_file = "/path/to/config" # Config file for remote node -# docker_compose_file =”/path/to/config“ # Config docker-compose file for remote node +# docker_compose_file ="/path/to/config" # Config docker-compose file for remote node # node_type = "fullnode/sr" # Fullnode or SR node # ssh_port = 22 # ssh_user = "user1" @@ -10,10 +17,10 @@ # # ssh_key = "/path/to/key1" # Optional; uncomment if using key auth # [[nodes]] -# node_ip = "192.168.1.2" # Changed IP to demonstrate different nodes +# node_ip = "127.0.0.1" # Changed IP to demonstrate different nodes # node_directory = "/path/to/directory" # config_file = "/path/to/config" -# docker_compose_file =”/path/to/config“ # Config docker-compose file for remote node +# docker_compose_file ="/path/to/config" # Config docker-compose file for remote node # node_type = "fullnode/sr" # ssh_port = 2222 # Custom SCP port for this node # ssh_user = "user2" @@ -21,11 +28,13 @@ [[nodes]] -node_ip = "ec2-3-25-116-244.ap-southeast-2.compute.amazonaws.com" -node_directory = "/home/ubuntu/mytest" -config_file = "/Users/ubuntu/conf/private_net_config_others.conf" -docker_compose_file = "/Users/ubuntu/docker-compose.yml" +node_ip = "127.0.0.1" # Replace with your actual node IP or hostname +node_directory = "/path/to/tron-node" # Replace with your actual node directory +config_file = "/path/to/private_net_config.conf" # Replace with your actual config path +docker_compose_file = "/path/to/docker-compose.yml" # Replace with your actual docker-compose path ssh_port = 22 -ssh_user = "ubuntu" -# ssh_password = "password1" -ssh_key = "/Users/ubuntu/Downloads/test-ci.pem" # Optional; uncomment if using key auth +ssh_user = "ubuntu" # Replace with your actual SSH user +# ssh_password = "password" # Optional; uncomment if using password auth (NOT RECOMMENDED) +# ssh_key = "/path/to/your/private/key" # Optional; uncomment if using key auth +# SECURITY WARNING: Never commit actual SSH keys or private credentials to the repository! +# Use environment variables or secure credential management systems instead. diff --git a/download_trond.sh b/download_trond.sh deleted file mode 100755 index e5f15058..00000000 --- a/download_trond.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# Configuration -REPO_OWNER="tronprotocol" -REPO_NAME="tron-docker" -RELEASE_TAG="v0.1.1" -CHECKSUM_FILE="checksums.txt" -RACKAGE_PREFIX="trond" - -# Determine the OS and architecture, then set the ASSET_NAME -OS=$(uname -s) -ARCH=$(uname -m) -case "$OS" in - Linux*) - case "$ARCH" in - x86_64) ASSET_NAME="${RACKAGE_PREFIX}_${RELEASE_TAG}_Linux_x86_64.tar.gz";; - arm64) ASSET_NAME="${RACKAGE_PREFIX}_${RELEASE_TAG}_Linux_arm64.tar.gz";; - *) echo "Unsupported architecture: $ARCH"; exit 1;; - esac - ;; - Darwin*) - case "$ARCH" in - x86_64) ASSET_NAME="${RACKAGE_PREFIX}_${RELEASE_TAG}_Darwin_x86_64.tar.gz";; - arm64) ASSET_NAME="${RACKAGE_PREFIX}_${RELEASE_TAG}_Darwin_arm64.tar.gz";; - *) echo "Unsupported architecture: $ARCH"; exit 1;; - esac - ;; - *) echo "Unsupported OS: $OS"; exit 1;; -esac - -# Download URLs -DOWNLOAD_URL="https://github.com/${REPO_OWNER}/${REPO_NAME}/releases/download/${RELEASE_TAG}/${ASSET_NAME}" -CHECKSUM_URL="https://github.com/${REPO_OWNER}/${REPO_NAME}/releases/download/${RELEASE_TAG}/${CHECKSUM_FILE}" - -# Download files -echo "Downloading ${ASSET_NAME}..." -echo ${DOWNLOAD_URL} -curl -L -O "${DOWNLOAD_URL}" --fail --progress-bar - -echo "Downloading checksum file..." -echo ${DOWNLOAD_URL} -curl -L -O "${CHECKSUM_URL}" --fail --progress-bar - -# Verify checksum -echo "Verifying checksum..." -if ! sha256sum --check --ignore-missing "${CHECKSUM_FILE}"; then - echo "Checksum validation failed!" - exit 1 -fi - -echo "✅ Success! ${ASSET_NAME} is valid." - -tar -xvzf ${ASSET_NAME} - -echo "Download completed successfully." diff --git a/metric_monitor/README.md b/metric_monitor/README.md index 9266a14d..9f9fa5e0 100644 --- a/metric_monitor/README.md +++ b/metric_monitor/README.md @@ -16,7 +16,7 @@ Download the `tron-docker` repository, enter the [metric_monitor](./) directory, ```sh docker-compose -f ./docker-compose/docker-compose-quick-start.yml up -d ``` -It will start a TRON FullNode that connects to the Mainnet, along with Prometheus and Grafana services. Note that in [main_net_config.conf](../conf/main_net_config.conf), it contains the configuration below to enable metrics. +It will start a TRON FullNode that connects to the Mainnet, along with Prometheus and Grafana services. Note that in [main_net_config_open_metric.conf](../conf/main_net_config_open_metric.conf), it contains the configuration below to enable metrics. ``` node.metrics{ prometheus{ @@ -121,6 +121,11 @@ Used to check the block process performance from TronNetDelegate: - `tron:block_process_latency_seconds_count`: Count of events - `tron:block_process_latency_seconds_sum`: Total sum of all observed values +Used to check the block generate performance from the Manager: +- `tron:block_generate_latency_seconds_bucket`: Cumulative counters +- `tron:block_generate_latency_seconds_count`: Count of events +- `tron:block_generate_latency_seconds_sum`: Total sum of all observed values + Used to check the block processing latency from the Manager, which is invoked by TronNetDelegate: - `tron:block_push_latency_seconds_bucket`: Cumulative counters - `tron:block_push_latency_seconds_count`: Count of events @@ -147,6 +152,11 @@ Verify the latency of all transactions' signatures when processing a block: - `tron:verify_sign_latency_seconds_count`: Count of events - `tron:verify_sign_latency_seconds_sum`: Total sum of all observed values +Histogram of transaction count per block, with buckets `[0, 20, 50, 80, 100, 120, 140, 160, 180, 200, 230, 260, 300, 500, 2000]` (densified around 0–300 for percentile interpolation in the typical per-SR TPS range). Empty blocks can be queried via the `le="0.0"` bucket; the distribution buckets enable transaction volume analysis (P95/P99, large-block ratio, etc.): +- `tron:block_transaction_count_bucket`: Cumulative counters per bucket. Label: `miner` (SR address). +- `tron:block_transaction_count_count`: Count of observed blocks +- `tron:block_transaction_count_sum`: Total sum of all observed transaction counts + Check the usage from dashboard panel (enter edit mode), or by searching in [grafana_dashboard_tron_server.json](grafana_dashboard/grafana_dashboard_tron_server.json). ![image](../images/metric_block_latency.png) diff --git a/metric_monitor/REMOTE_WRITE_WITH_THANOS.md b/metric_monitor/REMOTE_WRITE_WITH_THANOS.md index e103a949..eb994a39 100644 --- a/metric_monitor/REMOTE_WRITE_WITH_THANOS.md +++ b/metric_monitor/REMOTE_WRITE_WITH_THANOS.md @@ -145,12 +145,18 @@ docker-compose -f ./docker-compose/docker-compose-alloy.yml up -d The [Thanos Receive](https://thanos.io/tip/components/receive.md/#receiver) service implements the Prometheus Remote Write API. It builds on top of the existing Prometheus TSDB and retains its usefulness while extending its functionality with long-term-storage, horizontal scalability, and downsampling. Prometheus instances are configured to continuously write metrics to it. Thanos Receive exposes the StoreAPI so that Thanos Queriers can query received metrics in real-time. -First, deploy [Minio](https://github.com/minio/minio) for long-term metric storage. Minio offers S3-compatible object storage functionality, allowing Thanos Receive to upload TSDB blocks to storage buckets at 2-hour intervals. While this guide uses Minio, you can opt for any storage service from the [Thanos Supported Clients](https://thanos.io/tip/thanos/storage.md/#supported-clients) list. For long-term monitoring, we recommend implementing a retention policy on your storage service to efficiently manage historical metric data. For instance, you might configure an S3 lifecycle policy when using AWS to automatically remove metrics older than one year. +First, deploy [Minio](https://github.com/minio/minio) for long-term metric storage. Minio offers S3-compatible object storage functionality, allowing Thanos Receive to upload TSDB blocks to storage buckets at 2-hour intervals. + +**⚠️ Important**: The MinIO configuration in this guide uses demo credentials (`minio`/`melovethanos`) for local testing only. For production deployments, use AWS S3 or other cloud storage services with proper IAM credentials, or generate strong unique credentials if using MinIO. + +While this guide uses Minio, you can opt for any storage service from the [Thanos Supported Clients](https://thanos.io/tip/thanos/storage.md/#supported-clients) list. For long-term monitoring, we recommend implementing a retention policy on your storage service to efficiently manage historical metric data. For instance, you might configure an S3 lifecycle policy when using AWS to automatically remove metrics older than one year. + ```sh -# Start Minio +# Start Minio (for local testing only) docker-compose -f ./docker-compose/minio.yml up -d # First set the MinIO alias with root credentials to enable bucket creation permissions +# Note: These are demo credentials - replace with your own in production docker exec minio mc alias set local http://localhost:9000 minio melovethanos # Then create the bucket diff --git a/metric_monitor/conf/bucket_storage.yml b/metric_monitor/conf/bucket_storage.yml index a9d4eded..e90dffaa 100644 --- a/metric_monitor/conf/bucket_storage.yml +++ b/metric_monitor/conf/bucket_storage.yml @@ -3,5 +3,8 @@ config: bucket: "test-thanos-001" endpoint: "minio:9000" # for example: s3.ap-southeast-1.amazonaws.com for AWS S3 on region ap-southeast-1 insecure: true # True for local test using http instead of https + # ⚠️ DEMO CREDENTIALS FOR LOCAL TESTING ONLY ⚠️ + # These match the MinIO demo credentials and should NEVER be used in production + # For production: Use AWS S3 with proper IAM roles or access keys access_key: "minio" secret_key: "melovethanos" diff --git a/metric_monitor/conf/prometheus-loki.alloy b/metric_monitor/conf/prometheus-loki.alloy index d9fb2b4b..f9109131 100644 --- a/metric_monitor/conf/prometheus-loki.alloy +++ b/metric_monitor/conf/prometheus-loki.alloy @@ -57,23 +57,25 @@ prometheus.remote_write "default" { endpoint { url = "http://xxxx/api/v1/receive" - remote_timeout = "10s" + remote_timeout = "7s" headers = { "X-Auth-Token" = "token", "X-Service-Group" = "fullnode", } queue_config { - capacity = 55000 - max_shards = 200 - max_samples_per_send = 15000 - batch_send_deadline = "6s" - min_backoff = "200ms" + capacity = 3000 + max_shards = 10 // Maximum number of parallel requests + max_samples_per_send = 3000 // Maximum samples per send + batch_send_deadline = "3s" // Maximum delay before sending a batch + min_backoff = "3000ms" // Minimum retry interval + sample_age_limit = "7s" // The maximum age of samples to send. This value must be greater than the scrape interval and batch_send_deadline. + retry_on_http_429 = false // Drop requests when receiving an HTTP 429 (Too Many Requests) status code } metadata_config { - send_interval = "6s" - max_samples_per_send = 15000 + send_interval = "120s" // Low frequency for relative static value + max_samples_per_send = 1000 } } } diff --git a/metric_monitor/conf/prometheus-remote-write.yml b/metric_monitor/conf/prometheus-remote-write.yml index 51ab13c9..370d074b 100644 --- a/metric_monitor/conf/prometheus-remote-write.yml +++ b/metric_monitor/conf/prometheus-remote-write.yml @@ -32,16 +32,17 @@ remote_write: - url: http://thanos-receive:10908/api/v1/receive # if Thanos Receive service run on the same host with Prometheus headers: X-Service-Group: "tron-fullnode-group1" - remote_timeout: 15s + remote_timeout: 9s queue_config: - capacity: 50000 - max_shards: 200 # the maximum number of shards, or parallelism, Prometheus will use for each remote-write queue + capacity: 3000 + max_shards: 10 # the maximum number of shards, or parallelism, Prometheus will use for each remote-write queue min_shards: 1 - max_samples_per_send: 10000 - batch_send_deadline: 3s - min_backoff: 200ms - max_backoff: 5s + max_samples_per_send: 1000 + batch_send_deadline: 6s + min_backoff: 3000ms + sample_age_limit: 10s # The maximum age of samples to send. This value must be greater than the scrape interval and batch_send_deadline. + retry_on_http_429: false # Drop requests when receiving an HTTP 429 (Too Many Requests) status code metadata_config: send: true - send_interval: 3s # How frequently metric metadata is sent to remote storage. - max_samples_per_send: 50000 + send_interval: 120s # How frequently metric metadata is sent to remote storage. + max_samples_per_send: 1000 diff --git a/metric_monitor/docker-compose.yml b/metric_monitor/docker-compose.yml deleted file mode 100644 index 2619460c..00000000 --- a/metric_monitor/docker-compose.yml +++ /dev/null @@ -1,47 +0,0 @@ -version: '3.8' -services: - tron-node1: - image: tronprotocol/java-tron:latest - container_name: tron-node1 - networks: - - tron_network - deploy: - resources: - limits: - memory: 16g - ports: - - "8090:8090" # for external http API request - - "9527:9527" # used for metric API - volumes: - - ../conf:/java-tron/conf - - ./datadir:/java-tron/data # mount a local directory to make the blocks data persistent. - command: > - -jvm "{-Xmx16g -Xms12g -XX:+UseConcMarkSweepGC -XX:+PrintGC}" -c ./conf/main_net_config.conf -d /java-tron/data - - prometheus: - image: prom/prometheus:latest - container_name: prometheus - networks: - - tron_network - deploy: - resources: - limits: - memory: 1g - ports: - - "9090:9090" # used for external collect metrics - volumes: - - ./metric_conf/prometheus.yml:/etc/prometheus/prometheus.yml - grafana: - image: grafana/grafana-oss - container_name: grafana - networks: - - tron_network - deploy: - resources: - limits: - memory: 1g - ports: - - "3000:3000" -networks: - tron_network: - driver: bridge diff --git a/metric_monitor/docker-compose/docker-compose-alloy.yml b/metric_monitor/docker-compose/docker-compose-alloy.yml index 3b70a85d..cd0acfee 100644 --- a/metric_monitor/docker-compose/docker-compose-alloy.yml +++ b/metric_monitor/docker-compose/docker-compose-alloy.yml @@ -2,7 +2,7 @@ version: '3.8' services: alloy: - image: grafana/alloy:latest + image: grafana/alloy:v1.14.0 container_name: grafana-alloy volumes: - '/:/host:ro,rslave' diff --git a/metric_monitor/docker-compose/docker-compose-quick-start.yml b/metric_monitor/docker-compose/docker-compose-quick-start.yml index 575eee2a..35acb14d 100644 --- a/metric_monitor/docker-compose/docker-compose-quick-start.yml +++ b/metric_monitor/docker-compose/docker-compose-quick-start.yml @@ -6,7 +6,7 @@ services: service: tron-node prometheus: - image: prom/prometheus:latest + image: prom/prometheus:v3.11.3 container_name: prometheus deploy: resources: diff --git a/metric_monitor/docker-compose/grafana.yml b/metric_monitor/docker-compose/grafana.yml index 49feafd8..581e60ab 100644 --- a/metric_monitor/docker-compose/grafana.yml +++ b/metric_monitor/docker-compose/grafana.yml @@ -1,13 +1,16 @@ version: '3.8' services: grafana: - image: grafana/grafana-oss + image: grafana/grafana-oss:12.4.1 container_name: grafana - user: root + security_opt: + - no-new-privileges:true + cap_drop: + - ALL deploy: resources: limits: - memory: 1g + memory: 2g environment: - GF_METRICS_ENABLED=true # Enable Grafana metrics exposure - GF_SERVER_HTTP_PORT=3000 # Set Grafana's internal port to 3000 diff --git a/metric_monitor/docker-compose/minio.yml b/metric_monitor/docker-compose/minio.yml index a90c82b6..528732c2 100644 --- a/metric_monitor/docker-compose/minio.yml +++ b/metric_monitor/docker-compose/minio.yml @@ -1,13 +1,17 @@ version: '3.8' services: minio: - image: minio/minio:latest + image: minio/minio:RELEASE.2025-09-07T16-13-09Z container_name: minio ports: - "9000:9000" # Map port for API - "9001:9001" # Map port for UI command: minio server /data --console-address ":9001" environment: + # ⚠️ DEMO CREDENTIALS FOR LOCAL TESTING ONLY ⚠️ + # These are publicly known credentials for demonstration purposes + # For production: Use AWS S3 or other cloud storage with proper IAM credentials + # Never use these credentials in production environments MINIO_ROOT_USER: minio MINIO_ROOT_PASSWORD: melovethanos volumes: diff --git a/metric_monitor/docker-compose/node-exporter.yml b/metric_monitor/docker-compose/node-exporter.yml index da8bac20..c858c8e4 100644 --- a/metric_monitor/docker-compose/node-exporter.yml +++ b/metric_monitor/docker-compose/node-exporter.yml @@ -1,7 +1,7 @@ version: '3.8' services: node-exporter: - image: quay.io/prometheus/node-exporter:latest + image: quay.io/prometheus/node-exporter:v1.10.2 container_name: node-exporter network_mode: host pid: host diff --git a/metric_monitor/docker-compose/prometheus.yml b/metric_monitor/docker-compose/prometheus.yml index dace2739..9fe14e95 100644 --- a/metric_monitor/docker-compose/prometheus.yml +++ b/metric_monitor/docker-compose/prometheus.yml @@ -1,9 +1,12 @@ version: '3.8' services: prometheus: - image: quay.io/prometheus/prometheus:v2.38.0 # NOTE: It is highly recommended to use Prometheus v2.13.0+ due to its remote read improvements. + image: prom/prometheus:v3.11.3 container_name: prometheus - user: root + security_opt: + - no-new-privileges:true + cap_drop: + - ALL ports: - "9090:9090" # used for local query metrics volumes: diff --git a/metric_monitor/docker-compose/thanos-querier.yml b/metric_monitor/docker-compose/thanos-querier.yml index 4221da15..275bf138 100644 --- a/metric_monitor/docker-compose/thanos-querier.yml +++ b/metric_monitor/docker-compose/thanos-querier.yml @@ -1,9 +1,8 @@ version: '3.8' services: thanos-querier: - image: quay.io/thanos/thanos:v0.33.0 + image: quay.io/thanos/thanos:v0.41.0 container_name: thanos-querier - user: root ports: - "9091:9091" command: diff --git a/metric_monitor/docker-compose/thanos-receive.yml b/metric_monitor/docker-compose/thanos-receive.yml index 6f394c55..15a582ce 100644 --- a/metric_monitor/docker-compose/thanos-receive.yml +++ b/metric_monitor/docker-compose/thanos-receive.yml @@ -1,8 +1,7 @@ version: '3.8' services: thanos-receive: - image: quay.io/thanos/thanos:v0.33.0 - user: root + image: quay.io/thanos/thanos:v0.41.0 container_name: thanos-receive volumes: - ../receive-data-0:/receive/data diff --git a/metric_monitor/docker-compose/thanos-store.yml b/metric_monitor/docker-compose/thanos-store.yml index 69084528..0d17bdf4 100644 --- a/metric_monitor/docker-compose/thanos-store.yml +++ b/metric_monitor/docker-compose/thanos-store.yml @@ -1,9 +1,8 @@ version: '3.8' services: thanos-store: - image: quay.io/thanos/thanos:v0.33.0 + image: quay.io/thanos/thanos:v0.41.0 container_name: thanos-store - user: root ports: - "10911:10911" # HTTP - "10912:10912" # gRPC diff --git a/metric_monitor/docker-compose/tron-fullnode.yml b/metric_monitor/docker-compose/tron-fullnode.yml index 127425e6..c79acf98 100644 --- a/metric_monitor/docker-compose/tron-fullnode.yml +++ b/metric_monitor/docker-compose/tron-fullnode.yml @@ -1,9 +1,8 @@ version: '3.8' services: tron-node: - image: tronprotocol/java-tron:latest + image: tronprotocol/java-tron:latest # always use the lastest container_name: tron-node1 - user: root deploy: resources: limits: @@ -19,6 +18,6 @@ services: - ../output-directory:/java-tron/output-directory # mount a local directory to make the block data persistent. - ../logs:/java-tron/logs command: > - -jvm "{-Xmx14g -Xmn1g -XX:MaxDirectMemorySize=1G -XX:+UseConcMarkSweepGC -XX:+PrintGC -Xloggc:./logs/gc.log }" -c ./conf/main_net_config.conf + -jvm "{-Xmx14g -Xmn1g -XX:MaxDirectMemorySize=1G}" -c ./conf/main_net_config_open_metric.conf environment: - TZ=Asia/Shanghai # The metric and log date will be changed accordingly diff --git a/metric_monitor/grafana_dashboard/java-tron-api-statistic.json b/metric_monitor/grafana_dashboard/java-tron-api-statistic.json index 86ffbf59..230ba1e6 100644 --- a/metric_monitor/grafana_dashboard/java-tron-api-statistic.json +++ b/metric_monitor/grafana_dashboard/java-tron-api-statistic.json @@ -1243,6 +1243,30 @@ "regex": "", "sort": 1, "type": "query" + }, + { + "allowCustomValue": true, + "current": { + "text": "", + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values({group=\"$group\"},instance)", + "includeAll": false, + "label": "instance", + "name": "instance", + "options": [], + "query": { + "query": "label_values({group=\"$group\"},instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" } ] }, diff --git a/metric_monitor/grafana_dashboard/java-tron-api.json b/metric_monitor/grafana_dashboard/java-tron-api.json index 120f7f9d..38f1508c 100644 --- a/metric_monitor/grafana_dashboard/java-tron-api.json +++ b/metric_monitor/grafana_dashboard/java-tron-api.json @@ -192,7 +192,7 @@ "exemplar": true, "expr": "sum(rate(tron:http_service_latency_seconds_count{group=`$group`,instance=`$instance`}[$__interval]))", "hide": false, - "interval": "1s", + "interval": "60s", "legendFormat": "total", "refId": "B" }, @@ -204,7 +204,7 @@ "expr": "rate(tron:http_service_latency_seconds_count{group=`$group`,instance=`$instance`}[$__interval])", "format": "time_series", "instant": false, - "interval": "1s", + "interval": "60s", "intervalFactor": 1, "legendFormat": "{{url}}", "refId": "A" @@ -1084,20 +1084,20 @@ }, { "current": { - "text": "tron-docker-dev-001", - "value": "tron-docker-dev-001" + "text": "java-tron", + "value": "java-tron" }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(instance)", + "definition": "label_values(group)", "includeAll": false, - "label": "instance", - "name": "instance", + "label": "group", + "name": "group", "options": [], "query": { - "query": "label_values(instance)", + "query": "label_values(group)", "refId": "StandardVariableQuery" }, "refresh": 1, @@ -1107,20 +1107,20 @@ }, { "current": { - "text": "java-tron", - "value": "java-tron" + "text": "", + "value": "" }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(group)", + "definition": "label_values({group=\"$group\"},instance)", "includeAll": false, - "label": "group", - "name": "group", + "label": "instance", + "name": "instance", "options": [], "query": { - "query": "label_values(group)", + "query": "label_values({group=\"$group\"},instance)", "refId": "StandardVariableQuery" }, "refresh": 1, diff --git a/metric_monitor/grafana_dashboard/java-tron-mechanism.json b/metric_monitor/grafana_dashboard/java-tron-mechanism.json index c54037ad..8df46622 100644 --- a/metric_monitor/grafana_dashboard/java-tron-mechanism.json +++ b/metric_monitor/grafana_dashboard/java-tron-mechanism.json @@ -817,20 +817,20 @@ }, { "current": { - "text": "tron-docker-dev-001", - "value": "tron-docker-dev-001" + "text": "java-tron", + "value": "java-tron" }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(instance)", + "definition": "label_values(group)", "includeAll": false, - "label": "instance", - "name": "instance", + "label": "group", + "name": "group", "options": [], "query": { - "query": "label_values(instance)", + "query": "label_values(group)", "refId": "StandardVariableQuery" }, "refresh": 1, @@ -840,20 +840,20 @@ }, { "current": { - "text": "java-tron", - "value": "java-tron" + "text": "", + "value": "" }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(group)", + "definition": "label_values({group=\"$group\"},instance)", "includeAll": false, - "label": "group", - "name": "group", + "label": "instance", + "name": "instance", "options": [], "query": { - "query": "label_values(group)", + "query": "label_values({group=\"$group\"},instance)", "refId": "StandardVariableQuery" }, "refresh": 1, diff --git a/metric_monitor/grafana_dashboard/java-tron-server.json b/metric_monitor/grafana_dashboard/java-tron-server.json index b37e26cf..d28341e1 100644 --- a/metric_monitor/grafana_dashboard/java-tron-server.json +++ b/metric_monitor/grafana_dashboard/java-tron-server.json @@ -25,7 +25,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 4, + "id": null, "links": [], "panels": [ { @@ -37,7 +37,6 @@ "y": 0 }, "id": 86, - "panels": [], "title": "Chain", "type": "row" }, @@ -412,147 +411,13 @@ "h": 1, "w": 24, "x": 0, - "y": 16 + "y": 17 }, - "id": 34, + "id": 36, "panels": [ { "datasource": "${datasource}", - "description": "The number of processors available to the java virtual machine.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "noValue": "N/A", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 251 - }, - "id": 77, - "interval": "3s", - "maxDataPoints": 500, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "exemplar": true, - "expr": "system_available_cpus{group=`$group`,instance=`$instance`}", - "hide": false, - "interval": "", - "legendFormat": "{{instance}}", - "refId": "B" - } - ], - "title": "CPUS", - "type": "stat" - }, - { - "datasource": "${datasource}", - "description": "Total: total physical memory.\n\nFree: free physical memory.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "noValue": "N/A", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 251 - }, - "id": 78, - "interval": "3s", - "maxDataPoints": 500, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "exemplar": true, - "expr": "system_total_physical_memory_bytes{group=`$group`,instance=`$instance`}", - "hide": false, - "interval": "", - "legendFormat": "Total", - "refId": "B" - }, - { - "exemplar": true, - "expr": "system_free_physical_memory_bytes{group=`$group`,instance=`$instance`}", - "hide": false, - "interval": "", - "legendFormat": "Free", - "refId": "A" - } - ], - "title": "Memory", - "type": "stat" - }, - { - "datasource": "${datasource}", - "description": "The system load average for the last minute. The system load average is the sum of the number of runnable entities queued to the available processors and the number of runnable entities running on the available processors averaged over a period of time. The way in which the load average is calculated is operating system specific but is typically a damped time-dependent average.", + "description": "Waiting for packed tansactions, including pending and repush.", "fieldConfig": { "defaults": { "color": { @@ -597,28 +462,30 @@ "steps": [ { "color": "green" + }, + { + "color": "red", + "value": 1000 } ] - }, - "unit": "none" + } }, "overrides": [] }, "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 251 + "h": 8, + "w": 12, + "x": 0, + "y": 253 }, - "id": 81, + "id": 6, "interval": "3s", "maxDataPoints": 500, "options": { "legend": { "calcs": [ - "lastNotNull", - "min", - "max" + "max", + "mean" ], "displayMode": "list", "placement": "bottom", @@ -633,104 +500,21 @@ "targets": [ { "exemplar": true, - "expr": "system_load_average{group=`$group`,instance=`$instance`}", - "hide": false, + "expr": "sum(tron:manager_queue_size{group=`$group`,instance=`$instance`,type !=`popped`,type !=`queued`})", + "format": "time_series", + "instant": false, "interval": "", - "legendFormat": "load average", - "refId": "B" + "intervalFactor": 1, + "legendFormat": "pending pool", + "refId": "A" } ], - "title": "Load Average", + "title": "Pending Pool", "type": "timeseries" }, { "datasource": "${datasource}", - "description": "Max: maximum number of open file descriptors. \n\nCurrent: number of open file descriptors.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "noValue": "N/A", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 251 - }, - "id": 83, - "interval": "3s", - "maxDataPoints": 500, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "exemplar": true, - "expr": "process_max_fds{group=`$group`,instance=`$instance`}", - "hide": false, - "interval": "", - "legendFormat": "Max", - "refId": "B" - }, - { - "exemplar": true, - "expr": "process_open_fds{group=`$group`,instance=`$instance`}", - "hide": false, - "interval": "", - "legendFormat": "Current", - "refId": "A" - } - ], - "title": "Open Fds", - "type": "stat" - } - ], - "title": "System", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 17 - }, - "id": 36, - "panels": [ - { - "datasource": "${datasource}", - "description": "Waiting for packed tansactions, including pending and repush.", + "description": "Average transaction processing time.", "fieldConfig": { "defaults": { "color": { @@ -745,7 +529,7 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, @@ -759,7 +543,7 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -778,108 +562,11 @@ }, { "color": "red", - "value": 1000 + "value": 80 } ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 253 - }, - "id": 6, - "interval": "3s", - "maxDataPoints": 500, - "options": { - "legend": { - "calcs": [ - "max", - "mean" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "exemplar": true, - "expr": "sum(tron:manager_queue_size{group=`$group`,instance=`$instance`,type !=`popped`,type !=`queued`})", - "format": "time_series", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "pending pool", - "refId": "A" - } - ], - "title": "Pending Pool", - "type": "timeseries" - }, - { - "datasource": "${datasource}", - "description": "Average transaction processing time.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" + }, + "unit": "s" }, "overrides": [ { @@ -1034,7 +721,7 @@ "disableTextWrap": false, "editorMode": "builder", "exemplar": false, - "expr": "group by(contract) (rate(tron:process_transaction_latency_seconds_sum{type=\"block\"}[$__rate_interval])) / group by(contract) (rate(tron:process_transaction_latency_seconds_count{type=\"block\"}[$__rate_interval]))", + "expr": "rate(tron:process_transaction_latency_seconds_sum{group=\"$group\", instance=\"$instance\", subgroup=\"$subgroup\", type=\"block\"}[$__rate_interval]) / rate(tron:process_transaction_latency_seconds_count{group=\"$group\", instance=\"$instance\", subgroup=\"$subgroup\", type=\"block\"}[$__rate_interval])", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -1051,7 +738,7 @@ }, { "datasource": "${datasource}", - "description": "Average transaction TPS from broadcasted transactions, not from blocks.", + "description": "Average transaction processing time.", "fieldConfig": { "defaults": { "color": { @@ -1066,7 +753,7 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -1080,7 +767,8 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "never", + "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -1095,7 +783,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -1103,7 +792,7 @@ } ] }, - "unit": "short" + "unit": "s" }, "overrides": [ { @@ -1115,16 +804,7 @@ "value": 0 } }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] + "properties": [] }, { "matcher": { @@ -1135,16 +815,7 @@ "value": 0 } }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] + "properties": [] }, { "matcher": { @@ -1155,33 +826,54 @@ "value": 0 } }, + "properties": [] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [] + }, + { + "matcher": { + "id": "byName", + "options": "avg" + }, "properties": [ { - "id": "custom.hideFrom", + "id": "color", "value": { - "legend": true, - "tooltip": true, - "viz": false + "fixedColor": "#73BF69", + "mode": "fixed" } } ] }, { + "__systemRef": "hideSeriesFrom", "matcher": { - "id": "byValue", + "id": "byNames", "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 + "mode": "exclude", + "names": [ + "tron-vip-vg-003" + ], + "prefix": "All except:", + "readOnly": true } }, "properties": [ { "id": "custom.hideFrom", "value": { - "legend": true, + "legend": false, "tooltip": true, - "viz": false + "viz": true } } ] @@ -1192,78 +884,43 @@ "h": 8, "w": 12, "x": 0, - "y": 416 + "y": 33 }, - "id": 4, + "id": 109, "interval": "3s", - "maxDataPoints": 500, + "maxDataPoints": 200, "options": { - "alertThreshold": true, "legend": { "calcs": [ - "mean" + "mean", + "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "multi", - "sort": "desc" + "hideZeros": false, + "mode": "single", + "sort": "none" } }, - "pluginVersion": "11.4.0", "targets": [ { - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": true, - "expr": "sum(rate(tron:process_transaction_latency_seconds_count{group=\"$group\", instance=\"$instance\", type!=\"block\"}[$__interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "interval": "6s", - "legendFormat": "total", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": true, - "expr": "rate(tron:process_transaction_latency_seconds_count{group=\"$group\", instance=\"$instance\", type!=\"block\"}[$__interval])", - "format": "time_series", - "fullMetaSearch": false, - "includeNullMetadata": true, + "exemplar": false, + "expr": "rate(tron:process_transaction_latency_seconds_sum{type=\"block\", contract=\"TriggerSmartContract\", instance=\"$instance\"}[$__rate_interval]) / rate(tron:process_transaction_latency_seconds_count{type=\"block\"}[$__rate_interval])", "instant": false, "interval": "6s", - "intervalFactor": 1, - "legendFormat": "{{contract}}", - "refId": "A", - "useBackend": false + "legendFormat": "{{instance}}", + "refId": "C" } ], - "title": " Transaction TPS", + "title": "Transaction Process Latency - Smart Contract", "type": "timeseries" - } - ], - "title": "Txs", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 18 - }, - "id": 38, - "panels": [ + }, { "datasource": "${datasource}", - "description": "Average block generate time .", + "description": "Transaction TPS from Block/1d", "fieldConfig": { "defaults": { "color": { @@ -1293,6 +950,7 @@ "type": "linear" }, "showPoints": "never", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -1307,7 +965,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -1315,7 +974,7 @@ } ] }, - "unit": "s" + "unit": "short" }, "overrides": [ { @@ -1397,17 +1056,41 @@ } } ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "total" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": true, + "viz": true + } + } + ] } ] }, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 34 + "x": 12, + "y": 33 }, - "id": 84, - "interval": "1m", + "id": 123, + "interval": "3s", "maxDataPoints": 500, "options": { "alertThreshold": true, @@ -1418,46 +1101,30 @@ ], "displayMode": "table", "placement": "bottom", - "showLegend": true + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", - "sort": "none" + "sort": "desc" } }, - "pluginVersion": "11.6.0", "targets": [ { - "editorMode": "code", - "exemplar": false, - "expr": " rate(tron:block_generate_latency_seconds_sum{group=`$group`,instance=`$instance`}[$__interval])\n/\n rate(tron:block_generate_latency_seconds_count{group=`$group`,instance=`$instance`}[$__interval])", - "format": "time_series", - "hide": false, - "interval": "12s", - "legendFormat": "__auto", - "range": true, - "refId": "B" - }, - { - "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tron:block_generate_latency_seconds_bucket{group=`$group`,instance=`$instance`}[$__interval])) by (le))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "1m", - "intervalFactor": 1, - "legendFormat": "tp99", - "refId": "A" - } + "expr": "sum(tron:process_transaction_latency_seconds_count{group=\"$group\", instance=\"$instance\", type=\"block\"})-sum(tron:process_transaction_latency_seconds_count{group=\"$group\", instance=\"$instance\", type=\"block\"} offset 24h)", + "interval": "6s", + "legendFormat": "total", + "refId": "B" + } ], - "title": "Block Generate Latency", + "title": "Transaction TPS from Block/1d", "type": "timeseries" }, { "datasource": "${datasource}", - "description": "Average block processing time when is not catching up.", "fieldConfig": { "defaults": { "color": { @@ -1472,7 +1139,7 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -1486,7 +1153,8 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "never", + "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -1501,7 +1169,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -1509,7 +1178,7 @@ } ] }, - "unit": "s" + "unit": "none" }, "overrides": [ { @@ -1521,16 +1190,29 @@ "value": 0 } }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } + "properties": [] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 } - ] + }, + "properties": [] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [] }, { "matcher": { @@ -1541,17 +1223,215 @@ "value": 0 } }, + "properties": [] + }, + { + "matcher": { + "id": "byName", + "options": "avg" + }, "properties": [ { - "id": "custom.hideFrom", + "id": "color", "value": { - "legend": true, - "tooltip": true, - "viz": false + "fixedColor": "#73BF69", + "mode": "fixed" } } ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 15, + "x": 0, + "y": 41 + }, + "id": 130, + "interval": "3s", + "maxDataPoints": 200, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "exemplar": false, + "expr": "rate(tron:block_transaction_count_sum{instance=\"$instance\"}[$__rate_interval]) / (rate(tron:block_transaction_count_count[$__rate_interval]))", + "instant": false, + "interval": "6s", + "legendFormat": "{{miner}}", + "refId": "C" + } + ], + "title": "Transaction Cnt Per Block", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Number of blocks with 20 or fewer transactions produced by each miner in the selected dashboard time range. One row per miner; only miners with at least one low-transaction block are shown.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Low-Transaction Blocks" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 15, + "y": 41 + }, + "id": 131, + "options": { + "showHeader": true + }, + "targets": [ + { + "datasource": "${datasource}", + "exemplar": false, + "expr": "sum by (miner) (increase(tron:block_transaction_count_bucket{group=\"$group\", subgroup=\"$subgroup\", instance=\"$instance\", le=\"20.0\"}[$__range])) > 0", + "format": "table", + "instant": true, + "refId": "A" + } + ], + "title": "Low-Transaction Blocks (≤20 tx)", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Low-Transaction Blocks", + "miner": "Miner" + } + } + }, + { + "id": "sortBy", + "options": { + "sort": [ + { + "desc": true, + "displayName": "Low-Transaction Blocks" + } + ] + } + } + ], + "type": "table" + }, + { + "datasource": "${datasource}", + "description": "Average transaction TPS from broadcasted transactions, not from blocks.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] }, + "unit": "short" + }, + "overrides": [ { "matcher": { "id": "byValue", @@ -1631,6 +1511,32 @@ } } ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "total", + "AccountUpdateContract", + "AccountUpdateContract" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": true, + "viz": true + } + } + ] } ] }, @@ -1638,10 +1544,10 @@ "h": 8, "w": 12, "x": 12, - "y": 34 + "y": 17 }, - "id": 101, - "interval": "1m", + "id": 135, + "interval": "3s", "maxDataPoints": 500, "options": { "alertThreshold": true, @@ -1652,43 +1558,41 @@ ], "displayMode": "table", "placement": "bottom", - "showLegend": true + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", - "sort": "none" + "sort": "desc" } }, - "pluginVersion": "11.6.0", "targets": [ { - "editorMode": "code", "exemplar": true, - "expr": " rate(tron:block_process_latency_seconds_sum{group=`$group`,instance=`$instance`,sync =\"false\"}[$__interval])\n/\n rate(tron:block_process_latency_seconds_count{group=`$group`,instance=`$instance`,sync =\"false\"}[$__interval])", - "hide": false, - "interval": "1m", - "legendFormat": "avg", - "range": true, + "expr": "sum(rate(tron:process_transaction_latency_seconds_count{group=\"$group\", instance=\"$instance\", type!=\"block\"}[$__rate_interval]))", + "interval": "6s", + "legendFormat": "total", "refId": "B" }, { "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tron:block_process_latency_seconds_bucket{group=`$group`,instance=`$instance`,sync =\"false\"}[$__interval])) by (le))", + "expr": "rate(tron:process_transaction_latency_seconds_count{group=\"$group\", instance=\"$instance\", type!=\"block\"}[$__rate_interval])", "format": "time_series", "instant": false, - "interval": "1m", + "interval": "6s", "intervalFactor": 1, - "legendFormat": "tp99", + "legendFormat": "{{contract}}", "refId": "A" } ], - "title": "Block Process Latency", + "title": "Transaction TPS Broadcast", "type": "timeseries" }, { "datasource": "${datasource}", - "description": "Average block processing time when is not catching up.", + "description": "Average transaction TPS from blocks.", "fieldConfig": { "defaults": { "color": { @@ -1718,6 +1622,7 @@ "type": "linear" }, "showPoints": "never", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -1732,7 +1637,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -1740,7 +1646,7 @@ } ] }, - "unit": "s" + "unit": "short" }, "overrides": [ { @@ -1824,41 +1730,39 @@ ] }, { + "__systemRef": "hideSeriesFrom", "matcher": { - "id": "byValue", + "id": "byNames", "options": { - "op": "gte", - "reducer": "allIsZero", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - }, - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 + "mode": "exclude", + "names": [ + "total", + "AccountUpdateContract", + "AssetIssueContract", + "UpdateAssetContract", + "UpdateBrokerageContract", + "UpdateSettingContract", + "WitnessCreateContract", + "WitnessUpdateContract", + "AccountUpdateContract", + "AssetIssueContract", + "UpdateAssetContract", + "UpdateBrokerageContract", + "UpdateSettingContract", + "WitnessCreateContract", + "WitnessUpdateContract" + ], + "prefix": "All except:", + "readOnly": true } }, "properties": [ { "id": "custom.hideFrom", "value": { - "legend": true, + "legend": false, "tooltip": true, - "viz": false + "viz": true } } ] @@ -1869,10 +1773,10 @@ "h": 8, "w": 12, "x": 0, - "y": 110 + "y": 25 }, - "id": 102, - "interval": "1m", + "id": 136, + "interval": "3s", "maxDataPoints": 500, "options": { "alertThreshold": true, @@ -1883,44 +1787,40 @@ ], "displayMode": "table", "placement": "bottom", - "showLegend": true + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", - "sort": "none" + "sort": "desc" } }, - "pluginVersion": "11.6.0", "targets": [ { - "editorMode": "code", "exemplar": true, - "expr": " rate(tron:block_process_latency_seconds_sum{group=`$group`,instance=`$instance`,sync =\"true\"}[$__interval])\n/\n rate(tron:block_process_latency_seconds_count{group=`$group`,instance=`$instance`,sync =\"true\"}[$__interval])", - "hide": false, - "interval": "1m", - "legendFormat": "avg", - "range": true, + "expr": "sum(rate(tron:process_transaction_latency_seconds_count{group=\"$group\", instance=\"$instance\", type=\"block\"}[$__rate_interval]))", + "interval": "6s", + "legendFormat": "total", "refId": "B" }, { - "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tron:block_process_latency_seconds_bucket{group=`$group`,instance=`$instance`,sync =\"true\"}[$__interval])) by (le))", + "expr": "rate(tron:process_transaction_latency_seconds_count{group=\"$group\", instance=\"$instance\", type=\"block\"}[$__rate_interval])", "format": "time_series", "instant": false, - "interval": "1m", + "interval": "6s", "intervalFactor": 1, - "legendFormat": "tp99", + "legendFormat": "{{contract}}", "refId": "A" } ], - "title": "Block Process Latency From Sync", + "title": "Transaction TPS from Block", "type": "timeseries" }, { "datasource": "${datasource}", - "description": "Average parallel verification transactions signature processing time.", "fieldConfig": { "defaults": { "color": { @@ -1930,7 +1830,7 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "latency", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, @@ -1950,6 +1850,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -1964,7 +1865,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -1972,60 +1874,132 @@ } ] }, - "unit": "s" + "unit": "short" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] }, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 110 + "x": 0, + "y": 151 }, - "id": 7, - "interval": "1m", - "maxDataPoints": 500, + "id": 112, + "interval": "3s", "options": { "legend": { "calcs": [ - "mean" + "mean", + "max" ], - "displayMode": "list", + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, - "mode": "multi", + "mode": "single", "sort": "none" } }, - "pluginVersion": "11.6.0", "targets": [ { + "datasource": "${datasource}", "exemplar": true, - "expr": " rate(tron:verify_sign_latency_seconds_sum{group=`$group`,instance=`$instance`}[$__interval])\n/\n rate(tron:verify_sign_latency_seconds_count{group=`$group`,instance=`$instance`}[$__interval])", - "format": "time_series", - "hide": false, - "interval": "1m", - "legendFormat": "avg", - "refId": "D" + "expr": "sum(rate(tron:process_transaction_latency_seconds_count{group=\"$group\", instance=\"$instance\", type=\"block\"}[$__rate_interval]))", + "interval": "6s", + "legendFormat": "total", + "refId": "A" }, { + "datasource": "${datasource}", "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(tron:verify_sign_latency_seconds_bucket{group=`$group`,instance=`$instance`}[$__interval])) by (le))", - "hide": false, - "interval": "1m", - "legendFormat": "tp99", + "expr": "sum(rate(tron:process_transaction_latency_seconds_count{group=\"$group\", instance=\"$instance\", type=\"block\"}[$__rate_interval] offset 1w))", + "instant": false, + "interval": "6s", + "legendFormat": "total-7d", "refId": "C" + }, + { + "datasource": "${datasource}", + "exemplar": true, + "expr": "rate(tron:process_transaction_latency_seconds_count{group=\"$group\", instance=\"$instance\", type=\"block\"}[$__rate_interval])", + "instant": false, + "interval": "6s", + "legendFormat": "{{contract}}", + "refId": "B" + }, + { + "datasource": "${datasource}", + "exemplar": true, + "expr": "rate(tron:process_transaction_latency_seconds_count{group=\"$group\", instance=\"$instance\", type=\"block\"}[$__rate_interval] offset 1w)", + "instant": false, + "interval": "6s", + "legendFormat": "{{contract}}-7d", + "refId": "D" } ], - "title": "Verify Transactions Signature Latency", + "title": "Transaction TPS in Blocks", "type": "timeseries" - }, + } + ], + "title": "Txs", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 38, + "panels": [ { "datasource": "${datasource}", - "description": "Average block processing time.", + "description": "Average block generate time .", "fieldConfig": { "defaults": { "color": { @@ -2140,46 +2114,6 @@ } ] }, - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - }, - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsZero", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - }, { "matcher": { "id": "byValue", @@ -2206,9 +2140,9 @@ "h": 8, "w": 12, "x": 0, - "y": 118 + "y": 34 }, - "id": 103, + "id": 84, "interval": "1m", "maxDataPoints": 500, "options": { @@ -2231,41 +2165,35 @@ "pluginVersion": "11.6.0", "targets": [ { - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": true, - "expr": "rate(tron:block_push_latency_seconds_sum{group=\"$group\", instance=\"$instance\"}[$__interval]) / rate(tron:block_push_latency_seconds_count{group=\"$group\", instance=\"$instance\", sync=\"true\"}[$__interval])", - "fullMetaSearch": false, + "editorMode": "code", + "exemplar": false, + "expr": " rate(tron:block_generate_latency_seconds_sum{group=`$group`,instance=`$instance`}[$__interval])\n/\n rate(tron:block_generate_latency_seconds_count{group=`$group`,instance=`$instance`}[$__interval])", + "format": "time_series", "hide": false, - "includeNullMetadata": true, - "interval": "1m", - "legendFormat": "avg", + "interval": "12s", + "legendFormat": "__auto", "range": true, - "refId": "B", - "useBackend": false + "refId": "B" }, { - "disableTextWrap": false, - "editorMode": "builder", + "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(0.99, sum by(le) (rate(tron:block_push_latency_seconds_bucket{group=\"$group\", instance=\"$instance\"}[$__interval])))", + "expr": "histogram_quantile(0.99, sum(rate(tron:block_generate_latency_seconds_bucket{group=`$group`,instance=`$instance`}[$__interval])) by (le))", "format": "time_series", - "fullMetaSearch": false, - "includeNullMetadata": true, + "hide": false, "instant": false, "interval": "1m", "intervalFactor": 1, "legendFormat": "tp99", - "refId": "A", - "useBackend": false + "refId": "A" } ], - "title": "Block Push Latency ", + "title": "Block Generate Latency", "type": "timeseries" }, { "datasource": "${datasource}", - "description": "When handling the above block push logic, Tron's processing logic needs to acquire a synchronized lock. ", + "description": "Average block processing time when is not catching up.", "fieldConfig": { "defaults": { "color": { @@ -2445,10 +2373,10 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 118 + "x": 0, + "y": 110 }, - "id": 104, + "id": 102, "interval": "1m", "maxDataPoints": 500, "options": { @@ -2471,208 +2399,2430 @@ "pluginVersion": "11.6.0", "targets": [ { - "disableTextWrap": false, "editorMode": "code", "exemplar": true, - "expr": "rate(tron:lock_acquire_latency_seconds_sum{group=\"$group\", instance=\"$instance\"}[$__interval]) / rate(tron:lock_acquire_latency_seconds_count{group=\"$group\", instance=\"$instance\"}[$__interval])", - "fullMetaSearch": false, + "expr": " rate(tron:block_process_latency_seconds_sum{group=`$group`,instance=`$instance`,sync =\"true\"}[$__interval])\n/\n rate(tron:block_process_latency_seconds_count{group=`$group`,instance=`$instance`,sync =\"true\"}[$__interval])", "hide": false, - "includeNullMetadata": true, "interval": "1m", "legendFormat": "avg", "range": true, - "refId": "B", - "useBackend": false + "refId": "B" }, { - "disableTextWrap": false, "editorMode": "code", "exemplar": true, - "expr": "histogram_quantile(0.99, sum by(le) (rate(tron:lock_acquire_latency_seconds_bucket{group=\"$group\", instance=\"$instance\"}[$__interval])))", + "expr": "histogram_quantile(0.99, sum(rate(tron:block_process_latency_seconds_bucket{group=`$group`,instance=`$instance`,sync =\"true\"}[$__interval])) by (le))", "format": "time_series", - "fullMetaSearch": false, - "includeNullMetadata": true, "instant": false, "interval": "1m", "intervalFactor": 1, "legendFormat": "tp99", - "refId": "A", - "useBackend": false + "refId": "A" } ], - "title": "Block Push Lock Acquire Latency", + "title": "Block Process Latency From Sync", "type": "timeseries" - } - ], - "title": "Block", - "type": "row" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 19 - }, - "id": 46, - "panels": [], - "title": "Net", - "type": "row" - }, - { - "datasource": "${datasource}", - "description": "Node peers Info.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [], - "noValue": "N/A", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 20 - }, - "id": 20, - "interval": "3s", - "maxDataPoints": 500, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto", - "text": {} - }, - "pluginVersion": "11.6.0", - "targets": [ { - "editorMode": "builder", - "exemplar": true, - "expr": "tron:peers{group=`$group`,instance=`$instance`}", - "format": "time_series", - "instant": false, - "interval": "6s", - "intervalFactor": 1, - "legendFormat": "{{type}}", - "refId": "A" - } - ], - "title": "Peers", - "type": "gauge" - }, - { - "datasource": "${datasource}", - "description": "Http network data traffic statistics.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "size", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "datasource": "${datasource}", + "description": "Average parallel verification transactions signature processing time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "latency", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 110 + }, + "id": 7, + "interval": "1m", + "maxDataPoints": 500, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "solidity005-fail-expired" + "pluginVersion": "11.6.0", + "targets": [ + { + "exemplar": true, + "expr": " rate(tron:verify_sign_latency_seconds_sum{group=`$group`,instance=`$instance`}[$__interval])\n/\n rate(tron:verify_sign_latency_seconds_count{group=`$group`,instance=`$instance`}[$__interval])", + "format": "time_series", + "hide": false, + "interval": "1m", + "legendFormat": "avg", + "refId": "D" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "semi-dark-yellow", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 20 - }, - "id": 17, - "interval": "10m", + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(tron:verify_sign_latency_seconds_bucket{group=`$group`,instance=`$instance`}[$__interval])) by (le))", + "hide": false, + "interval": "1m", + "legendFormat": "tp99", + "refId": "C" + } + ], + "title": "Verify Transactions Signature Latency", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Average block processing time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 118 + }, + "id": 103, + "interval": "1m", + "maxDataPoints": 500, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": true, + "expr": "rate(tron:block_push_latency_seconds_sum{group=\"$group\", instance=\"$instance\"}[$__rate_interval]) / rate(tron:block_push_latency_seconds_count{group=\"$group\", instance=\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "interval": "1m", + "legendFormat": "avg", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by(le) (rate(tron:block_push_latency_seconds_bucket{group=\"$group\", instance=\"$instance\"}[$__interval])))", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "tp99", + "refId": "A", + "useBackend": false + } + ], + "title": "Block Push Latency ", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "When handling the above block push logic, Tron's processing logic needs to acquire a synchronized lock. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 118 + }, + "id": 104, + "interval": "1m", + "maxDataPoints": 500, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "code", + "exemplar": true, + "expr": "rate(tron:lock_acquire_latency_seconds_sum{group=\"$group\", instance=\"$instance\"}[$__interval]) / rate(tron:lock_acquire_latency_seconds_count{group=\"$group\", instance=\"$instance\"}[$__interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "interval": "1m", + "legendFormat": "avg", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "disableTextWrap": false, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.99, sum by(le) (rate(tron:lock_acquire_latency_seconds_bucket{group=\"$group\", instance=\"$instance\"}[$__interval])))", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "tp99", + "refId": "A", + "useBackend": false + } + ], + "title": "Block Push Lock Acquire Latency", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "The total Block Fork Count after startup", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 74 + }, + "id": 127, + "interval": "1m", + "maxDataPoints": 500, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "tron:block_fork_total{group=\"$group\", instance=\"$instance\"}", + "interval": "1m", + "legendFormat": "avg", + "refId": "B" + } + ], + "title": "Block Fork Count", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 74 + }, + "id": 128, + "interval": "1m", + "maxDataPoints": 500, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "changes(tron:miner_total{group=\"$group\", type=\"miss\", instance=\"$instance\"}[1m])", + "interval": "1m", + "legendFormat": "avg", + "refId": "B" + } + ], + "title": "Block Miss change", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Average block processing time when is not catching up.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 50 + }, + "id": 137, + "interval": "1m", + "maxDataPoints": 500, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "exemplar": true, + "expr": "irate(tron:block_process_latency_seconds_sum{group=\"$group\", instance=\"$instance\", sync=\"false\"}[$__interval]) / irate(tron:block_process_latency_seconds_count{group=\"$group\", instance=\"$instance\", sync=\"false\"}[$__interval])", + "interval": "1m", + "legendFormat": "avg", + "refId": "B" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(tron:block_process_latency_seconds_bucket{group=`$group`,instance=`$instance`,sync =\"false\"}[$__interval])) by (le))", + "format": "time_series", + "instant": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "tp99", + "refId": "A" + } + ], + "title": "Block Process Latency From Broadcast", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "The latency calculated by the FullNode using nowTime - block.getTimeStamp(). Different FullNode may received differently.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 7, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 110, + "interval": "2m", + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${datasource}", + "exemplar": false, + "expr": "rate(tron:miner_latency_seconds_sum{group=\"$group\", instance=\"$instance\"}[$__rate_interval]) / rate(tron:miner_latency_seconds_count{group=\"$group\", instance=\"$instance\"}[$__rate_interval])", + "format": "time_series", + "instant": true, + "legendFormat": "{{miner}}", + "refId": "A" + } + ], + "title": "SR Block Latency", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "The average time to fetch a block from a peer. \n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 132 + }, + "id": 124, + "options": { + "legend": { + "calcs": [ + "median", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Count", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "expr": "rate(tron:block_fetch_latency_seconds_sum{group=\"$group\"}[$__interval]) / (rate(tron:block_fetch_latency_seconds_count{group=\"$group\", instance=\"$instance\"}[$__interval]))", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Block Fetch Latency second", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "The worst time to fetch a block from a peer. \n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 132 + }, + "id": 129, + "options": { + "legend": { + "calcs": [ + "count" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Count", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by(le, instance) (rate(tron:block_fetch_latency_seconds_bucket{group=\"$group\"}[$__rate_interval]))) >= 2.5", + "instant": false, + "legendFormat": "{{instance}} P99", + "refId": "B" + } + ], + "title": "Block Fetch Latency > 2.5 counter", + "type": "timeseries" + } + ], + "title": "Block", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 115, + "title": "SR Set", + "type": "row" + }, + { + "datasource": "${datasource}", + "description": "Counts active-SR membership additions and removals observed by the selected node during the selected dashboard time range. A syncing or replaying node may report historical changes.", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "action" + }, + "properties": [ + { + "id": "displayName", + "value": "Action" + }, + { + "id": "custom.width", + "value": 80 + }, + { + "id": "mappings", + "value": [ + { + "options": { + "add": { + "color": "green", + "text": "ADD" + }, + "remove": { + "color": "red", + "text": "REMOVE" + } + }, + "type": "value" + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "witness" + }, + "properties": [ + { + "id": "displayName", + "value": "Witness" + }, + { + "id": "custom.width", + "value": 360 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "displayName", + "value": "Count" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 0 + }, + { + "id": "custom.width", + "value": 80 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 116, + "options": { + "cellHeight": "sm", + "footer": { + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Count" + } + ] + }, + "targets": [ + { + "exemplar": true, + "expr": "sum by (action, witness) (increase(tron:sr_set_change_total{group=\"$group\", subgroup=\"$subgroup\", instance=\"$instance\"}[$__range])) > 0", + "format": "table", + "instant": true, + "refId": "A" + } + ], + "title": "SR Set Change Counts", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "group": true, + "instance": true, + "job": true, + "monitor": true, + "receive_cluster": true, + "subgroup": true, + "tenant_id": true + }, + "indexByName": { + "Time": 0, + "Value": 3, + "action": 1, + "witness": 2 + } + } + } + ], + "type": "table" + }, + { + "datasource": "${datasource}", + "description": "Total active-SR membership additions and removals observed by the selected node during the selected dashboard time range. A syncing or replaying node may report historical changes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "SR Added" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SR Removed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 117, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "exemplar": true, + "expr": "sum(increase(tron:sr_set_change_total{group=\"$group\", subgroup=\"$subgroup\", instance=\"$instance\", action=\"add\"}[$__range])) or vector(0)", + "instant": true, + "legendFormat": "SR Added", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(increase(tron:sr_set_change_total{group=\"$group\", subgroup=\"$subgroup\", instance=\"$instance\", action=\"remove\"}[$__range])) or vector(0)", + "instant": true, + "legendFormat": "SR Removed", + "refId": "B" + } + ], + "title": "SR Set Change Total", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 46, + "title": "Net", + "type": "row" + }, + { + "datasource": "${datasource}", + "description": "Node peers Info.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "noValue": "N/A", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 20, + "interval": "3s", + "maxDataPoints": 500, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "text": {} + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "editorMode": "builder", + "exemplar": true, + "expr": "tron:peers{group=`$group`,instance=`$instance`}", + "format": "time_series", + "instant": false, + "interval": "6s", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "title": "Peers", + "type": "gauge" + }, + { + "datasource": "${datasource}", + "description": "Http network data traffic statistics.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "size", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "solidity005-fail-expired" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 17, + "interval": "10m", + "maxDataPoints": 500, + "options": { + "legend": { + "calcs": [ + "sum", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "increase(tron:http_bytes_sum{group=\"$group\", instance=\"$instance\"}[$__interval])", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "{{url}}.{{status}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "HTTP Traffic /1 Minute", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "Node abnormal peers statstics during past 1 minute", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "noValue": "N/A", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 46 + }, + "id": 108, + "interval": "3s", + "maxDataPoints": 500, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "text": {} + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": true, + "expr": "increase(tron:p2p_disconnect_total[$__rate_interval])", + "format": "time_series", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A", + "useBackend": false + }, + { + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "increase(tron:p2p_error_total[$__rate_interval])", + "format": "time_series", + "fullMetaSearch": false, + "hide": true, + "includeNullMetadata": true, + "instant": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Peers Abnormal/1min", + "type": "gauge" + }, + { + "datasource": "${datasource}", + "description": "UDP network data traffic statistics rate over the past 30 seconds.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "solidity005-fail-expired" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 46 + }, + "id": 107, + "interval": "10m", + "maxDataPoints": 500, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "increase(tron:udp_bytes_sum{group=`$group`,instance=`$instance`}[$__interval])", + "format": "time_series", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "interval": "30s", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "expr": "", + "hide": false, + "instant": false, + "range": true, + "refId": "B" + } + ], + "title": "UDP Traffic/s", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "TCP network data traffic statistics, average TCP bytes per second over the past 30 seconds.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "solidity005-fail-expired" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 54 + }, + "id": 16, + "interval": "10m", "maxDataPoints": 500, "options": { "legend": { - "calcs": [ - "sum", - "mean" - ], - "displayMode": "table", + "calcs": [ + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": true, + "expr": "rate(tron:tcp_bytes_sum{group=`$group`,instance=`$instance`}[$__interval])", + "format": "time_series", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "interval": "30s", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A", + "useBackend": false + } + ], + "title": "TCP Traffic Rate/s", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "The average time to process a message received from a peer, when the message processing time exceeds 50ms, will be logged by the below metric. \n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 54 + }, + "id": 106, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", "placement": "bottom", "showLegend": true }, @@ -2685,461 +4835,834 @@ "pluginVersion": "11.6.0", "targets": [ { - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "increase(tron:http_bytes_sum{group=\"$group\", instance=\"$instance\"}[$__interval])", - "format": "time_series", - "fullMetaSearch": false, - "includeNullMetadata": true, + "editorMode": "code", + "expr": "rate(tron:message_process_latency_seconds_sum{group=\"$group\"}[$__interval])/ rate(tron:message_process_latency_seconds_count{group=\"$group\", instance=\"$instance\"}[$__interval])\n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(le) (rate(tron:message_process_latency_seconds_bucket{group=\"$group\", instance=\"$instance\"}[$__interval])))\n", + "hide": false, "instant": false, - "interval": "1m", - "intervalFactor": 1, - "legendFormat": "{{url}}.{{status}}", + "legendFormat": "__auto", "range": true, - "refId": "A", - "useBackend": false + "refId": "B" } ], - "title": "HTTP Traffic /1 Minute", + "title": "Message Lantency > 50ms", "type": "timeseries" }, { - "datasource": "${datasource}", - "description": "Node abnormal peers statstics during past 1 minute", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 60 + }, + "id": 44, + "panels": [ + { + "datasource": "${datasource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 76 + }, + "id": 99, + "interval": "3s", + "maxDataPoints": 500, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(tron:http_service_latency_seconds_count{group=`$group`,instance=`$instance`}[$__interval]))", + "hide": false, + "interval": "6s", + "legendFormat": "total", + "range": true, + "refId": "B" + }, + { + "editorMode": "code", + "exemplar": true, + "expr": "rate(tron:http_service_latency_seconds_count{group=`$group`,instance=`$instance`}[$__interval])", + "format": "time_series", + "instant": false, + "interval": "6s", + "intervalFactor": 1, + "legendFormat": "{{url}}", + "refId": "A" + } + ], + "title": " HTTP QPS", + "type": "timeseries" + }, + { + "datasource": "${datasource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 76 + }, + "id": 8, + "interval": "1m", + "maxDataPoints": 500, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } }, - "mappings": [], - "noValue": "N/A", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 28 - }, - "id": 108, - "interval": "3s", - "maxDataPoints": 500, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" + "pluginVersion": "11.6.0", + "targets": [ + { + "exemplar": true, + "expr": "rate(tron:http_service_latency_seconds_sum{group=`$group`,instance=`$instance`}[$__interval])/rate(tron:http_service_latency_seconds_count{group=`$group`,instance=`$instance`}[$__interval])", + "format": "time_series", + "instant": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "{{url}}", + "refId": "A" + } ], - "fields": "", - "values": false + "title": "HTTP Latency", + "type": "timeseries" }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto", - "text": {} - }, - "pluginVersion": "11.6.0", - "targets": [ { - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": true, - "expr": "increase(tron:p2p_disconnect_total[$__rate_interval])", - "format": "time_series", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "interval": "1m", - "intervalFactor": 1, - "legendFormat": "{{type}}", - "refId": "A", - "useBackend": false + "datasource": "${datasource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 112 + }, + "id": 100, + "interval": "3s", + "maxDataPoints": 500, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(tron:grpc_service_latency_seconds_sum{group=`$group`,instance=`$instance`}[$__interval]))", + "hide": false, + "interval": "6s", + "legendFormat": "total", + "range": true, + "refId": "B" + }, + { + "editorMode": "code", + "exemplar": true, + "expr": "rate(tron:grpc_service_latency_seconds_count{group=`$group`,instance=`$instance`}[$__interval])", + "format": "time_series", + "instant": false, + "interval": "6s", + "intervalFactor": 1, + "legendFormat": "{{endpoint}}", + "refId": "A" + } + ], + "title": " GRPC QPS", + "type": "timeseries" }, { - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "increase(tron:p2p_error_total[$__rate_interval])", - "format": "time_series", - "fullMetaSearch": false, - "hide": true, - "includeNullMetadata": true, - "instant": false, + "datasource": "${datasource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 112 + }, + "id": 10, "interval": "1m", - "intervalFactor": 1, - "legendFormat": "{{type}}", - "range": true, - "refId": "B", - "useBackend": false + "maxDataPoints": 500, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "rate(tron:grpc_service_latency_seconds_sum{group=`$group`,instance=`$instance`}[$__interval])/rate(tron:grpc_service_latency_seconds_count{group=`$group`,instance=`$instance`}[$__interval])", + "format": "time_series", + "instant": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "{{endpoint}}", + "refId": "A" + } + ], + "title": "GRPC Latency", + "type": "timeseries" } ], - "title": "Peers Abnormal/1min", - "type": "gauge" + "title": "API", + "type": "row" }, { - "datasource": "${datasource}", - "description": "UDP network data traffic statistics rate over the past 30 seconds.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 61 + }, + "id": 48, + "panels": [ + { + "datasource": "${datasource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "N/A", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" }, - "thresholdsStyle": { - "mode": "off" - } + "overrides": [] }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 77 }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "solidity005-fail-expired" + "id": 56, + "interval": "5m", + "maxDataPoints": 1000, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "semi-dark-yellow", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 28 - }, - "id": 107, - "interval": "10m", - "maxDataPoints": 500, - "options": { - "legend": { - "calcs": [ - "mean", - "max" + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": "${datasource}", + "exemplar": true, + "expr": "sum(tron:db_size_bytes{group=`$group`,instance=`$instance`})", + "format": "time_series", + "instant": false, + "interval": "2m", + "intervalFactor": 1, + "legendFormat": "total", + "refId": "A" + }, + { + "datasource": "${datasource}", + "exemplar": true, + "expr": "sum(delta(tron:db_size_bytes{group=`$group`,instance=`$instance`}[24h]))", + "hide": false, + "interval": "", + "legendFormat": "delta/24H", + "refId": "B" + } ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "increase(tron:udp_bytes_sum{group=`$group`,instance=`$instance`}[$__interval])", - "format": "time_series", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "interval": "30s", - "intervalFactor": 1, - "legendFormat": "{{type}}", - "range": true, - "refId": "A", - "useBackend": false + "title": "total", + "type": "stat" }, { - "expr": "", - "hide": false, - "instant": false, - "range": true, - "refId": "B" - } - ], - "title": "UDP Traffic/s", - "type": "timeseries" - }, - { - "datasource": "${datasource}", - "description": "TCP network data traffic statistics, average TCP bytes per second over the past 30 seconds.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "datasource": "${datasource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "noValue": "N/A", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 83 + }, + "id": 21, + "interval": "5m", + "maxDataPoints": 1000, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": "${datasource}", + "exemplar": true, + "expr": "sum(tron:db_size_bytes{group=`$group`,instance=`$instance`,db=\"block\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "block", + "refId": "A" }, - "thresholdsStyle": { - "mode": "off" + { + "datasource": "${datasource}", + "exemplar": true, + "expr": "sum(delta(tron:db_size_bytes{group=`$group`,instance=`$instance`,db=\"block\"}[24h]))", + "hide": false, + "interval": "", + "legendFormat": "delta/24H", + "refId": "B" } + ], + "title": "block", + "type": "stat" + }, + { + "datasource": "${datasource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "noValue": "N/A", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 83 }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "solidity005-fail-expired" + "id": 51, + "interval": "5m", + "maxDataPoints": 1000, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "semi-dark-yellow", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 36 - }, - "id": 16, - "interval": "10m", - "maxDataPoints": 500, - "options": { - "legend": { - "calcs": [ - "mean", - "max" + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": "${datasource}", + "exemplar": true, + "expr": "sum(tron:db_size_bytes{group=`$group`,instance=`$instance`,db=\"transactionRetStore\"})", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "transactionRetStore", + "refId": "A" + }, + { + "datasource": "${datasource}", + "exemplar": true, + "expr": "sum(delta(tron:db_size_bytes{group=`$group`,instance=`$instance`,db=\"transactionRetStore\"}[24h]))", + "hide": false, + "interval": "", + "legendFormat": "delta/24H", + "refId": "B" + } ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "title": "transactionRetStore", + "type": "stat" }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ { - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": true, - "expr": "rate(tron:tcp_bytes_sum[$__interval])", - "format": "time_series", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "interval": "30s", - "intervalFactor": 1, - "legendFormat": "{{type}}", - "refId": "A", - "useBackend": false - } - ], - "title": "TCP Traffic Rate/s", - "type": "timeseries" - }, - { - "datasource": "${datasource}", - "description": "The average time to process a message received from a peer, when the message processing time exceeds 50ms, will be logged by the below metric. \n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "datasource": "${datasource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "noValue": "N/A", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 89 + }, + "id": 49, + "interval": "5m", + "maxDataPoints": 1000, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": "${datasource}", + "exemplar": true, + "expr": "sum(tron:db_size_bytes{group=`$group`,instance=`$instance`,db=\"trans\"})", + "format": "time_series", + "instant": false, + "interval": "2m", + "intervalFactor": 1, + "legendFormat": "trans", + "refId": "A" }, - "thresholdsStyle": { - "mode": "off" + { + "datasource": "${datasource}", + "exemplar": true, + "expr": "sum(delta(tron:db_size_bytes{group=`$group`,instance=`$instance`,db=\"trans\"}[24h]))", + "hide": false, + "interval": "", + "legendFormat": "delta/24H", + "refId": "B" } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 36 - }, - "id": 106, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + ], + "title": "trans", + "type": "stat" }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ { - "editorMode": "code", - "expr": "rate(tron:message_process_latency_seconds_sum{group=\"$group\"}[$__interval])/ rate(tron:message_process_latency_seconds_count{group=\"$group\", instance=\"$instance\"}[$__interval])\n", - "legendFormat": "__auto", - "range": true, - "refId": "A" + "datasource": "${datasource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "noValue": "N/A", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 89 + }, + "id": 50, + "interval": "5m", + "maxDataPoints": 1000, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": "${datasource}", + "exemplar": true, + "expr": "sum(tron:db_size_bytes{group=`$group`,instance=`$instance`,db=\"transactionHistoryStore\"})", + "format": "time_series", + "instant": false, + "interval": "2m", + "intervalFactor": 1, + "legendFormat": "transactionHistoryStore", + "refId": "A" + }, + { + "datasource": "${datasource}", + "exemplar": true, + "expr": "sum(delta(tron:db_size_bytes{group=`$group`,instance=`$instance`,db=\"transactionHistoryStore\"}[24h]))", + "hide": false, + "interval": "", + "legendFormat": "delta/24H", + "refId": "B" + } + ], + "title": "transactionHistoryStore", + "type": "stat" }, - { - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le) (rate(tron:message_process_latency_seconds_bucket{group=\"$group\", instance=\"$instance\"}[$__interval])))\n", - "hide": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "B" - } - ], - "title": "Message Lantency > 50ms", - "type": "timeseries" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 44 - }, - "id": 44, - "panels": [ { "datasource": "${datasource}", "description": "", @@ -3148,104 +5671,147 @@ "color": { "mode": "palette-classic" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mappings": [], + "noValue": "N/A", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 95 + }, + "id": 52, + "interval": "5m", + "maxDataPoints": 1000, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": "${datasource}", + "exemplar": true, + "expr": "sum(tron:db_size_bytes{group=`$group`,instance=`$instance`,db=\"storage-row\"})", + "format": "time_series", + "instant": false, + "interval": "2m", + "intervalFactor": 1, + "legendFormat": "storage-row", + "refId": "A" + }, + { + "datasource": "${datasource}", + "exemplar": true, + "expr": "sum(delta(tron:db_size_bytes{group=`$group`,instance=`$instance`,db=\"storage-row\"}[24h]))", + "hide": false, + "interval": "", + "legendFormat": "delta/24H", + "refId": "B" + } + ], + "title": "storage-row", + "type": "stat" + }, + { + "datasource": "${datasource}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, "mappings": [], + "noValue": "N/A", "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] - } + }, + "unit": "bytes" }, "overrides": [] }, "gridPos": { - "h": 8, + "h": 5, "w": 12, - "x": 0, - "y": 60 + "x": 12, + "y": 95 }, - "id": 99, - "interval": "3s", - "maxDataPoints": 500, + "id": 53, + "interval": "5m", + "maxDataPoints": 1000, "options": { - "alertThreshold": true, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true }, "pluginVersion": "11.6.0", "targets": [ { - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(tron:http_service_latency_seconds_count{group=`$group`,instance=`$instance`}[$__interval]))", - "hide": false, - "interval": "6s", - "legendFormat": "total", - "range": true, - "refId": "B" - }, - { - "editorMode": "code", + "datasource": "${datasource}", "exemplar": true, - "expr": "rate(tron:http_service_latency_seconds_count{group=`$group`,instance=`$instance`}[$__interval])", + "expr": "sum(tron:db_size_bytes{group=`$group`,instance=`$instance`,db=\"account\"})", "format": "time_series", "instant": false, - "interval": "6s", + "interval": "2m", "intervalFactor": 1, - "legendFormat": "{{url}}", + "legendFormat": "account", "refId": "A" + }, + { + "datasource": "${datasource}", + "exemplar": true, + "expr": "sum(delta(tron:db_size_bytes{group=`$group`,instance=`$instance`,db=\"account\"}[24h]))", + "hide": false, + "interval": "", + "legendFormat": "delta/24H", + "refId": "B" } ], - "title": " HTTP QPS", - "type": "timeseries" + "title": "account", + "type": "stat" }, { "datasource": "${datasource}", @@ -3255,94 +5821,72 @@ "color": { "mode": "palette-classic" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, "mappings": [], + "noValue": "N/A", "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "s" + "unit": "bytes" }, "overrides": [] }, "gridPos": { - "h": 8, + "h": 5, "w": 12, - "x": 12, - "y": 60 + "x": 0, + "y": 100 }, - "id": 8, - "interval": "1m", - "maxDataPoints": 500, + "id": 54, + "interval": "5m", + "maxDataPoints": 1000, "options": { - "alertThreshold": true, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true }, "pluginVersion": "11.6.0", "targets": [ { + "datasource": "${datasource}", "exemplar": true, - "expr": "rate(tron:http_service_latency_seconds_sum{group=`$group`,instance=`$instance`}[$__interval])/rate(tron:http_service_latency_seconds_count{group=`$group`,instance=`$instance`}[$__interval])", + "expr": "sum(tron:db_size_bytes{group=`$group`,instance=`$instance`,db=\"delegation\"})", "format": "time_series", "instant": false, - "interval": "1m", + "interval": "2m", "intervalFactor": 1, - "legendFormat": "{{url}}", + "legendFormat": "delegation", "refId": "A" + }, + { + "datasource": "${datasource}", + "exemplar": true, + "expr": "sum(delta(tron:db_size_bytes{group=`$group`,instance=`$instance`,db=\"delegation\"}[24h]))", + "hide": false, + "interval": "", + "legendFormat": "delta/24H", + "refId": "B" } ], - "title": "HTTP Latency", - "type": "timeseries" + "title": "delegation", + "type": "stat" }, { "datasource": "${datasource}", @@ -3352,224 +5896,151 @@ "color": { "mode": "palette-classic" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, "mappings": [], + "noValue": "N/A", "thresholds": { "mode": "absolute", "steps": [ { "color": "green" - }, - { - "color": "red", - "value": 80 } ] - } + }, + "unit": "bytes" }, "overrides": [] }, "gridPos": { - "h": 8, + "h": 5, "w": 12, - "x": 0, - "y": 96 + "x": 12, + "y": 100 }, - "id": 100, - "interval": "3s", - "maxDataPoints": 500, + "id": 55, + "interval": "5m", + "maxDataPoints": 1000, "options": { - "alertThreshold": true, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true }, "pluginVersion": "11.6.0", "targets": [ { - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(tron:grpc_service_latency_seconds_sum{group=`$group`,instance=`$instance`}[$__interval]))", - "hide": false, - "interval": "6s", - "legendFormat": "total", - "range": true, - "refId": "B" - }, - { - "editorMode": "code", + "datasource": "${datasource}", "exemplar": true, - "expr": "rate(tron:grpc_service_latency_seconds_count{group=`$group`,instance=`$instance`}[$__interval])", + "expr": "sum(tron:db_size_bytes{group=`$group`,instance=`$instance`,db=\"block-index\"})", "format": "time_series", "instant": false, - "interval": "6s", + "interval": "2m", "intervalFactor": 1, - "legendFormat": "{{endpoint}}", + "legendFormat": "block-index", "refId": "A" + }, + { + "datasource": "${datasource}", + "exemplar": true, + "expr": "sum(delta(tron:db_size_bytes{group=`$group`,instance=`$instance`,db=\"block-index\"}[24h]))", + "hide": false, + "interval": "", + "legendFormat": "delta/24H", + "refId": "B" } ], - "title": " GRPC QPS", - "type": "timeseries" + "title": "block-index", + "type": "stat" }, { "datasource": "${datasource}", - "description": "", + "description": "all storage > 0", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" + "noValue": "N/A", + "unit": "bytes" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 96 + "h": 12, + "w": 17, + "x": 7, + "y": 146 }, - "id": 10, - "interval": "1m", - "maxDataPoints": 500, + "id": 139, + "interval": "5m", + "maxDataPoints": 1000, "options": { - "alertThreshold": true, "legend": { - "calcs": [], - "displayMode": "list", + "displayMode": "table", "placement": "bottom", - "showLegend": true + "showLegend": true, + "sortBy": "Value", + "sortDesc": true, + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, + "sort": "desc", "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "11.6.0", "targets": [ { - "editorMode": "code", + "datasource": "${datasource}", "exemplar": true, - "expr": "rate(tron:grpc_service_latency_seconds_sum{group=`$group`,instance=`$instance`}[$__interval])/rate(tron:grpc_service_latency_seconds_count{group=`$group`,instance=`$instance`}[$__interval])", + "expr": "sum by(db) (tron:db_size_bytes{group=\"$group\", instance=\"$instance\"}) > 0", "format": "time_series", "instant": false, - "interval": "1m", + "interval": "", "intervalFactor": 1, - "legendFormat": "{{endpoint}}", + "legendFormat": "__auto", "refId": "A" } ], - "title": "GRPC Latency", - "type": "timeseries" - } - ], - "title": "API", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 45 - }, - "id": 48, - "panels": [ + "title": "Storage", + "type": "piechart" + }, { "datasource": "${datasource}", - "description": "", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" }, "mappings": [], "noValue": "N/A", @@ -3577,7 +6048,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 } ] }, @@ -3587,11 +6059,11 @@ }, "gridPos": { "h": 6, - "w": 24, - "x": 0, - "y": 61 + "w": 16, + "x": 8, + "y": 188 }, - "id": 56, + "id": 119, "interval": "5m", "maxDataPoints": 1000, "options": { @@ -3612,35 +6084,24 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.6.0", "targets": [ { "datasource": "${datasource}", "exemplar": true, - "expr": "sum(tron:db_size_bytes{group=`$group`,instance=`$instance`,type = \"LEVELDB\"})", + "expr": "sum by(level) (tron:db_size_bytes{group=\"$group\", instance=\"$instance\", db=\"account\"})", "format": "time_series", "instant": false, - "interval": "2m", + "interval": "", "intervalFactor": 1, - "legendFormat": "total", + "legendFormat": "Level - {{level}}", "refId": "A" - }, - { - "datasource": "${datasource}", - "exemplar": true, - "expr": "sum(delta(tron:db_size_bytes{group=`$group`,instance=`$instance`,type = \"LEVELDB\"}[24h]))", - "hide": false, - "interval": "", - "legendFormat": "delta/24H", - "refId": "B" } ], - "title": "total", + "title": "account levels", "type": "stat" }, { "datasource": "${datasource}", - "description": "", "fieldConfig": { "defaults": { "color": { @@ -3652,7 +6113,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 } ] }, @@ -3661,12 +6123,12 @@ "overrides": [] }, "gridPos": { - "h": 6, - "w": 12, + "h": 7, + "w": 8, "x": 0, - "y": 67 + "y": 207 }, - "id": 21, + "id": 125, "interval": "5m", "maxDataPoints": 1000, "options": { @@ -3687,36 +6149,32 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.6.0", "targets": [ { "datasource": "${datasource}", "exemplar": true, - "expr": "sum(tron:db_size_bytes{group=`$group`,instance=`$instance`,type=\"LEVELDB\",db=\"block\"})", + "expr": "sum(tron:db_size_bytes{group=\"$group\", instance=\"$instance\", db=\"account-asset\"})", "format": "time_series", - "hide": false, "instant": false, - "interval": "", + "interval": "2m", "intervalFactor": 1, - "legendFormat": "block", + "legendFormat": "account-asset", "refId": "A" }, { "datasource": "${datasource}", "exemplar": true, - "expr": "sum(delta(tron:db_size_bytes{group=`$group`,instance=`$instance`,type = \"LEVELDB\",db=\"block\"}[24h]))", - "hide": false, + "expr": "sum(delta(tron:db_size_bytes{group=\"$group\", instance=\"$instance\", db=\"account-asset\"}[24h]))", "interval": "", "legendFormat": "delta/24H", "refId": "B" } ], - "title": "block", + "title": "account-asset", "type": "stat" }, { "datasource": "${datasource}", - "description": "", "fieldConfig": { "defaults": { "color": { @@ -3728,7 +6186,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 } ] }, @@ -3737,12 +6196,12 @@ "overrides": [] }, "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 67 + "h": 7, + "w": 16, + "x": 8, + "y": 207 }, - "id": 51, + "id": 126, "interval": "5m", "maxDataPoints": 1000, "options": { @@ -3763,35 +6222,24 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.6.0", "targets": [ { "datasource": "${datasource}", "exemplar": true, - "expr": "sum(tron:db_size_bytes{group=`$group`,instance=`$instance`,type=\"LEVELDB\",db=\"transactionRetStore\"})", + "expr": "sum by(level) (tron:db_size_bytes{group=\"$group\", instance=\"$instance\", db=\"account-asset\"})", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "transactionRetStore", + "legendFormat": "Level - {{level}}", "refId": "A" - }, - { - "datasource": "${datasource}", - "exemplar": true, - "expr": "sum(delta(tron:db_size_bytes{group=`$group`,instance=`$instance`,type = \"LEVELDB\",db=\"transactionRetStore\"}[24h]))", - "hide": false, - "interval": "", - "legendFormat": "delta/24H", - "refId": "B" } ], - "title": "transactionRetStore", + "title": "account-asset levels", "type": "stat" }, { "datasource": "${datasource}", - "description": "", "fieldConfig": { "defaults": { "color": { @@ -3803,7 +6251,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 } ] }, @@ -3813,11 +6262,11 @@ }, "gridPos": { "h": 6, - "w": 12, - "x": 0, - "y": 73 + "w": 16, + "x": 8, + "y": 158 }, - "id": 49, + "id": 114, "interval": "5m", "maxDataPoints": 1000, "options": { @@ -3838,35 +6287,24 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.6.0", "targets": [ { "datasource": "${datasource}", "exemplar": true, - "expr": "sum(tron:db_size_bytes{group=`$group`,instance=`$instance`,type=\"LEVELDB\",db=\"trans\"})", + "expr": "sum by(level) (tron:db_size_bytes{group=\"$group\", instance=\"$instance\", db=\"block\"})", "format": "time_series", "instant": false, - "interval": "2m", + "interval": "", "intervalFactor": 1, - "legendFormat": "trans", + "legendFormat": "Level - {{level}}", "refId": "A" - }, - { - "datasource": "${datasource}", - "exemplar": true, - "expr": "sum(delta(tron:db_size_bytes{group=`$group`,instance=`$instance`,type = \"LEVELDB\",db=\"trans\"}[24h]))", - "hide": false, - "interval": "", - "legendFormat": "delta/24H", - "refId": "B" } ], - "title": "trans", + "title": "block levels", "type": "stat" }, { "datasource": "${datasource}", - "description": "", "fieldConfig": { "defaults": { "color": { @@ -3878,7 +6316,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 } ] }, @@ -3887,12 +6326,12 @@ "overrides": [] }, "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 73 + "h": 7, + "w": 16, + "x": 8, + "y": 200 }, - "id": 50, + "id": 121, "interval": "5m", "maxDataPoints": 1000, "options": { @@ -3913,35 +6352,24 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.6.0", "targets": [ { "datasource": "${datasource}", "exemplar": true, - "expr": "sum(tron:db_size_bytes{group=`$group`,instance=`$instance`,type=\"LEVELDB\",db=\"transactionHistoryStore\"})", + "expr": "sum by(level) (tron:db_size_bytes{group=\"$group\", instance=\"$instance\", db=\"block-index\"})", "format": "time_series", "instant": false, - "interval": "2m", + "interval": "", "intervalFactor": 1, - "legendFormat": "transactionHistoryStore", + "legendFormat": "Level - {{level}}", "refId": "A" - }, - { - "datasource": "${datasource}", - "exemplar": true, - "expr": "sum(delta(tron:db_size_bytes{group=`$group`,instance=`$instance`,type = \"LEVELDB\",db=\"transactionHistoryStore\"}[24h]))", - "hide": false, - "interval": "", - "legendFormat": "delta/24H", - "refId": "B" } ], - "title": "transactionHistoryStore", + "title": "blockIndex levels", "type": "stat" }, { "datasource": "${datasource}", - "description": "", "fieldConfig": { "defaults": { "color": { @@ -3953,7 +6381,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 } ] }, @@ -3962,12 +6391,12 @@ "overrides": [] }, "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 79 + "h": 6, + "w": 16, + "x": 8, + "y": 194 }, - "id": 52, + "id": 120, "interval": "5m", "maxDataPoints": 1000, "options": { @@ -3988,35 +6417,24 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.6.0", "targets": [ { "datasource": "${datasource}", "exemplar": true, - "expr": "sum(tron:db_size_bytes{group=`$group`,instance=`$instance`,type=\"LEVELDB\",db=\"storage-row\"})", + "expr": "sum by(level) (tron:db_size_bytes{group=\"$group\", instance=\"$instance\", db=\"delegation\"})", "format": "time_series", "instant": false, - "interval": "2m", + "interval": "", "intervalFactor": 1, - "legendFormat": "storage-row", + "legendFormat": "Level - {{level}}", "refId": "A" - }, - { - "datasource": "${datasource}", - "exemplar": true, - "expr": "sum(delta(tron:db_size_bytes{group=`$group`,instance=`$instance`,type = \"LEVELDB\",db=\"storage-row\"}[24h]))", - "hide": false, - "interval": "", - "legendFormat": "delta/24H", - "refId": "B" } ], - "title": "storage-row", + "title": "delegation levels", "type": "stat" }, { "datasource": "${datasource}", - "description": "", "fieldConfig": { "defaults": { "color": { @@ -4028,7 +6446,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 } ] }, @@ -4037,12 +6456,12 @@ "overrides": [] }, "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 79 + "h": 6, + "w": 16, + "x": 8, + "y": 170 }, - "id": 53, + "id": 140, "interval": "5m", "maxDataPoints": 1000, "options": { @@ -4063,35 +6482,24 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.6.0", "targets": [ { "datasource": "${datasource}", "exemplar": true, - "expr": "sum(tron:db_size_bytes{group=`$group`,instance=`$instance`,type=\"LEVELDB\",db=\"account\"})", + "expr": "sum by(level) (tron:db_size_bytes{group=\"$group\", instance=\"$instance\", db=\"trans\"})", "format": "time_series", "instant": false, - "interval": "2m", + "interval": "", "intervalFactor": 1, - "legendFormat": "account", + "legendFormat": "Level - {{level}}", "refId": "A" - }, - { - "datasource": "${datasource}", - "exemplar": true, - "expr": "sum(delta(tron:db_size_bytes{group=`$group`,instance=`$instance`,type = \"LEVELDB\",db=\"account\"}[24h]))", - "hide": false, - "interval": "", - "legendFormat": "delta/24H", - "refId": "B" } ], - "title": "account", + "title": "trans levels", "type": "stat" }, { "datasource": "${datasource}", - "description": "", "fieldConfig": { "defaults": { "color": { @@ -4103,7 +6511,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 } ] }, @@ -4112,12 +6521,12 @@ "overrides": [] }, "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 84 + "h": 6, + "w": 16, + "x": 8, + "y": 176 }, - "id": 54, + "id": 141, "interval": "5m", "maxDataPoints": 1000, "options": { @@ -4138,35 +6547,24 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.6.0", "targets": [ { "datasource": "${datasource}", "exemplar": true, - "expr": "sum(tron:db_size_bytes{group=`$group`,instance=`$instance`,type=\"LEVELDB\",db=\"delegation\"})", + "expr": "sum by(level) (tron:db_size_bytes{group=\"$group\", instance=\"$instance\", db=\"transactionHistoryStore\"})", "format": "time_series", "instant": false, - "interval": "2m", + "interval": "", "intervalFactor": 1, - "legendFormat": "delegation", + "legendFormat": "Level - {{level}}", "refId": "A" - }, - { - "datasource": "${datasource}", - "exemplar": true, - "expr": "sum(delta(tron:db_size_bytes{group=`$group`,instance=`$instance`,type = \"LEVELDB\",db=\"delegation\"}[24h]))", - "hide": false, - "interval": "", - "legendFormat": "delta/24H", - "refId": "B" } ], - "title": "delegation", + "title": "transactionHistoryStore levels", "type": "stat" }, { "datasource": "${datasource}", - "description": "", "fieldConfig": { "defaults": { "color": { @@ -4178,7 +6576,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 } ] }, @@ -4187,12 +6586,12 @@ "overrides": [] }, "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 84 + "h": 6, + "w": 16, + "x": 8, + "y": 164 }, - "id": 55, + "id": 142, "interval": "5m", "maxDataPoints": 1000, "options": { @@ -4213,30 +6612,20 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.6.0", "targets": [ { "datasource": "${datasource}", "exemplar": true, - "expr": "sum(tron:db_size_bytes{group=`$group`,instance=`$instance`,type=\"LEVELDB\",db=\"block-index\"})", + "expr": "sum by(level) (tron:db_size_bytes{group=\"$group\", instance=\"$instance\", db=\"transactionRetStore\"})", "format": "time_series", "instant": false, - "interval": "2m", + "interval": "", "intervalFactor": 1, - "legendFormat": "block-index", + "legendFormat": "Level - {{level}}", "refId": "A" - }, - { - "datasource": "${datasource}", - "exemplar": true, - "expr": "sum(delta(tron:db_size_bytes{group=`$group`,instance=`$instance`,type = \"LEVELDB\",db=\"block-index\"}[24h]))", - "hide": false, - "interval": "", - "legendFormat": "delta/24H", - "refId": "B" } ], - "title": "block-index", + "title": "transactionRetStore levels", "type": "stat" } ], @@ -4249,7 +6638,7 @@ "h": 1, "w": 24, "x": 0, - "y": 46 + "y": 62 }, "id": 64, "panels": [ @@ -4292,7 +6681,7 @@ "h": 2, "w": 24, "x": 0, - "y": 106 + "y": 122 }, "id": 91, "maxDataPoints": 100, @@ -4333,7 +6722,7 @@ }, { "datasource": "${datasource}", - "description": "JVM heap used / JVM heap init.\nJVM heap used / JVM heap max.", + "description": "JVM heap detail.", "fieldConfig": { "defaults": { "color": { @@ -4348,7 +6737,7 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, @@ -4362,7 +6751,7 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -4380,34 +6769,62 @@ "color": "green" }, { - "color": "#EAB839", - "value": 0.81 + "color": "red", + "value": 80 } ] }, - "unit": "percentunit" + "unit": "bytes" }, - "overrides": [] + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Par Survivor Space" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] }, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 108 + "x": 12, + "y": 124 }, - "id": 69, + "id": 71, "interval": "3s", "maxDataPoints": 500, "options": { + "alertThreshold": true, "legend": { "calcs": [ + "mean", "lastNotNull", - "max", - "min" + "max" ], - "displayMode": "list", + "displayMode": "table", "placement": "bottom", - "showLegend": true + "showLegend": true, + "sortBy": "Name", + "sortDesc": false }, "tooltip": { "hideZeros": false, @@ -4419,32 +6836,28 @@ "targets": [ { "datasource": "${datasource}", - "editorMode": "code", "exemplar": true, - "expr": "sum(jvm_memory_bytes_used{group=`$group`,instance=`$instance`})/sum(jvm_memory_bytes_init{group=`$group`,instance=`$instance`})", - "hide": false, + "expr": "jvm_memory_bytes_used{group=`$group`,instance=`$instance`,area=\"heap\"}", "interval": "", - "legendFormat": "Usage", - "range": true, - "refId": "B" + "legendFormat": "{{area}}", + "refId": "A" }, { "datasource": "${datasource}", - "editorMode": "code", - "expr": "sum(jvm_memory_bytes_used{group=`$group`,instance=`$instance`})/sum(jvm_memory_bytes_max{group=`$group`,instance=`$instance`})", + "exemplar": true, + "expr": "jvm_memory_pool_bytes_used{group=`$group`,instance=`$instance`}", "hide": false, - "instant": false, - "legendFormat": "{{label_name}}", - "range": true, - "refId": "A" + "interval": "", + "legendFormat": "{{pool}}", + "refId": "B" } ], - "title": "Memory Usage Rate ", + "title": "Memory Used", "type": "timeseries" }, { "datasource": "${datasource}", - "description": "JVM heap detail.", + "description": "GC time every minute.", "fieldConfig": { "defaults": { "color": { @@ -4496,44 +6909,17 @@ } ] }, - "unit": "bytes" + "unit": "s" }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "Par Survivor Space" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] + "overrides": [] }, "gridPos": { - "h": 8, + "h": 9, "w": 12, - "x": 12, - "y": 108 + "x": 0, + "y": 132 }, - "id": 71, - "interval": "3s", - "maxDataPoints": 500, + "id": 93, "options": { "alertThreshold": true, "legend": { @@ -4544,9 +6930,7 @@ ], "displayMode": "table", "placement": "bottom", - "showLegend": true, - "sortBy": "Name", - "sortDesc": false + "showLegend": true }, "tooltip": { "hideZeros": false, @@ -4559,27 +6943,22 @@ { "datasource": "${datasource}", "exemplar": true, - "expr": "jvm_memory_bytes_used{group=`$group`,instance=`$instance`,area=\"heap\"}", - "interval": "", - "legendFormat": "{{area}}", - "refId": "A" - }, - { - "datasource": "${datasource}", - "exemplar": true, - "expr": "jvm_memory_pool_bytes_used{group=`$group`,instance=`$instance`}", - "hide": false, - "interval": "", - "legendFormat": "{{pool}}", - "refId": "B" + "expr": "increase(jvm_gc_collection_seconds_sum{group=\"$group\",instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "interval": "60s", + "intervalFactor": 1, + "legendFormat": "{{gc}}", + "metric": "jvm_gc_collection_seconds_sum", + "refId": "A", + "step": 10 } ], - "title": "Memory Used", + "title": "GC Time/Minute", "type": "timeseries" }, { "datasource": "${datasource}", - "description": "GC time every minute.", + "description": "GC count every minute.", "fieldConfig": { "defaults": { "color": { @@ -4618,6 +6997,7 @@ "mode": "off" } }, + "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", @@ -4631,17 +7011,17 @@ } ] }, - "unit": "s" + "unit": "short" }, "overrides": [] }, "gridPos": { "h": 9, "w": 12, - "x": 0, - "y": 116 + "x": 12, + "y": 132 }, - "id": 93, + "id": 97, "options": { "alertThreshold": true, "legend": { @@ -4665,22 +7045,22 @@ { "datasource": "${datasource}", "exemplar": true, - "expr": "increase(jvm_gc_collection_seconds_sum{group=\"$group\",instance=~\"$instance\"}[$__interval])", + "expr": "increase(jvm_gc_collection_seconds_count{group=\"$group\",instance=~\"$instance\"}[$__interval])", "format": "time_series", "interval": "60s", "intervalFactor": 1, "legendFormat": "{{gc}}", - "metric": "jvm_gc_collection_seconds_sum", + "metric": "", "refId": "A", "step": 10 } ], - "title": "GC Time/Minute", + "title": "GC Count/Minute", "type": "timeseries" }, { "datasource": "${datasource}", - "description": "GC count every minute.", + "description": "JVM heap used / JVM heap init.\nJVM heap used / JVM heap max.", "fieldConfig": { "defaults": { "color": { @@ -4695,7 +7075,7 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -4709,7 +7089,8 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "never", + "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -4719,40 +7100,41 @@ "mode": "off" } }, - "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { - "color": "red", - "value": 80 + "color": "#EAB839", + "value": 0.81 } ] }, - "unit": "short" + "unit": "percentunit" }, "overrides": [] }, "gridPos": { - "h": 9, + "h": 8, "w": 12, - "x": 12, - "y": 116 + "x": 0, + "y": 141 }, - "id": 97, + "id": 138, + "interval": "3s", + "maxDataPoints": 500, "options": { - "alertThreshold": true, "legend": { "calcs": [ - "mean", "lastNotNull", - "max" + "max", + "min" ], - "displayMode": "table", + "displayMode": "list", "placement": "bottom", "showLegend": true }, @@ -4762,22 +7144,24 @@ "sort": "none" } }, - "pluginVersion": "11.6.0", "targets": [ { "datasource": "${datasource}", "exemplar": true, - "expr": "increase(jvm_gc_collection_seconds_count{group=\"$group\",instance=~\"$instance\"}[$__interval])", - "format": "time_series", - "interval": "60s", - "intervalFactor": 1, - "legendFormat": "{{gc}}", - "metric": "", - "refId": "A", - "step": 10 + "expr": "sum(jvm_memory_bytes_used{group=`$group`,instance=`$instance`})/sum(jvm_memory_bytes_init{group=`$group`,instance=`$instance`})", + "interval": "", + "legendFormat": "Used/Init", + "refId": "B" + }, + { + "datasource": "${datasource}", + "expr": "sum(jvm_memory_bytes_used{group=`$group`,instance=`$instance`})/sum(jvm_memory_bytes_max{group=`$group`,instance=`$instance`})", + "instant": false, + "legendFormat": "Used/Max", + "refId": "A" } ], - "title": "GC Count/Minute", + "title": "JVM Memory Usage Rate", "type": "timeseries" } ], @@ -4875,6 +7259,6 @@ }, "timezone": "", "title": "java-tron-server", - "uid": "WAY66gQ07X", + "uid": "java-tron-server", "version": 8 } diff --git a/private_net/README.md b/private_net/README.md index 597fd18f..11541602 100644 --- a/private_net/README.md +++ b/private_net/README.md @@ -4,6 +4,17 @@ Here is a quick-start guide for setting up a TRON private network using Docker. A private chain needs at least one fullnode run by a [Super Representative (SR)](https://tronprotocol.github.io/documentation-en/mechanism-algorithm/sr/) to produce blocks, and any number of fullnodes to synchronize blocks and broadcast transactions. +## ⚠️ Security Notice + +The configuration files in this directory contain **DEMO private keys for local testing purposes ONLY**. + +**IMPORTANT:** +- These keys are publicly known and should NEVER be used in production +- Always generate new private keys for any real deployment +- NEVER use these keys on mainnet or with real funds +- For production use, consider using encrypted keystore files instead of plaintext private keys +- Refer to the [localwitnesskeystore documentation](https://tronprotocol.github.io/documentation-en/using_javatron/installing_javatron/?h=keystore#specifying-super-representative-account-private-key-using-keystore-password) for secure key management + ## Prerequisites ### Minimum hardware requirements diff --git a/private_net/docker-compose.yml b/private_net/docker-compose.yml index defbf00e..682091c6 100644 --- a/private_net/docker-compose.yml +++ b/private_net/docker-compose.yml @@ -17,7 +17,7 @@ services: - ./datadir:/java-tron/data # mount a local directory to make the blocks data persistent. - ./logs/tron-witness1:/java-tron/logs # map to host logs command: > - -jvm "{-Xmx12g -Xmn2g -XX:+UseConcMarkSweepGC -Xloggc:./logs/gc.log}" -c /java-tron/conf/private_net_config_witness1.conf -d /java-tron/data -w + -jvm "{-Xmx12g -Xmn2g}" -c /java-tron/conf/private_net_config_witness1.conf -d /java-tron/data -w # tron-witness2: # image: tronprotocol/java-tron:latest @@ -32,7 +32,7 @@ services: # - ../conf:/java-tron/conf # - ./logs/tron-witness2:/java-tron/logs # command: > - # -jvm "{-Xmx12g -Xmn2g -XX:+UseConcMarkSweepGC -Xloggc:./logs/gc.log}" + # -jvm "{-Xmx12g -Xmn2g}" # -c /java-tron/conf/private_net_config_witness2.conf # -w tron-node1: @@ -48,7 +48,7 @@ services: - ../conf:/java-tron/conf - ./logs/tron-node1:/java-tron/logs command: > - -jvm "{-Xmx12g -Xmn2g -XX:+UseConcMarkSweepGC -Xloggc:./logs/gc.log}" -c /java-tron/conf/private_net_config_others.conf + -jvm "{-Xmx12g -Xmn2g}" -c /java-tron/conf/private_net_config_others.conf # tron-node2: # image: tronprotocol/java-tron:latest @@ -63,7 +63,7 @@ services: # - ../conf:/java-tron/conf # - ./logs/tron-node2:/java-tron/logs # command: > -# -jvm "{-Xmx12g -Xmn2g -XX:+UseConcMarkSweepGC -Xloggc:./logs/gc.log}" +# -jvm "{-Xmx12g -Xmn2g}" # -c /java-tron/conf/private_net_config_others.conf networks: tron_private_network: diff --git a/run-checkstyle.sh b/run-checkstyle.sh index 3d574ff6..db2d1b1a 100755 --- a/run-checkstyle.sh +++ b/run-checkstyle.sh @@ -1,64 +1,67 @@ -#!/bin/sh +#!/bin/bash +set -eu # exit on any error -# Define the URL for the Checkstyle JAR file +# ─── Configuration ─── CHECKSTYLE_VERSION="8.42" CHECKSTYLE_JAR="checkstyle-${CHECKSTYLE_VERSION}-all.jar" CHECKSTYLE_URL="https://github.com/checkstyle/checkstyle/releases/download/checkstyle-${CHECKSTYLE_VERSION}/${CHECKSTYLE_JAR}" -# Define the directory where the JAR file should be stored -LIB_DIR="libs" +# SHA-256 digest of the official release JAR +# Obtain from: https://github.com/checkstyle/checkstyle/releases/tag/checkstyle-8.42 +# or run: sha256sum checkstyle-8.42-all.jar +CHECKSTYLE_SHA256="4982ebeaa429fe41f3be2c3309a5c49d84c71ee1f78f967344b8bc82cf3101aa" -# Create the directory if it does not exist +LIB_DIR="libs" mkdir -p "$LIB_DIR" - -# Define the full path to the JAR file CHECKSTYLE_PATH="${LIB_DIR}/${CHECKSTYLE_JAR}" -# Check if the JAR file already exists +# ─── Download with integrity verification ─── +verify_checksum() { + local file="$1" + local expected="$2" + local actual + actual=$(sha256sum "$file" | awk '{print $1}') + if [ "$actual" != "$expected" ]; then + echo "ERROR: Checksum mismatch for ${file}" >&2 + echo " Expected: ${expected}" >&2 + echo " Actual: ${actual}" >&2 + rm -f "$file" + exit 1 + fi + echo "Checksum verified: ${file}" +} + if [ -f "$CHECKSTYLE_PATH" ]; then - echo "Checkstyle JAR file already exists at ${CHECKSTYLE_PATH}" + echo "Checkstyle JAR exists at ${CHECKSTYLE_PATH}, verifying integrity..." + verify_checksum "$CHECKSTYLE_PATH" "$CHECKSTYLE_SHA256" else - echo "Checkstyle JAR file not found. Downloading from ${CHECKSTYLE_URL}..." - curl -L -o "$CHECKSTYLE_PATH" "$CHECKSTYLE_URL" && echo "downloaded successfully" + echo "Downloading Checkstyle from ${CHECKSTYLE_URL}..." + curl --fail --silent --show-error -L -o "$CHECKSTYLE_PATH" "$CHECKSTYLE_URL" + verify_checksum "$CHECKSTYLE_PATH" "$CHECKSTYLE_SHA256" + echo "Downloaded and verified successfully." fi -# Add the lib directory to .gitignore if it's not already present +# ─── .gitignore handling ─── GITIGNORE_FILE=".gitignore" - -if ! grep -q "^${LIB_DIR}/$" "$GITIGNORE_FILE"; then - echo "Adding ${LIB_DIR}/ to ${GITIGNORE_FILE}" +if [ -f "$GITIGNORE_FILE" ] && ! grep -q "^${LIB_DIR}/$" "$GITIGNORE_FILE"; then echo "${LIB_DIR}/" >> "$GITIGNORE_FILE" -else - echo "${LIB_DIR}/ is already in ${GITIGNORE_FILE}" fi -# Check if there are any Java files in the project -JAVA_FILES_FOUND=$(find . -name "*.java") +# ─── Find and check Java files ─── +CHECKSTYLE_CONFIG="./conf/checkstyle/checkStyleAll.xml" -# if there is java file go ahead to trigger checkStyle for all java files -if [ -z "$JAVA_FILES_FOUND" ]; then - echo "No Java files found in the project." +# Use find with -print0 / xargs -0 to handle filenames with spaces +JAVA_COUNT=$(find . -name "*.java" | wc -l) +if [ "$JAVA_COUNT" -eq 0 ]; then + echo "No Java files found." exit 0 -else - echo "Java files found in the project. Now run further checks." fi -# Path to your Checkstyle configuration file -CHECKSTYLE_CONFIG="./conf/checkstyle/checkStyleAll.xml" - -# shellcheck disable=SC2027 -# shellcheck disable=SC2046 - -All_Java_Files=$(find . -name "*.java") - -# Run Checkstyle on all Java files in the repository -java -jar "$CHECKSTYLE_PATH" -c "$CHECKSTYLE_CONFIG" "$All_Java_Files" - -echo "finish ...." +echo "Found ${JAVA_COUNT} Java files, running Checkstyle..." -# Capture the exit code of Checkstyle -# shellcheck disable=SC2320 +# Capture exit code correctly (not after echo) +find . -name "*.java" -print0 | xargs -0 java -jar "$CHECKSTYLE_PATH" -c "$CHECKSTYLE_CONFIG" STATUS=$? -# Exit with the same status code as Checkstyle +echo "Checkstyle finished with exit code: ${STATUS}" exit $STATUS diff --git a/single_node/README.md b/single_node/README.md index 68824eff..02e20617 100644 --- a/single_node/README.md +++ b/single_node/README.md @@ -88,7 +88,7 @@ services: - ../logs:/java-tron/logs - ../output-directory:/java-tron/output-directory command: > - -jvm "{-Xmx14g -Xms12g -XX:+UseConcMarkSweepGC}" + -jvm "{-Xmx14g -Xms12g}" -c /java-tron/conf/main_net_config.conf -d /java-tron/output-directory -w @@ -98,7 +98,7 @@ In the example above, the host file `../conf/main_net_config.conf` will be used. Logs generated by docker container will be mapped to your `../logs/mainnet/logs`. Flags in `command` are used for java-tron start-up arguments: -- `-jvm` used for java virtual machine, the parameters must be enclosed in double quotes and braces. `"{-Xmx14g -Xms12g}"` sets the maximum and initial heap size to 14GB and 12GB respectively. If you want to set up a long run FullNode, please use the best practice jvm flags with `"{-Xmx14g -Xmn2g XX:ReservedCodeCacheSize=256m -XX:MetaspaceSize=256m -XX:MaxMetaspaceSize=512m -XX:MaxDirectMemorySize=1G -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:gc.log -XX:+UseConcMarkSweepGC -XX:NewRatio=2 -XX:+CMSScavengeBeforeRemark -XX:+ParallelRefProcEnabled -XX:+HeapDumpOnOutOfMemoryError -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70}"`. +- `-jvm` used for java virtual machine, the parameters must be enclosed in double quotes and braces. `"{-Xmx14g -Xms12g}"` sets the maximum and initial heap size to 14GB and 12GB respectively. If you want to set up a long run FullNode, please use the best practice jvm flags with `"{-Xmx14g -Xmn2g -XX:ReservedCodeCacheSize=256m -XX:MetaspaceSize=256m -XX:MaxMetaspaceSize=512m -XX:MaxDirectMemorySize=1G -XX:+ParallelRefProcEnabled -XX:+HeapDumpOnOutOfMemoryError}"`. Do not add garbage collector or GC logging options here. The image ships an architecture-specific `bin/java-tron.vmoptions` that already selects a collector: CMS on the amd64/JDK 8 image, ZGC on the arm64/JDK 17 image. Adding them stops the arm64 container from starting, for two different reasons: `-XX:+UseConcMarkSweepGC` was removed in JDK 14, so JDK 17 rejects it outright as an unrecognized option, while `-Xloggc:` is still accepted as a deprecated alias for unified logging but, unlike JDK 8, aborts startup when its target directory does not exist. - `-c` defines the configuration file to use. - `-d` defines the database file to use. You can mount a directory for `datadir` with snapshots. This can save time by syncing from a near-latest block number. Please refer to [**Fullnode and Lite-Fullnode**](https://tronprotocol.github.io/documentation-en/using_javatron/backup_restore/#fullnode-data-snapshot) for Mainnet data. For the Nile testnet, you can download the snapshot [here](https://database.nileex.io/). - `-w` means to start as a witness. You need to fill the `localwitness` field with private keys in configure file. Refer to the [**Run as Witness**](https://tronprotocol.github.io/documentation-en/using_javatron/installing_javatron/#startup-a-fullnode-that-produces-blocks). If you want to use keystore + password method, make sure the keystore is inside the mounted directory and remove `-d` to interact with console for password input. diff --git a/single_node/docker-compose-configure-example.yml b/single_node/docker-compose-configure-example.yml index 08469869..b733edd4 100644 --- a/single_node/docker-compose-configure-example.yml +++ b/single_node/docker-compose-configure-example.yml @@ -14,7 +14,7 @@ services: - ../logs:/java-tron/logs - ../output-directory:/java-tron/output-directory command: > - -jvm "{-Xmx14g -Xmn2g -XX:MaxDirectMemorySize=1G -XX:+UseConcMarkSweepGC -XX:+PrintGC -Xloggc:./logs/gc.log}" + -jvm "{-Xmx14g -Xmn2g -XX:MaxDirectMemorySize=1G}" -c /java-tron/conf/main_net_config.conf -d /java-tron/output-directory -w diff --git a/single_node/docker-compose-quick-start.yml b/single_node/docker-compose-quick-start.yml index f6e1950f..80a78054 100644 --- a/single_node/docker-compose-quick-start.yml +++ b/single_node/docker-compose-quick-start.yml @@ -10,4 +10,4 @@ services: - "18888:18888/udp" - "50051:50051" command: > - -jvm "{-Xmx14g -Xmn2g -XX:MaxDirectMemorySize=1G -XX:+UseConcMarkSweepGC -XX:+PrintGC -Xloggc:./logs/gc.log}" + -jvm "{-Xmx14g -Xmn2g -XX:MaxDirectMemorySize=1G}" diff --git a/single_node/docker-compose.fullnode.main.yml b/single_node/docker-compose.fullnode.main.yml index 2b1aeb2a..ff786a91 100644 --- a/single_node/docker-compose.fullnode.main.yml +++ b/single_node/docker-compose.fullnode.main.yml @@ -15,6 +15,6 @@ services: - ../output-directory/mainnet:/java-tron/output-directory command: - -jvm - - "{-Xmx14g -Xmn2g -XX:MaxDirectMemorySize=1G -XX:+UseConcMarkSweepGC -XX:+PrintGC -Xloggc:./logs/gc.log }" # Using environment variable for JVM arguments + - "{-Xmx14g -Xmn2g -XX:MaxDirectMemorySize=1G}" # Using environment variable for JVM arguments - -c - /java-tron/conf/main_net_config.conf diff --git a/single_node/docker-compose.fullnode.nile.yml b/single_node/docker-compose.fullnode.nile.yml index 59e78bdc..8a590626 100644 --- a/single_node/docker-compose.fullnode.nile.yml +++ b/single_node/docker-compose.fullnode.nile.yml @@ -15,6 +15,6 @@ services: - ../output-directory/nile:/java-tron/output-directory command: - -jvm - - "{-Xmx14g -XX:+UseConcMarkSweepGC -XX:+PrintGC -Xloggc:./logs/gc.log }" # Using environment variable for JVM arguments + - "{-Xmx14g}" # Using environment variable for JVM arguments - -c - /java-tron/conf/nile_net_config.conf diff --git a/single_node/docker-compose.fullnode.private.yml b/single_node/docker-compose.fullnode.private.yml index 2c7b6551..5880caf4 100644 --- a/single_node/docker-compose.fullnode.private.yml +++ b/single_node/docker-compose.fullnode.private.yml @@ -15,6 +15,6 @@ services: - ../output-directory/private:/java-tron/output-directory command: - -jvm - - "{-Xmx4g -Xms2g -XX:+UseConcMarkSweepGC}" # Using environment variable for JVM arguments + - "{-Xmx4g -Xms2g}" # Using environment variable for JVM arguments - -c - /java-tron/conf/private_net_config_others.conf diff --git a/single_node/docker-compose.witness.private.yml b/single_node/docker-compose.witness.private.yml index 7b5575ab..1619fd34 100644 --- a/single_node/docker-compose.witness.private.yml +++ b/single_node/docker-compose.witness.private.yml @@ -15,7 +15,7 @@ services: - ../output-directory/private:/java-tron/output-directory command: - -jvm - - "{-Xmx4g -Xms2g -XX:+UseConcMarkSweepGC -XX:+PrintGC -Xloggc:./logs/gc.log}" # Using environment variable for JVM arguments + - "{-Xmx4g -Xms2g}" # Using environment variable for JVM arguments - -c - /java-tron/conf/private_net_config_witness1.conf - -w diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile index 860c70b7..0b4477c8 100644 --- a/tools/docker/Dockerfile +++ b/tools/docker/Dockerfile @@ -3,19 +3,27 @@ ARG VERSION="dev" ENV NO_PROXY_CACHE="-o Acquire::BrokenProxy=true -o Acquire::http::No-Cache=true -o Acquire::http::Pipeline-Depth=0" ENV TMP_DIR="/tron-build" -ENV JDK_TAR="jdk-8u202-linux-x64.tar.gz" -ENV JDK_DIR="jdk1.8.0_202" -ENV JDK_MD5="0029351f7a946f6c05b582100c7d45b7" +ENV OPENJDK8_URL="https://api.adoptium.net/v3/binary/latest/8/ga/linux/x64/jdk/hotspot/normal/eclipse" +ENV ADOPTIUM_SIGNING_FINGERPRINT="3B04D753C9050D9A5D343F39843C48A565F8F04B" +ENV JDK_DIR="/usr/local/openjdk-8" ENV BASE_DIR="/java-tron" # Update and install dependencies without using any cache RUN apt-get update $NO_PROXY_CACHE && \ - apt-get --quiet --yes install git wget 7zip curl jq && \ - wget -P /usr/local https://github.com/frekele/oracle-java/releases/download/8u202-b08/$JDK_TAR \ - && echo "$JDK_MD5 /usr/local/$JDK_TAR" | md5sum -c \ - && tar -zxf /usr/local/$JDK_TAR -C /usr/local\ - && rm /usr/local/$JDK_TAR \ - && export JAVA_HOME=/usr/local/$JDK_DIR \ + apt-get --quiet --yes install git wget 7zip curl jq libtcmalloc-minimal4 gnupg dirmngr ca-certificates && \ + cd /usr/local \ + && FETCH_URL="$(curl -fsS -w "%{redirect_url}" -o /dev/null "$OPENJDK8_URL")" \ + && JDK_TAR="$(curl -fsSL -w "%{filename_effective}" -O "$FETCH_URL")" \ + && curl -fsSLo "$JDK_TAR.sig" "$FETCH_URL.sig" \ + && GNUPGHOME="$(mktemp -d)" \ + && export GNUPGHOME \ + && gpg --batch --keyserver hkps://keyserver.ubuntu.com --recv-keys "$ADOPTIUM_SIGNING_FINGERPRINT" \ + && gpg --batch --verify "$JDK_TAR.sig" "$JDK_TAR" \ + && rm -rf "$GNUPGHOME" "$JDK_TAR.sig" \ + && mkdir -p "$JDK_DIR" \ + && tar -zxf "$JDK_TAR" -C "$JDK_DIR" --strip-components=1 \ + && rm "$JDK_TAR" \ + && export JAVA_HOME=$JDK_DIR \ && export CLASSPATH=$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar \ && export PATH=$PATH:$JAVA_HOME/bin \ && echo "git clone" \ @@ -24,16 +32,15 @@ RUN apt-get update $NO_PROXY_CACHE && \ && git clone https://github.com/tronprotocol/java-tron.git \ && cd java-tron \ && git checkout master \ - && ./gradlew build -x test \ + && ./gradlew clean build -x test -x check --no-daemon \ && cd build/distributions \ && 7z x -y java-tron-1.0.0.zip \ && mv java-tron-1.0.0 $BASE_DIR \ && rm -rf $TMP_DIR \ && rm -rf ~/.gradle \ - && mv /usr/local/$JDK_DIR/jre /usr/local \ - && rm -rf /usr/local/$JDK_DIR \ - && wget -P $BASE_DIR/config https://raw.githubusercontent.com/tronprotocol/tron-deployment/master/main_net_config.conf \ - && mv $BASE_DIR/config/main_net_config.conf $BASE_DIR/config.conf \ + && mv $JDK_DIR/jre /usr/local \ + && rm -rf $JDK_DIR \ + && wget -O $BASE_DIR/config.conf https://raw.githubusercontent.com/tronprotocol/tron-deployment/master/main_net_config.conf \ # Clean apt cache && apt-get clean \ && rm -rf /var/cache/apt/archives/* /var/cache/apt/archives/partial/* \ @@ -41,6 +48,8 @@ RUN apt-get update $NO_PROXY_CACHE && \ ENV JAVA_HOME="/usr/local/jre" ENV PATH=$PATH:$JAVA_HOME/bin +ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4" +ENV TCMALLOC_RELEASE_RATE=10 COPY docker-entrypoint.sh $BASE_DIR/bin diff --git a/tools/docker/Dockerfile.arm64 b/tools/docker/Dockerfile.arm64 new file mode 100644 index 00000000..280af0d6 --- /dev/null +++ b/tools/docker/Dockerfile.arm64 @@ -0,0 +1,51 @@ +FROM ubuntu:24.04 +ARG VERSION="dev" +ENV NO_PROXY_CACHE="-o Acquire::BrokenProxy=true -o Acquire::http::No-Cache=true -o Acquire::http::Pipeline-Depth=0" + +ENV TMP_DIR="/tron-build" +ENV BASE_DIR="/java-tron" + +# Update and install dependencies without using any cache +RUN apt-get update $NO_PROXY_CACHE \ + && apt-get --quiet --yes install git wget 7zip curl jq libtcmalloc-minimal4 openjdk-17-jre-headless=17* \ + && echo "git clone" \ + && mkdir -p $TMP_DIR \ + && cd $TMP_DIR \ + && git clone https://github.com/tronprotocol/java-tron.git \ + && cd java-tron \ + && git checkout master \ + && ./gradlew clean build -x test -x check --no-daemon \ + && cd build/distributions \ + && 7z x -y java-tron-1.0.0.zip \ + && mv java-tron-1.0.0 $BASE_DIR \ + && rm -rf $TMP_DIR \ + && rm -rf ~/.gradle \ + && wget -4 -O $BASE_DIR/config.conf https://raw.githubusercontent.com/tronprotocol/tron-deployment/master/main_net_config.conf \ + # Clean apt cache + && apt-get clean \ + && rm -rf /var/cache/apt/archives/* /var/cache/apt/archives/partial/* \ + && rm -rf /var/lib/apt/lists/* + +ENV JAVA_HOME="/usr/lib/jvm/java-17-openjdk-arm64" +ENV PATH=$PATH:$JAVA_HOME/bin +ENV LD_PRELOAD="/usr/lib/aarch64-linux-gnu/libtcmalloc_minimal.so.4" +ENV TCMALLOC_RELEASE_RATE=10 + +COPY docker-entrypoint.sh $BASE_DIR/bin + +WORKDIR $BASE_DIR + +ENTRYPOINT ["./bin/docker-entrypoint.sh"] + +# Build-time metadata as defined at http://label-schema.org +ARG BUILD_DATE +ARG VCS_REF +LABEL org.label-schema.build-date=$BUILD_DATE \ + org.label-schema.name="Java-TRON" \ + org.label-schema.description="TRON protocol" \ + org.label-schema.url="https://tron.network/" \ + org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.vcs-url="https://github.com/tronprotocol/java-tron.git" \ + org.label-schema.vendor="TRON protocol" \ + org.label-schema.version=$VERSION \ + org.label-schema.schema-version="1.0" diff --git a/tools/docker/Dockerfile.nile b/tools/docker/Dockerfile.nile index 3d687624..97684282 100644 --- a/tools/docker/Dockerfile.nile +++ b/tools/docker/Dockerfile.nile @@ -3,19 +3,27 @@ ARG VERSION="dev" ENV NO_PROXY_CACHE="-o Acquire::BrokenProxy=true -o Acquire::http::No-Cache=true -o Acquire::http::Pipeline-Depth=0" ENV TMP_DIR="/tron-build" -ENV JDK_TAR="jdk-8u202-linux-x64.tar.gz" -ENV JDK_DIR="jdk1.8.0_202" -ENV JDK_MD5="0029351f7a946f6c05b582100c7d45b7" +ENV OPENJDK8_URL="https://api.adoptium.net/v3/binary/latest/8/ga/linux/x64/jdk/hotspot/normal/eclipse" +ENV ADOPTIUM_SIGNING_FINGERPRINT="3B04D753C9050D9A5D343F39843C48A565F8F04B" +ENV JDK_DIR="/usr/local/openjdk-8" ENV BASE_DIR="/java-tron" # Update and install dependencies without using any cache RUN apt-get update $NO_PROXY_CACHE && \ - apt-get --quiet --yes install git wget 7zip curl jq && \ - wget -P /usr/local https://github.com/frekele/oracle-java/releases/download/8u202-b08/$JDK_TAR \ - && echo "$JDK_MD5 /usr/local/$JDK_TAR" | md5sum -c \ - && tar -zxf /usr/local/$JDK_TAR -C /usr/local\ - && rm /usr/local/$JDK_TAR \ - && export JAVA_HOME=/usr/local/$JDK_DIR \ + apt-get --quiet --yes install git wget 7zip curl jq libtcmalloc-minimal4 gnupg dirmngr ca-certificates && \ + cd /usr/local \ + && FETCH_URL="$(curl -fsS -w "%{redirect_url}" -o /dev/null "$OPENJDK8_URL")" \ + && JDK_TAR="$(curl -fsSL -w "%{filename_effective}" -O "$FETCH_URL")" \ + && curl -fsSLo "$JDK_TAR.sig" "$FETCH_URL.sig" \ + && GNUPGHOME="$(mktemp -d)" \ + && export GNUPGHOME \ + && gpg --batch --keyserver hkps://keyserver.ubuntu.com --recv-keys "$ADOPTIUM_SIGNING_FINGERPRINT" \ + && gpg --batch --verify "$JDK_TAR.sig" "$JDK_TAR" \ + && rm -rf "$GNUPGHOME" "$JDK_TAR.sig" \ + && mkdir -p "$JDK_DIR" \ + && tar -zxf "$JDK_TAR" -C "$JDK_DIR" --strip-components=1 \ + && rm "$JDK_TAR" \ + && export JAVA_HOME=$JDK_DIR \ && export CLASSPATH=$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar \ && export PATH=$PATH:$JAVA_HOME/bin \ && echo "git clone" \ @@ -24,16 +32,15 @@ RUN apt-get update $NO_PROXY_CACHE && \ && git clone https://github.com/tron-nile-testnet/nile-testnet.git \ && cd nile-testnet \ && git checkout master \ - && ./gradlew build -x test \ + && ./gradlew clean build -x test -x check --no-daemon \ && cd build/distributions \ && 7z x -y java-tron-1.0.0.zip \ && mv java-tron-1.0.0 $BASE_DIR \ && rm -rf $TMP_DIR \ && rm -rf ~/.gradle \ - && mv /usr/local/$JDK_DIR/jre /usr/local \ - && rm -rf /usr/local/$JDK_DIR \ - && wget -P $BASE_DIR/config https://raw.githubusercontent.com/tronprotocol/tron-deployment/master/test_net_config.conf \ - && mv $BASE_DIR/config/test_net_config.conf $BASE_DIR/config.conf \ + && mv $JDK_DIR/jre /usr/local \ + && rm -rf $JDK_DIR \ + && wget -O $BASE_DIR/config.conf https://raw.githubusercontent.com/tronprotocol/tron-deployment/master/test_net_config.conf \ # Clean apt cache && apt-get clean \ && rm -rf /var/cache/apt/archives/* /var/cache/apt/archives/partial/* \ @@ -41,6 +48,8 @@ RUN apt-get update $NO_PROXY_CACHE && \ ENV JAVA_HOME="/usr/local/jre" ENV PATH=$PATH:$JAVA_HOME/bin +ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4" +ENV TCMALLOC_RELEASE_RATE=10 COPY docker-entrypoint.sh $BASE_DIR/bin diff --git a/tools/docker/Dockerfile.nile.arm64 b/tools/docker/Dockerfile.nile.arm64 new file mode 100644 index 00000000..7940c4de --- /dev/null +++ b/tools/docker/Dockerfile.nile.arm64 @@ -0,0 +1,51 @@ +FROM ubuntu:24.04 +ARG VERSION="dev" +ENV NO_PROXY_CACHE="-o Acquire::BrokenProxy=true -o Acquire::http::No-Cache=true -o Acquire::http::Pipeline-Depth=0" + +ENV TMP_DIR="/tron-build" +ENV BASE_DIR="/java-tron" + +# Update and install dependencies without using any cache +RUN apt-get update $NO_PROXY_CACHE \ + && apt-get --quiet --yes install git wget 7zip curl jq libtcmalloc-minimal4 openjdk-17-jre-headless=17* \ + && echo "git clone" \ + && mkdir -p $TMP_DIR \ + && cd $TMP_DIR \ + && git clone https://github.com/tron-nile-testnet/nile-testnet.git \ + && cd nile-testnet \ + && git checkout master \ + && ./gradlew clean build -x test -x check --no-daemon \ + && cd build/distributions \ + && 7z x -y java-tron-1.0.0.zip \ + && mv java-tron-1.0.0 $BASE_DIR \ + && rm -rf $TMP_DIR \ + && rm -rf ~/.gradle \ + && wget -4 -O $BASE_DIR/config.conf https://raw.githubusercontent.com/tronprotocol/tron-deployment/master/test_net_config.conf \ + # Clean apt cache + && apt-get clean \ + && rm -rf /var/cache/apt/archives/* /var/cache/apt/archives/partial/* \ + && rm -rf /var/lib/apt/lists/* + +ENV JAVA_HOME="/usr/lib/jvm/java-17-openjdk-arm64" +ENV PATH=$PATH:$JAVA_HOME/bin +ENV LD_PRELOAD="/usr/lib/aarch64-linux-gnu/libtcmalloc_minimal.so.4" +ENV TCMALLOC_RELEASE_RATE=10 + +COPY docker-entrypoint.sh $BASE_DIR/bin + +WORKDIR $BASE_DIR + +ENTRYPOINT ["./bin/docker-entrypoint.sh"] + +# Build-time metadata as defined at http://label-schema.org +ARG BUILD_DATE +ARG VCS_REF +LABEL org.label-schema.build-date=$BUILD_DATE \ + org.label-schema.name="Java-TRON" \ + org.label-schema.description="TRON protocol" \ + org.label-schema.url="https://tron.network/" \ + org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.vcs-url="https://github.com/tron-nile-testnet/nile-testnet.git" \ + org.label-schema.vendor="TRON protocol" \ + org.label-schema.version=$VERSION \ + org.label-schema.schema-version="1.0" diff --git a/tools/docker/README.md b/tools/docker/README.md index 16319b4d..1f129115 100644 --- a/tools/docker/README.md +++ b/tools/docker/README.md @@ -86,14 +86,14 @@ Building for default linux/amd64 platform #3 transferring context: 2B done #3 DONE 0.0s -#4 [1/5] FROM docker.io/library/ubuntu:24.04@sha256:80dd3c3b9c6cecb9f1667e9290b3bc61b78c2678c02cbdae5f0fea92cc6734ab +#4 [1/5] FROM docker.io/library/ubuntu:24.04 #4 DONE 0.0s #5 [internal] load build context #5 transferring context: 160B done #5 DONE 0.0s -#6 [2/5] RUN apt-get update -o Acquire::BrokenProxy=true -o Acquire::http::No-Cache=true -o Acquire::http::Pipeline-Depth=0 && apt-get --quiet --yes install git wget 7zip curl jq && wget -P /usr/local https://github.com/frekele/oracle-java/releases/download/8u202-b08/jdk-8u202-linux-x64.tar.gz && echo "0029351f7a946f6c05b582100c7d45b7 /usr/local/jdk-8u202-linux-x64.tar.gz" | md5sum -c && tar -zxf /usr/local/jdk-8u202-linux-x64.tar.gz -C /usr/local && rm /usr/local/jdk-8u202-linux-x64.tar.gz && export JAVA_HOME=/usr/local/jdk1.8.0_202 && export CLASSPATH=$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar && export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$JAVA_HOME/bin && echo "git clone" && mkdir -p /tron-build && cd /tron-build && git clone https://github.com/tronprotocol/java-tron.git && cd java-tron && git checkout master && ./gradlew build -x test && cd build/distributions && 7z x -y java-tron-1.0.0.zip && mv java-tron-1.0.0 /java-tron && rm -rf /tron-build && rm -rf ~/.gradle && mv /usr/local/jdk1.8.0_202/jre /usr/local && rm -rf /usr/local/jdk1.8.0_202 apt-get clean && rm -rf /var/cache/apt/archives/* /var/cache/apt/archives/partial/* && rm -rf /var/lib/apt/lists/* +#6 [2/5] RUN apt-get update -o Acquire::BrokenProxy=true -o Acquire::http::No-Cache=true -o Acquire::http::Pipeline-Depth=0 && apt-get --quiet --yes install git wget 7zip curl jq && wget -P /usr/local https://github.com/frekele/oracle-java/releases/download/8u202-b08/jdk-8u202-linux-x64.tar.gz && echo "9a5c32411a6a06e22b69c495b7975034409fa1652d03aeb8eb5b6f59fd4594e0 /usr/local/jdk-8u202-linux-x64.tar.gz" | sha256sum -c && tar -zxf /usr/local/jdk-8u202-linux-x64.tar.gz -C /usr/local && rm /usr/local/jdk-8u202-linux-x64.tar.gz && export JAVA_HOME=/usr/local/jdk1.8.0_202 && export CLASSPATH=$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar && export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$JAVA_HOME/bin && echo "git clone" && mkdir -p /tron-build && cd /tron-build && git clone https://github.com/tronprotocol/java-tron.git && cd java-tron && git checkout master && ./gradlew build -x test && cd build/distributions && 7z x -y java-tron-1.0.0.zip && mv java-tron-1.0.0 /java-tron && rm -rf /tron-build && rm -rf ~/.gradle && mv /usr/local/jdk1.8.0_202/jre /usr/local && rm -rf /usr/local/jdk1.8.0_202 apt-get clean && rm -rf /var/cache/apt/archives/* /var/cache/apt/archives/partial/* && rm -rf /var/lib/apt/lists/* #6 CACHED #7 [3/5] RUN wget -P /java-tron/config https://raw.githubusercontent.com/tronprotocol/tron-deployment/master/main_net_config.conf diff --git a/tools/docker/build.gradle b/tools/docker/build.gradle index e30036a7..70a2934f 100644 --- a/tools/docker/build.gradle +++ b/tools/docker/build.gradle @@ -11,8 +11,12 @@ import java.text.SimpleDateFormat // apply plugin: "java-library" //} +plugins { + id "java-library" + id "de.undercouch.download" version "5.6.0" +} + version = "1.0.0" -apply plugin: "java-library" def getGitCommitDetails(length = 8) { try { @@ -62,6 +66,8 @@ def dockerOrgName = project.hasProperty('dockerOrgName') ? project.getProperty(" def dockerArtifactName = project.hasProperty("dockerArtifactName") ? project.getProperty("dockerArtifactName") : "java-tron" def dockerImageName = "${dockerOrgName}/${dockerArtifactName}" def dockerNetwork = project.hasProperty("network") ? project.getProperty("network") : "mainnet" +def targetingPlatform = project.hasProperty("platform") ? project.getProperty("platform").toLowerCase() : "linux/amd64" +def imageArch = "amd64" task sourceDocker { def dockerBuildDir = "${rootDir}/build/docker-tron/" @@ -72,6 +78,13 @@ task sourceDocker { dockerfileName = "${dockerfileName}.${dockerNetwork}" } + if (targetingPlatform.contains("aarch64") || targetingPlatform.contains("arm64")) { + dockerfileName = "${dockerfileName}.arm64" + image = "${image}-arm64" + } else { + image = "${image}-amd64" + } + doLast { copy { from file("${rootDir}/../docker/${dockerfileName}") @@ -84,26 +97,87 @@ task sourceDocker { into "${dockerBuildDir}" } exec { - def dockerPlatform = "" - if (project.hasProperty('docker-platform')){ - dockerPlatform = "--platform ${project.getProperty('docker-platform')}" - println "Building for platform ${project.getProperty('docker-platform')}" - } else { - dockerPlatform = "--platform linux/amd64" - println "Building for default linux/amd64 platform" - } + def dockerPlatform = "--platform ${targetingPlatform}" + println "Building for platform ${targetingPlatform}" + def gitDetails = getGitCommitDetails(7) executable shell workingDir dockerBuildDir - args "-c", "docker build ${dockerPlatform} --build-arg BUILD_DATE=${buildTime()} --build-arg VERSION=${dockerBuildVersion} --build-arg VCS_REF=${gitDetails.hash} -t ${image} ." + /* if (targetingPlatform.contains("aarch64") || targetingPlatform.contains("arm64")) { + environment 'DOCKER_BUILDKIT', '0' + } */ + // Add `--provenance=false --sbom=false` to fix unknown dirty manifests: https://github.com/orgs/community/discussions/45969 + args "-c", "docker build --provenance=false --sbom=false --no-cache ${dockerPlatform} --build-arg BUILD_DATE=${buildTime()} --build-arg VERSION=${dockerBuildVersion} --build-arg VCS_REF=${gitDetails.hash} -t ${image} ." } } } +tasks.register("downloadGossBinaries") { + def gossVersion = 'v0.4.9' + def baseUrl = "https://github.com/goss-org/goss/releases/download/${gossVersion}" + def testDir = "${projectDir}/tests" + println "testDir:${testDir}" + + // current image architecture + def currentArch = imageArch + println "currentArch:${currentArch}" + + doLast { + // Download goss binary for current architecture + def binaryFile = new File(testDir, "goss-linux-${currentArch}") + + // Download binary if it doesn't exist + if (!binaryFile.exists()) { + println "Downloading goss-linux-${currentArch}..." + download.run { + src "${baseUrl}/goss-linux-${currentArch}" + dest binaryFile + overwrite false + onlyIfModified true + } + + binaryFile.setExecutable(true) + println "Successfully downloaded goss-linux-${currentArch}" + } else { + println "goss-linux-${currentArch} already exists" + } + + // Download dgoss if it doesn't exist + def dgossFile = new File(testDir, 'dgoss') + if (!dgossFile.exists()) { + println "Downloading dgoss..." + download.run { + src "${baseUrl}/dgoss" + dest dgossFile + overwrite false + onlyIfModified true + } + + dgossFile.setExecutable(true) + println "Successfully downloaded dgoss" + } else { + println "dgoss already exists" + } + } +} + task testDocker { def dockerReportsDir = "${rootDir}/../docker/reports/" def image = "${dockerImageName}:${dockerBuildVersion}" + if (targetingPlatform.contains("aarch64") || targetingPlatform.contains("arm64")) { + image = "${image}-arm64" + } else { + image = "${image}-amd64" + } + imageArch = ["sh", "-c", "docker inspect ${image} --format='{{.Architecture}}'"] + .execute() + .text + .trim() + + println "Image architecture: ${imageArch}" + + dependsOn downloadGossBinaries doFirst { new File(dockerReportsDir).mkdir() @@ -113,7 +187,7 @@ task testDocker { exec { workingDir "${rootDir}/../docker" executable shell - args "-c", "./test.sh ${image}" + args "-c", "architecture=${imageArch} ./test.sh ${image}" } } } @@ -121,15 +195,75 @@ task testDocker { // Make sure to `docker login` first task dockerUpload { def image = "${dockerImageName}:${dockerBuildVersion}" + def archs = [ + "arm64", + "amd64"] doLast { - exec { - def cmd = "docker push '${image}'" - println "Executing '${cmd}'" - executable shell - args "-c", cmd + archs.forEach { arch -> + println "Uploading for ${image}-${arch}..." + exec { + def cmd = "docker push '${image}-${arch}'" + println "Executing '${cmd}'" + executable shell + args "-c", cmd + } } + + } +} + +task manifestDocker { + def image = "${dockerImageName}:${dockerBuildVersion}" + def archs = [ + "arm64", + "amd64"] //TODO: this assumes dockerUpload task has already been run on 2 different archs! + doLast { + exec { + def targets = "" + archs.forEach { arch -> targets += "'${image}-${arch}' " } + def cmd = "docker manifest create '${image}' ${targets}" + println "Executing '${cmd}'" + executable shell + args "-c", cmd + } + exec { + def cmd = "docker manifest push '${image}'" + println "Executing '${cmd}'" + executable shell + args "-c", cmd + } + } +} + +task manifestDockerAsLatest { + def image = "${dockerImageName}:${dockerBuildVersion}" + def targetImage = "${dockerImageName}:latest" + def archs = [ + "arm64", + "amd64"] //TODO: this assumes dockerUpload task has already been run on 2 different archs! + doLast { + exec { + def cmd = "docker manifest rm '${targetImage}' " + println "Executing '${cmd}'" + executable shell + args "-c", cmd + } + exec { + def targets = "" + archs.forEach { arch -> targets += "'${image}-${arch}' " } + def cmd = "docker manifest create '${targetImage}' ${targets}" + println "Executing '${cmd}'" + executable shell + args "-c", cmd + } + exec { + def cmd = "docker manifest push '${targetImage}'" + println "Executing '${cmd}'" + executable shell + args "-c", cmd } + } } tasks.named('jar') { diff --git a/tools/docker/codepipeline/docker-compose.fullnode.main.yml b/tools/docker/codepipeline/docker-compose.fullnode.main.yml index a1b1d074..c61032df 100644 --- a/tools/docker/codepipeline/docker-compose.fullnode.main.yml +++ b/tools/docker/codepipeline/docker-compose.fullnode.main.yml @@ -18,6 +18,6 @@ services: - ../output-directory:/java-tron/output-directory command: - -jvm - - "{-Xmx30g -Xms24g -XX:MaxDirectMemorySize=1G -XX:+UseConcMarkSweepGC -XX:+PrintGC -Xloggc:./logs/gc.log }" # Using environment variable for JVM arguments + - "{-Xmx30g -Xms24g -XX:MaxDirectMemorySize=1G}" # Using environment variable for JVM arguments - -c - /java-tron/conf/main_net_config.conf diff --git a/tools/docker/codepipeline/docker-compose.fullnode.nile.yml b/tools/docker/codepipeline/docker-compose.fullnode.nile.yml index 501978a2..7238e95d 100644 --- a/tools/docker/codepipeline/docker-compose.fullnode.nile.yml +++ b/tools/docker/codepipeline/docker-compose.fullnode.nile.yml @@ -15,6 +15,6 @@ services: - ./output-directory/nile:/java-tron/output-directory command: - -jvm - - "{-Xmx5g -Xms4g -XX:+UseConcMarkSweepGC}" # Using environment variable for JVM arguments + - "{-Xmx5g -Xms4g}" # Using environment variable for JVM arguments - -c - /java-tron/conf/nile_net_config.conf diff --git a/tools/docker/docker_env/check-install-docker.sh b/tools/docker/docker_env/check-install-docker.sh index 9352aa3f..376f3fda 100755 --- a/tools/docker/docker_env/check-install-docker.sh +++ b/tools/docker/docker_env/check-install-docker.sh @@ -31,8 +31,42 @@ check_docker_compose() { exit 1 elif [[ "$OSTYPE" == "linux-gnu"* ]]; then # Linux - sudo curl -L "https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose - sudo chmod +x /usr/local/bin/docker-compose + compose_os="$(uname -s | tr '[:upper:]' '[:lower:]')" + compose_arch="$(uname -m)" + compose_binary="docker-compose-${compose_os}-${compose_arch}" + compose_version="$(curl -fsSLI https://github.com/docker/compose/releases/latest \ + | awk -F'/' '/^location:/ {gsub(/\r/,"",$NF); print $NF}')" + + if [[ -z "$compose_version" ]]; then + echo "Failed to determine latest Docker Compose version" + exit 1 + fi + + tmp_dir="$(mktemp -d)" + compose_url="https://github.com/docker/compose/releases/download/${compose_version}/${compose_binary}" + checksums_url="https://github.com/docker/compose/releases/download/${compose_version}/checksums.txt" + + if ! curl -fsSL "$compose_url" -o "${tmp_dir}/${compose_binary}"; then + rm -rf "$tmp_dir" + echo "Failed to download Docker Compose binary: ${compose_url}" + exit 1 + fi + + if ! curl -fsSL "$checksums_url" -o "${tmp_dir}/checksums.txt"; then + rm -rf "$tmp_dir" + echo "Failed to download Docker Compose checksums: ${checksums_url}" + exit 1 + fi + + if ! grep -E "[[:space:]]\\*${compose_binary}$" "${tmp_dir}/checksums.txt" \ + | (cd "$tmp_dir" && sha256sum -c -); then + rm -rf "$tmp_dir" + echo "Docker Compose checksum verification failed" + exit 1 + fi + + sudo install -m 0755 "${tmp_dir}/${compose_binary}" /usr/local/bin/docker-compose + rm -rf "$tmp_dir" fi fi } diff --git a/tools/docker/test.sh b/tools/docker/test.sh index b5402a84..ca2b2bad 100755 --- a/tools/docker/test.sh +++ b/tools/docker/test.sh @@ -14,14 +14,36 @@ ## SPDX-License-Identifier: Apache-2.0 ## +set -e # Exit immediately if any command fails + +# Detect architecture if not already set +if [ -z "$architecture" ]; then + case $(uname -m) in + x86_64) + architecture="amd64" + ;; + aarch64|arm64) + architecture="arm64" + ;; + arm*) + architecture="arm64" + ;; + *) + echo "Unsupported architecture: $(uname -m)" + exit 1 + ;; + esac +fi + export TEST_PATH=./tests -#export GOSS_PATH=$TEST_PATH/goss-linux-${architecture} # TODO. fixed by https://github.com/goss-org/goss/tree/master/extras/dgoss#mac-osx -export GOSS_PATH=$TEST_PATH/goss-linux-amd64 +export GOSS_PATH=$TEST_PATH/goss-linux-${architecture} +echo "GOSS_PATH is: ${GOSS_PATH}" +#export GOSS_PATH=$TEST_PATH/goss-linux-amd64 export GOSS_OPTS="$GOSS_OPTS --format junit" export GOSS_FILES_STRATEGY=cp DOCKER_IMAGE=$1 # shellcheck disable=SC2034 -DOCKER_FILE="${2:-$PWD/Dockerfile}" +#DOCKER_FILE="${2:-$PWD/Dockerfile}" i=0 # This initializes a counter variable i that will be used to track the number of failed tests. diff --git a/tools/docker/tests/dgoss b/tools/docker/tests/dgoss index 5608281f..ced8e3d9 100755 --- a/tools/docker/tests/dgoss +++ b/tools/docker/tests/dgoss @@ -80,7 +80,7 @@ trap 'ret=$?;cleanup;exit $ret' EXIT GOSS_PATH="${GOSS_PATH:-$(which goss 2> /dev/null || true)}" [[ $GOSS_PATH ]] || { error "Couldn't find goss installation, please set GOSS_PATH to it"; } [[ ${GOSS_OPTS+x} ]] || GOSS_OPTS="--color --format documentation" -[[ ${GOSS_WAIT_OPTS+x} ]] || GOSS_WAIT_OPTS="-r 30s -s 1s > /dev/null" +[[ ${GOSS_WAIT_OPTS+x} ]] || GOSS_WAIT_OPTS="-r 60s -s 2s > /dev/null" GOSS_SLEEP=${GOSS_SLEEP:-1.0} case "$1" in diff --git a/tools/docker/tests/goss-linux-arm64 b/tools/docker/tests/goss-linux-arm64 new file mode 100755 index 00000000..2b5244c7 Binary files /dev/null and b/tools/docker/tests/goss-linux-arm64 differ diff --git a/tools/gradlew/build.gradle b/tools/gradlew/build.gradle index 6029ef7e..60970845 100644 --- a/tools/gradlew/build.gradle +++ b/tools/gradlew/build.gradle @@ -1,7 +1,49 @@ +import org.gradle.nativeplatform.platform.internal.Architectures +import org.gradle.internal.os.OperatingSystem + plugins { id 'java' } +def arch = System.getProperty("os.arch").toLowerCase() +def javaVersion = JavaVersion.current() +def isArm64 = Architectures.AARCH64.isAlias(arch) +def archSource = isArm64 ? "arm" : "x86" +def isMac = OperatingSystem.current().isMacOsX() + +ext.archInfo = [ + name : arch, + java : javaVersion, + isArm64 : isArm64, + sourceSets: [ + main: [ + java: [ + srcDirs: ["src/main/java/common", "src/main/java/${archSource}"] + ] + ], + test: [ + java: [ + srcDirs: ["src/test/java"] + ] + ] + ], + requires: [ + JavaVersion: isArm64 ? JavaVersion.VERSION_17 : JavaVersion.VERSION_1_8, + RocksdbVersion: isArm64 ? '9.7.4' : '5.15.10', + // https://github.com/grpc/grpc-java/issues/7690 + // https://github.com/grpc/grpc-java/pull/12319, Add support for macOS aarch64 with universal binary + // https://github.com/grpc/grpc-java/pull/11371 , 1.64.x is not supported CentOS 7. + ProtocGenVersion: isArm64 && isMac ? '1.76.0' : '1.60.0' + ], + VMOptions: isArm64 ? "${rootDir}/gradle/jdk17/java-tron.vmoptions" : "${rootDir}/gradle/java-tron.vmoptions" +] + +if (!archInfo.java.is(archInfo.requires.JavaVersion)) { + throw new GradleException("Java ${archInfo.requires.JavaVersion} is required for ${archInfo.name}. Detected version ${archInfo.java}") +} + +println "Building for architecture: ${archInfo.name}, Java version: ${archInfo.java}" + tasks.named('jar') { enabled = false } diff --git a/tools/slack_sr_monitor/.env.example b/tools/slack_sr_monitor/.env.example new file mode 100644 index 00000000..59143800 --- /dev/null +++ b/tools/slack_sr_monitor/.env.example @@ -0,0 +1,22 @@ +# Slack SR Monitor Configuration + +# The Slack Webhook URL for sending notifications +SLACK_WEBHOOK=your_slack_webhook_url_here + +# Optional per-tool Slack webhook URLs. +# If not set, each tool falls back to SLACK_WEBHOOK. +# SLACK_SR_MONITOR_WEBHOOK=your_monitor_slack_webhook_url_here +# SLACK_SR_GUARD_WEBHOOK=your_guard_slack_webhook_url_here + +# The Tron node API endpoint +# Default: https://api.trongrid.io +TRON_NODE=https://api.trongrid.io + +# Optional comma-separated fallback Tron node API endpoints. +# If set, TRON_NODES takes precedence over TRON_NODE. +# TRON_NODES=https://api.trongrid.io,http://127.0.0.1:8090 + +# Optional: have slack_sr_guard also trigger a phone/voice call alert when +# the Top 27 changes, by POSTing a short message to this webhook endpoint. +# If unset, this is skipped entirely and only the Slack alert is sent. +# PHONE_ALERT_URL=your_phone_alert_webhook_url_here diff --git a/tools/slack_sr_monitor/.gitignore b/tools/slack_sr_monitor/.gitignore new file mode 100644 index 00000000..a38f37f7 --- /dev/null +++ b/tools/slack_sr_monitor/.gitignore @@ -0,0 +1,4 @@ +/slack_sr_monitor +/slack_sr_guard +logs/ +.env diff --git a/tools/slack_sr_monitor/Dockerfile b/tools/slack_sr_monitor/Dockerfile new file mode 100644 index 00000000..7c3b5dbd --- /dev/null +++ b/tools/slack_sr_monitor/Dockerfile @@ -0,0 +1,33 @@ +# Build stage +FROM golang:1.25-alpine@sha256:8d22e29d960bc50cd025d93d5b7c7d220b1ee9aa7a239b3c8f55a57e987e8d45 AS builder + +WORKDIR /app + +# Install dependencies +COPY go.mod go.sum ./ +RUN go mod download + +# Copy source code +COPY . . + +# Build the application +RUN CGO_ENABLED=0 GOOS=linux go build -o slack_sr_monitor . + +# Final stage +FROM alpine:3.20@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc + +RUN apk --no-cache add ca-certificates tzdata \ + && addgroup -S monitor && adduser -S -G monitor monitor + +WORKDIR /home/monitor + +# Create logs directory owned by the non-root user +RUN mkdir -p logs && chown -R monitor:monitor /home/monitor + +# Copy the binary from builder +COPY --from=builder --chown=monitor:monitor /app/slack_sr_monitor . + +USER monitor + +# Command to run +CMD ["./slack_sr_monitor"] diff --git a/tools/slack_sr_monitor/README.md b/tools/slack_sr_monitor/README.md new file mode 100644 index 00000000..395f4953 --- /dev/null +++ b/tools/slack_sr_monitor/README.md @@ -0,0 +1,97 @@ +## Slack SR Monitor Tool +The Slack SR Monitor tool is designed to monitor TRON Super Representatives (SRs) and notify a Slack channel after every maintenance period. +It automatically tracks vote changes and detects replacements in the top 27 SR positions, providing a clear and formatted report. + +This directory also includes `slack_sr_guard`, a lightweight early-warning tool that polls the live SR list every minute and alerts Slack as soon as the current Top 27 membership changes. + +### Build and Run +To run either tool, choose native Go execution or Docker deployment. + +#### Native Go Execution +Make sure you have Go 1.25+ installed. +```shell +# enter the directory +cd tools/slack_sr_monitor +# install dependencies +go mod tidy +# run both tools by default +go run . + +# run only the maintenance-period monitor +go run . monitor + +# run only the early-warning guard +go run . guard +``` + +#### Docker Deployment +We provide a Docker-based deployment for easier management in production environments. +```shell +# build and start both tools +docker compose up -d --build + +# build and start only one tool +docker compose up -d --build slack-sr-monitor +docker compose up -d --build slack-sr-guard + +# check logs +docker logs -f slack-sr-monitor +docker logs -f slack-sr-guard +``` + +### Configuration +All configurations are managed via environment variables or a `.env` file in the project root. Please refer to [.env.example](./.env.example) as an example. + +- `SLACK_WEBHOOK`: The Slack Incoming Webhook URL used to send notifications. +- `SLACK_SR_MONITOR_WEBHOOK`: Optional Slack webhook URL for `slack_sr_monitor`. Falls back to `SLACK_WEBHOOK` if unset. +- `SLACK_SR_GUARD_WEBHOOK`: Optional Slack webhook URL for `slack_sr_guard`. Falls back to `SLACK_WEBHOOK` if unset. +- `TRON_NODE`: The TRON node HTTP API endpoint (e.g., `http://https://api.trongrid.io`). Default is Trongrid. +- `TRON_NODES`: Optional comma-separated TRON node HTTP API endpoints. If set, the tools try each node in order for every TRON API request and only return an error after all nodes fail. This takes precedence over `TRON_NODE`. +- `PHONE_ALERT_URL`: Optional webhook endpoint for phone/voice call escalation. When set, `slack_sr_guard` sends a short alert message here on every Top 27 change, in addition to the Slack alert, so on-call staff can be reached even outside Slack. Skipped entirely when unset. + +### State Persistence + +Both tools persist lightweight JSON state under `logs/`, which is already mounted by Docker Compose: + +- `logs/sr_monitor_state.json`: previous vote counts and Top 27 snapshot for maintenance-period reports. +- `logs/sr_guard_state.json`: previous Top 27 snapshot and SR name cache for one-minute guard checks. + +If these files are deleted, the next run starts with a fresh baseline and recreates them after the first successful fetch. + +### Key Features + +#### SR vote monitor +Use `/wallet/getpaginatednowwitnesslist` to get the top **28** real-time votes, also the SR address and URL. + +#### Dynamic Scheduling +Instead of a fixed interval, the tool queries `/wallet/getnextmaintenancetime` to calculate the exact wait time. It triggers the report **1 minute** after each maintenance period begins to ensure data consistency. + +#### Parallel Data Acquisition +The tool uses Go routines to fetch `account_name` for all 28 witnesses in parallel from the `/wallet/getaccount` interface, significantly reducing the collection time. + +#### Vote Change Tracking +The tool persists a snapshot of the previous period's votes. It calculates the `Change` for each SR: +```text +*1. Poloniex* +Current: `3,228,089,488` Change: `+89,488` +``` + +#### Top 27 Replacement Detection +After each report, it compares the current Top 27 list with the previous one and highlights any changes: +```text +SR Replacement Detected: +>:inbox_tray: *Entered:* New_SR_Name +>:outbox_tray: *Left:* Old_SR_Name +``` +If no changes occur, it displays `Top 27 SRs remain unchanged.` + +#### Top 27 Early Warning Guard +The `slack_sr_guard` process calls `/wallet/getpaginatednowwitnesslist` every minute with `limit=28`, keeps a persisted Top 27 snapshot, and sends a Slack alert only when an SR enters or leaves the Top 27 before the next maintenance period. + +The alert includes the entered SRs, left SRs, current 27th/28th vote gap, and a Top 28 vote table. + +When `PHONE_ALERT_URL` is configured, the same Top 27 change also triggers a phone call escalation, independent of the Slack alert (a failure to reach it is logged but never blocks or fails the Slack notification). + +### Notifications + +This monitor only support java-tron node v4.8.1+, because of the API it used. diff --git a/tools/slack_sr_monitor/docker-compose.yml b/tools/slack_sr_monitor/docker-compose.yml new file mode 100644 index 00000000..cae336fc --- /dev/null +++ b/tools/slack_sr_monitor/docker-compose.yml @@ -0,0 +1,27 @@ +services: + slack-sr-monitor: + build: . + container_name: slack-sr-monitor + restart: unless-stopped + command: ["./slack_sr_monitor", "monitor"] + environment: + - SLACK_WEBHOOK=${SLACK_WEBHOOK} + - SLACK_SR_MONITOR_WEBHOOK=${SLACK_SR_MONITOR_WEBHOOK:-} + - TRON_NODES=${TRON_NODES:-} + - TRON_NODE=${TRON_NODE:-https://api.trongrid.io} + volumes: + - ./logs:/home/monitor/logs + + slack-sr-guard: + build: . + container_name: slack-sr-guard + restart: unless-stopped + command: ["./slack_sr_monitor", "guard"] + environment: + - SLACK_WEBHOOK=${SLACK_WEBHOOK} + - SLACK_SR_GUARD_WEBHOOK=${SLACK_SR_GUARD_WEBHOOK:-} + - TRON_NODES=${TRON_NODES:-} + - TRON_NODE=${TRON_NODE:-https://api.trongrid.io} + - PHONE_ALERT_URL=${PHONE_ALERT_URL:-} + volumes: + - ./logs:/home/monitor/logs diff --git a/tools/slack_sr_monitor/go.mod b/tools/slack_sr_monitor/go.mod new file mode 100644 index 00000000..f2cf959b --- /dev/null +++ b/tools/slack_sr_monitor/go.mod @@ -0,0 +1,5 @@ +module github.com/tronprotocol/tron-docker/tools/slack_sr_monitor + +go 1.25.5 + +require github.com/joho/godotenv v1.5.1 diff --git a/tools/slack_sr_monitor/go.sum b/tools/slack_sr_monitor/go.sum new file mode 100644 index 00000000..d61b19e1 --- /dev/null +++ b/tools/slack_sr_monitor/go.sum @@ -0,0 +1,2 @@ +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= diff --git a/tools/slack_sr_monitor/main.go b/tools/slack_sr_monitor/main.go new file mode 100644 index 00000000..b8c9793d --- /dev/null +++ b/tools/slack_sr_monitor/main.go @@ -0,0 +1,87 @@ +package main + +import ( + "fmt" + "io" + "log" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/joho/godotenv" +) + +func setupLogging() { + logDir := "logs" + if _, err := os.Stat(logDir); os.IsNotExist(err) { + _ = os.Mkdir(logDir, 0755) + } + + logPath := filepath.Join(logDir, "slack_sr_tools.log") + logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) + if err != nil { + fmt.Printf("Error opening log file: %v, falling back to stdout only\n", err) + } else { + log.SetOutput(io.MultiWriter(os.Stdout, logFile)) + } + log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) +} + +func main() { + setupLogging() + + if err := godotenv.Load(); err != nil { + log.Println("Warning: No .env file found, using system environment variables") + } + + tronNodes := parseTronNodes() + mode := "both" + if len(os.Args) > 1 { + mode = strings.ToLower(os.Args[1]) + } + + switch mode { + case "monitor", "sr-monitor", "slack-sr-monitor": + slackWebhook := getSlackWebhook("SLACK_SR_MONITOR_WEBHOOK") + if slackWebhook == "" { + log.Println("Error: SLACK_SR_MONITOR_WEBHOOK or SLACK_WEBHOOK environment variable is not set") + log.Println("Usage: SLACK_SR_MONITOR_WEBHOOK=https://hooks.slack.com/... [TRON_NODES=...] go run . monitor") + os.Exit(1) + } + runSRMonitor(tronNodes, slackWebhook) + case "guard", "sr-guard", "slack-sr-guard": + slackWebhook := getSlackWebhook("SLACK_SR_GUARD_WEBHOOK") + if slackWebhook == "" { + log.Println("Error: SLACK_SR_GUARD_WEBHOOK or SLACK_WEBHOOK environment variable is not set") + log.Println("Usage: SLACK_SR_GUARD_WEBHOOK=https://hooks.slack.com/... [TRON_NODES=...] go run . guard") + os.Exit(1) + } + runSRGuard(tronNodes, slackWebhook, getPhoneAlertURL()) + case "both", "": + monitorWebhook := getSlackWebhook("SLACK_SR_MONITOR_WEBHOOK") + guardWebhook := getSlackWebhook("SLACK_SR_GUARD_WEBHOOK") + if monitorWebhook == "" || guardWebhook == "" { + log.Println("Error: SLACK_SR_MONITOR_WEBHOOK/SLACK_SR_GUARD_WEBHOOK or SLACK_WEBHOOK environment variable is not set") + log.Println("Usage: SLACK_WEBHOOK=https://hooks.slack.com/... [TRON_NODES=...] go run .") + os.Exit(1) + } + phoneAlertURL := getPhoneAlertURL() + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + runSRMonitor(tronNodes, monitorWebhook) + }() + go func() { + defer wg.Done() + runSRGuard(tronNodes, guardWebhook, phoneAlertURL) + }() + wg.Wait() + default: + log.Printf("Error: unknown mode %q", mode) + log.Println("Usage: go run . [monitor|guard|both]") + os.Exit(1) + } +} diff --git a/tools/slack_sr_monitor/sr_guard.go b/tools/slack_sr_monitor/sr_guard.go new file mode 100644 index 00000000..75b5c9a2 --- /dev/null +++ b/tools/slack_sr_monitor/sr_guard.go @@ -0,0 +1,420 @@ +package main + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +const ( + pollInterval = time.Minute + topSRLimit = 27 + fetchLimit = 28 + guardStatePath = "logs/sr_guard_state.json" + guardStateVersion = 1 +) + +type guardState struct { + Version int `json:"version"` + UpdatedAt time.Time `json:"updated_at"` + PreviousTop27 []string `json:"previous_top_27"` + NameCache map[string]string `json:"name_cache"` +} + +func getGuardWitnessListFromNodes(nodeURLs []string) ([]Witness, error) { + payload := []byte(fmt.Sprintf(`{"offset":0,"limit":%d,"visible":true}`, fetchLimit)) + var errors []string + for _, nodeURL := range nodeURLs { + body, _, err := postToTron([]string{nodeURL}, "/wallet/getpaginatednowwitnesslist", payload, 10*time.Second) + if err != nil { + errors = append(errors, err.Error()) + continue + } + + var result WitnessListResponse + if err := json.Unmarshal(body, &result); err != nil { + errors = append(errors, fmt.Sprintf("%s: failed to unmarshal JSON: %v", nodeURL, err)) + continue + } + + return result.Witnesses, nil + } + + return nil, fmt.Errorf("all TRON nodes failed for /wallet/getpaginatednowwitnesslist: %s", strings.Join(errors, "; ")) +} + +func getGuardAccountNameFromNodes(nodeURLs []string, address string) string { + payload := map[string]interface{}{ + "address": address, + "visible": true, + } + jsonPayload, err := json.Marshal(payload) + if err != nil { + log.Printf("failed to marshal account name payload for %s: %v", address, err) + return "" + } + + var errors []string + for _, nodeURL := range nodeURLs { + body, _, err := postToTron([]string{nodeURL}, "/wallet/getaccount", jsonPayload, 5*time.Second) + if err != nil { + errors = append(errors, err.Error()) + continue + } + + name, ok := parseAccountName(address, body) + if !ok { + errors = append(errors, fmt.Sprintf("%s: invalid account JSON", nodeURL)) + continue + } + + return name + } + + log.Printf("failed to request account name for %s: %s", address, strings.Join(errors, "; ")) + return "" +} + +func parseAccountName(address string, body []byte) (string, bool) { + var data map[string]interface{} + if err := json.Unmarshal(body, &data); err != nil { + log.Printf("failed to unmarshal account JSON for %s: %v", address, err) + return "", false + } + + val, ok := data["account_name"] + if !ok { + return "", true + } + + str, ok := val.(string) + if !ok { + str = fmt.Sprintf("%v", val) + } + decoded, err := hex.DecodeString(str) + if err != nil { + return str, true + } + return string(decoded), true +} + +func fillDisplayNames(nodeURLs []string, witnesses []Witness, nameCache map[string]string) { + var wg sync.WaitGroup + var mu sync.Mutex + + for i := range witnesses { + if name, ok := nameCache[witnesses[i].Address]; ok { + witnesses[i].DisplayName = name + continue + } + + wg.Add(1) + go func(idx int) { + defer wg.Done() + + witness := witnesses[idx] + name := getGuardAccountNameFromNodes(nodeURLs, witness.Address) + if name == "" { + name = witness.URL + } + if name == "" { + name = witness.Address + } + + mu.Lock() + nameCache[witness.Address] = name + witnesses[idx].DisplayName = name + mu.Unlock() + }(i) + } + + wg.Wait() +} + +func topAddresses(witnesses []Witness) []string { + limit := topSRLimit + if len(witnesses) < limit { + limit = len(witnesses) + } + + addresses := make([]string, 0, limit) + for i := 0; i < limit; i++ { + addresses = append(addresses, witnesses[i].Address) + } + return addresses +} + +func findTopSRChanges(previous []string, current []Witness) ([]Witness, []string) { + previousSet := make(map[string]bool, len(previous)) + for _, address := range previous { + previousSet[address] = true + } + + currentSet := make(map[string]bool, topSRLimit) + var entered []Witness + for i := 0; i < topSRLimit && i < len(current); i++ { + witness := current[i] + currentSet[witness.Address] = true + if !previousSet[witness.Address] { + entered = append(entered, witness) + } + } + + var left []string + for _, address := range previous { + if !currentSet[address] { + left = append(left, address) + } + } + + return entered, left +} + +func witnessName(w Witness) string { + if w.DisplayName != "" { + return w.DisplayName + } + if w.URL != "" { + return w.URL + } + return w.Address +} + +func sendAlert(webhookURL string, witnesses []Witness, entered []Witness, left []string, nameCache map[string]string) error { + var text strings.Builder + text.WriteString("*:warning: TRON Top 27 SR Change Detected*\n") + text.WriteString(fmt.Sprintf("Time: %s\n", time.Now().UTC().Format(time.RFC1123))) + + if len(entered) > 0 { + var names []string + for _, witness := range entered { + names = append(names, fmt.Sprintf("%s (`%s` votes)", witnessName(witness), formatComma(witness.VoteCount))) + } + text.WriteString(fmt.Sprintf(">:inbox_tray: *Entered Top 27:* %s\n", strings.Join(names, ", "))) + } + + if len(left) > 0 { + var names []string + for _, address := range left { + name := nameCache[address] + if name == "" { + name = address + } + names = append(names, name) + } + text.WriteString(fmt.Sprintf(">:outbox_tray: *Left Top 27:* %s\n", strings.Join(names, ", "))) + } + + var details strings.Builder + details.WriteString("```\n") + for i, witness := range witnesses { + details.WriteString(fmt.Sprintf("%2d. %-25s %15s\n", i+1, witnessName(witness), formatComma(witness.VoteCount))) + } + details.WriteString("```") + + payload := map[string]interface{}{ + "text": text.String(), + "attachments": []map[string]interface{}{ + { + "text": details.String(), + "color": "#e01e5a", + }, + }, + } + + jsonPayload, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("failed to marshal slack payload: %v", err) + } + + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Post(webhookURL, "application/json", bytes.NewBuffer(jsonPayload)) + if err != nil { + return fmt.Errorf("failed to send to slack: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("slack returned status %d: %s", resp.StatusCode, string(body)) + } + + return nil +} + +// buildPhoneAlertSummary produces a plain-text (no Slack markdown) summary, describing +// which SRs entered/left the Top 27, for the phone alert message. +func buildPhoneAlertSummary(entered []Witness, left []string, nameCache map[string]string) string { + var parts []string + + if len(entered) > 0 { + var names []string + for _, witness := range entered { + names = append(names, witnessName(witness)) + } + parts = append(parts, fmt.Sprintf("entered: %s", strings.Join(names, ", "))) + } + + if len(left) > 0 { + var names []string + for _, address := range left { + name := nameCache[address] + if name == "" { + name = address + } + names = append(names, name) + } + parts = append(parts, fmt.Sprintf("left: %s", strings.Join(names, ", "))) + } + + return "TRON Top 27 SR change detected (" + strings.Join(parts, "; ") + ")" +} + +// triggerPhoneAlert posts a short message to a webhook that triggers a phone/voice +// call alert after the Slack alert has been delivered successfully. +func triggerPhoneAlert(phoneAlertURL string, entered []Witness, left []string, nameCache map[string]string) error { + summary := buildPhoneAlertSummary(entered, left, nameCache) + + payload := map[string]string{ + "message": summary, + } + + jsonPayload, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("failed to marshal phone alert payload: %v", err) + } + + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Post(phoneAlertURL, "application/json", bytes.NewBuffer(jsonPayload)) + if err != nil { + return fmt.Errorf("failed to send phone alert: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("phone alert endpoint returned status %d: %s", resp.StatusCode, string(body)) + } + + return nil +} + +func loadGuardState() guardState { + state := guardState{ + Version: guardStateVersion, + NameCache: make(map[string]string), + } + + body, err := os.ReadFile(guardStatePath) + if err != nil { + if !os.IsNotExist(err) { + log.Printf("failed to read guard state: %v", err) + } + return state + } + + if err := json.Unmarshal(body, &state); err != nil { + log.Printf("failed to parse guard state: %v", err) + return guardState{Version: guardStateVersion, NameCache: make(map[string]string)} + } + if state.NameCache == nil { + state.NameCache = make(map[string]string) + } + log.Printf("loaded guard state: top27=%d names=%d updated_at=%s", len(state.PreviousTop27), len(state.NameCache), state.UpdatedAt.UTC().Format(time.RFC3339)) + return state +} + +func saveGuardState(previousTop27 []string, nameCache map[string]string) { + state := guardState{ + Version: guardStateVersion, + UpdatedAt: time.Now().UTC(), + PreviousTop27: previousTop27, + NameCache: nameCache, + } + body, err := json.MarshalIndent(state, "", " ") + if err != nil { + log.Printf("failed to marshal guard state: %v", err) + return + } + if err := os.MkdirAll(filepath.Dir(guardStatePath), 0755); err != nil { + log.Printf("failed to create guard state directory: %v", err) + return + } + if err := os.WriteFile(guardStatePath, body, 0644); err != nil { + log.Printf("failed to write guard state: %v", err) + } +} + +func isBlockFresh(blockTime time.Time) bool { + return time.Since(blockTime) <= maxBlockAge +} + +func runSRGuard(tronNodes []string, slackWebhook string, phoneAlertURL string) { + phoneLog := "disabled" + if phoneAlertURL != "" { + phoneLog = maskWebhook(phoneAlertURL) + } + log.Printf("Starting SR guard. Nodes: %s Slack Webhook: %s Phone Alert URL: %s Poll interval: %s", + strings.Join(tronNodes, ", "), maskWebhook(slackWebhook), phoneLog, pollInterval) + + state := loadGuardState() + nameCache := state.NameCache + previousTop27 := state.PreviousTop27 + ticker := time.NewTicker(pollInterval) + defer ticker.Stop() + + for { + blockTime, err := getLatestBlockTimeFromNodes(tronNodes) + if err != nil { + log.Printf("skipping SR guard round: failed to get latest block time: %v", err) + } else if !isBlockFresh(blockTime) { + log.Printf("skipping SR guard round: latest block is stale (age=%s)", time.Since(blockTime)) + } else if witnesses, err := getGuardWitnessListFromNodes(tronNodes); err != nil { + log.Printf("failed to fetch witness list: %v", err) + } else { + fillDisplayNames(tronNodes, witnesses, nameCache) + + if len(previousTop27) == 0 { + previousTop27 = topAddresses(witnesses) + saveGuardState(previousTop27, nameCache) + log.Printf("initialized Top 27 snapshot with %d witnesses", len(previousTop27)) + } else { + entered, left := findTopSRChanges(previousTop27, witnesses) + if len(entered) > 0 || len(left) > 0 { + log.Printf("Top 27 change detected: entered=%d left=%d", len(entered), len(left)) + + if err := sendAlert(slackWebhook, witnesses, entered, left, nameCache); err != nil { + log.Printf("failed to send Slack alert: %v", err) + } else { + previousTop27 = topAddresses(witnesses) + saveGuardState(previousTop27, nameCache) + log.Println("sent Slack alert and updated Top 27 snapshot") + + if phoneAlertURL != "" { + if err := triggerPhoneAlert(phoneAlertURL, entered, left, nameCache); err != nil { + log.Printf("failed to trigger phone alert: %v", err) + } else { + log.Println("triggered phone alert") + } + } + } + } else { + previousTop27 = topAddresses(witnesses) + saveGuardState(previousTop27, nameCache) + log.Println("Top 27 unchanged") + } + } + } + + <-ticker.C + } +} diff --git a/tools/slack_sr_monitor/sr_guard_freshness_test.go b/tools/slack_sr_monitor/sr_guard_freshness_test.go new file mode 100644 index 00000000..a2ba65bf --- /dev/null +++ b/tools/slack_sr_monitor/sr_guard_freshness_test.go @@ -0,0 +1,15 @@ +package main + +import ( + "testing" + "time" +) + +func TestIsBlockFresh(t *testing.T) { + if !isBlockFresh(time.Now()) { + t.Error("expected current block time to be fresh") + } + if isBlockFresh(time.Date(2018, time.January, 1, 0, 0, 0, 0, time.UTC)) { + t.Error("expected year-2018 block time to be stale") + } +} diff --git a/tools/slack_sr_monitor/sr_guard_phone_test.go b/tools/slack_sr_monitor/sr_guard_phone_test.go new file mode 100644 index 00000000..4982cabd --- /dev/null +++ b/tools/slack_sr_monitor/sr_guard_phone_test.go @@ -0,0 +1,40 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestTriggerPhoneAlert(t *testing.T) { + var gotBody map[string]interface{} + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil { + t.Fatalf("failed to decode request body: %v", err) + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + entered := []Witness{{Address: "TEntered1", DisplayName: "New SR", VoteCount: 1000}} + left := []string{"TLeft1"} + nameCache := map[string]string{"TLeft1": "Old SR"} + + if err := triggerPhoneAlert(server.URL, entered, left, nameCache); err != nil { + t.Fatalf("triggerPhoneAlert returned error: %v", err) + } + + message, _ := gotBody["message"].(string) + if message == "" { + t.Fatalf("expected non-empty message field") + } + if !strings.Contains(message, "New SR") { + t.Errorf("expected message to mention entered witness 'New SR', got %q", message) + } + if !strings.Contains(message, "Old SR") { + t.Errorf("expected message to mention left witness 'Old SR', got %q", message) + } +} diff --git a/tools/slack_sr_monitor/sr_monitor.go b/tools/slack_sr_monitor/sr_monitor.go new file mode 100644 index 00000000..39c4ab75 --- /dev/null +++ b/tools/slack_sr_monitor/sr_monitor.go @@ -0,0 +1,667 @@ +package main + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +const ( + DefaultTronNode = "https://api.trongrid.io" + monitorStatePath = "logs/sr_monitor_state.json" + monitorStateVersion = 1 +) + +type monitorState struct { + Version int `json:"version"` + UpdatedAt time.Time `json:"updated_at"` + LastVotes map[string]int64 `json:"last_votes"` + LastTop27 []string `json:"last_top_27"` +} + +// Witness represents the structure of a witness returned by the Tron API +type Witness struct { + Address string `json:"address"` + VoteCount int64 `json:"voteCount"` + PubKey string `json:"pubKey"` + URL string `json:"url"` + TotalProduced int64 `json:"totalProduced"` + TotalMissed int64 `json:"totalMissed"` + LatestBlock int64 `json:"latestBlockNum"` + IsJobs bool `json:"isJobs"` + DisplayName string `json:"-"` +} + +func getAccountNameFromNodes(nodeURLs []string, address string) string { + payload := map[string]interface{}{ + "address": address, + "visible": true, + } + jsonPayload, err := json.Marshal(payload) + if err != nil { + log.Printf("Error: failed to marshal account name payload for %s: %v", address, err) + return "" + } + + var errors []string + for _, nodeURL := range nodeURLs { + body, _, err := postToTron([]string{nodeURL}, "/wallet/getaccount", jsonPayload, 5*time.Second) + if err != nil { + errors = append(errors, err.Error()) + continue + } + + var data map[string]interface{} + if err := json.Unmarshal(body, &data); err != nil { + errors = append(errors, fmt.Sprintf("%s: failed to unmarshal account JSON: %v", nodeURL, err)) + continue + } + + if val, ok := data["account_name"]; ok { + if str, ok := val.(string); ok { + return str + } + return fmt.Sprintf("%v", val) + } + + return "" + } + + log.Printf("Error: failed to request account name for %s: %s", address, strings.Join(errors, "; ")) + return "" +} + +func parseTronNodes() []string { + raw := os.Getenv("TRON_NODES") + if raw == "" { + raw = os.Getenv("TRON_NODE") + } + if raw == "" { + raw = DefaultTronNode + } + + var nodes []string + for _, node := range strings.Split(raw, ",") { + node = strings.TrimSpace(node) + if node != "" { + nodes = append(nodes, strings.TrimRight(node, "/")) + } + } + if len(nodes) == 0 { + return []string{DefaultTronNode} + } + return nodes +} + +func getSlackWebhook(specificEnv string) string { + if webhook := os.Getenv(specificEnv); webhook != "" { + return webhook + } + return os.Getenv("SLACK_WEBHOOK") +} + +// getPhoneAlertURL returns the Phone_AWS API Gateway endpoint used by slack_sr_guard to +// trigger a phone call directly. Optional; when unset, the phone alert is skipped. +func getPhoneAlertURL() string { + return os.Getenv("PHONE_ALERT_URL") +} + +func postToTron(nodeURLs []string, path string, payload []byte, timeout time.Duration) ([]byte, string, error) { + client := &http.Client{Timeout: timeout} + var errors []string + + for _, nodeURL := range nodeURLs { + resp, err := client.Post(nodeURL+path, "application/json", bytes.NewBuffer(payload)) + if err != nil { + errors = append(errors, fmt.Sprintf("%s: %v", nodeURL, err)) + continue + } + + body, readErr := io.ReadAll(resp.Body) + resp.Body.Close() + if readErr != nil { + errors = append(errors, fmt.Sprintf("%s: failed to read response body: %v", nodeURL, readErr)) + continue + } + if resp.StatusCode != http.StatusOK { + errors = append(errors, fmt.Sprintf("%s: status %d: %s", nodeURL, resp.StatusCode, string(body))) + continue + } + + return body, nodeURL, nil + } + + return nil, "", fmt.Errorf("all TRON nodes failed for %s: %s", path, strings.Join(errors, "; ")) +} + +// WitnessListResponse is the wrapper for the witness list API response +type WitnessListResponse struct { + Witnesses []Witness `json:"witnesses"` +} + +// NextMaintenanceResponse is the wrapper for the maintenance time API response +type NextMaintenanceResponse struct { + Num int64 `json:"num"` +} + +func getNextMaintenanceTimeFromNodes(nodeURLs []string) (time.Time, error) { + var errors []string + for _, nodeURL := range nodeURLs { + body, _, err := postToTron([]string{nodeURL}, "/wallet/getnextmaintenancetime", nil, 10*time.Second) + if err != nil { + errors = append(errors, err.Error()) + continue + } + + var result NextMaintenanceResponse + if err := json.Unmarshal(body, &result); err != nil { + errors = append(errors, fmt.Sprintf("%s: failed to unmarshal next maintenance JSON: %v", nodeURL, err)) + continue + } + + return time.Unix(result.Num/1000, (result.Num%1000)*1000000), nil + } + + return time.Time{}, fmt.Errorf("all TRON nodes failed for /wallet/getnextmaintenancetime: %s", strings.Join(errors, "; ")) +} + +// NowBlockResponse captures the timestamp of the node's latest block. +type NowBlockResponse struct { + BlockHeader struct { + RawData struct { + Number int64 `json:"number"` + Timestamp int64 `json:"timestamp"` + } `json:"raw_data"` + } `json:"block_header"` +} + +// A fresh TRON node should always be within a few block intervals (block time is 3s). +// Anything beyond this threshold means the node is still catching up to the chain head. +const ( + maxBlockAge = 30 * time.Second + freshnessRetryInterval = 2 * time.Minute + freshnessRetryTimeout = 2 * time.Hour + witnessRetryInterval = 30 * time.Second + witnessRetryTimeout = 10 * time.Minute +) + +func getLatestBlockTimeFromNodes(nodeURLs []string) (time.Time, error) { + var errors []string + for _, nodeURL := range nodeURLs { + body, _, err := postToTron([]string{nodeURL}, "/wallet/getnowblock", nil, 10*time.Second) + if err != nil { + errors = append(errors, err.Error()) + continue + } + + var result NowBlockResponse + if err := json.Unmarshal(body, &result); err != nil { + errors = append(errors, fmt.Sprintf("%s: failed to unmarshal now block JSON: %v", nodeURL, err)) + continue + } + + ts := result.BlockHeader.RawData.Timestamp + return time.Unix(ts/1000, (ts%1000)*1000000), nil + } + + return time.Time{}, fmt.Errorf("all TRON nodes failed for /wallet/getnowblock: %s", strings.Join(errors, "; ")) +} + +func getWitnessListFromNodes(nodeURLs []string) ([]Witness, error) { + payload := []byte(`{"offset": 0, "limit": 28, "visible": true}`) + var errors []string + for _, nodeURL := range nodeURLs { + body, _, err := postToTron([]string{nodeURL}, "/wallet/getpaginatednowwitnesslist", payload, 10*time.Second) + if err != nil { + errors = append(errors, err.Error()) + continue + } + + witnesses, err := parseWitnessList(nodeURLs, body) + if err != nil { + errors = append(errors, fmt.Sprintf("%s: %v", nodeURL, err)) + continue + } + + return witnesses, nil + } + + return nil, fmt.Errorf("all TRON nodes failed for /wallet/getpaginatednowwitnesslist: %s", strings.Join(errors, "; ")) +} + +func waitForFreshNode(nodeURLs []string) error { + deadline := time.Now().Add(freshnessRetryTimeout) + + for { + blockTime, err := getLatestBlockTimeFromNodes(nodeURLs) + if err != nil { + log.Printf("Error fetching latest block: %v", err) + } else if age := time.Since(blockTime); age <= maxBlockAge { + log.Printf("Node is fresh (latest block at %s, age %s)", blockTime.UTC().Format(time.RFC3339), age.Truncate(time.Second)) + return nil + } else { + log.Printf("Node is %s behind chain head (latest block at %s), waiting %s...", + age.Truncate(time.Second), blockTime.UTC().Format(time.RFC3339), freshnessRetryInterval) + } + + if time.Now().Add(freshnessRetryInterval).After(deadline) { + return fmt.Errorf("node did not become fresh within %s", freshnessRetryTimeout) + } + time.Sleep(freshnessRetryInterval) + } +} + +func getWitnessListWithRetry(nodeURLs []string) ([]Witness, error) { + deadline := time.Now().Add(witnessRetryTimeout) + + for { + witnesses, err := getWitnessListFromNodes(nodeURLs) + if err == nil { + return witnesses, nil + } + + log.Printf("Error fetching witness list: %v", err) + if time.Now().Add(witnessRetryInterval).After(deadline) { + return nil, fmt.Errorf("failed to fetch witness list within %s: %v", witnessRetryTimeout, err) + } + log.Printf("Retrying witness list in %s...", witnessRetryInterval) + time.Sleep(witnessRetryInterval) + } +} + +func parseWitnessList(nodeURLs []string, body []byte) ([]Witness, error) { + var result WitnessListResponse + if err := json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("failed to unmarshal JSON: %v (body: %s)", err, string(body)) + } + + // For each witness, try to get the account name in parallel + log.Printf("Fetching account names for %d witnesses in parallel...\n", len(result.Witnesses)) + var wg sync.WaitGroup + for i := range result.Witnesses { + wg.Add(1) + go func(idx int) { + defer wg.Done() + w := &result.Witnesses[idx] + accName := getAccountNameFromNodes(nodeURLs, w.Address) + if accName != "" { + if decoded, err := hex.DecodeString(accName); err == nil { + w.DisplayName = string(decoded) + } else { + w.DisplayName = accName + } + } + if w.DisplayName == "" { + w.DisplayName = w.URL + } + }(i) + } + wg.Wait() + + return result.Witnesses, nil +} + +func formatComma(n int64) string { + in := fmt.Sprintf("%d", n) + var out strings.Builder + if n < 0 { + out.WriteByte('-') + in = in[1:] + } + l := len(in) + for i, c := range in { + if i > 0 && (l-i)%3 == 0 { + out.WriteByte(',') + } + out.WriteRune(c) + } + return out.String() +} + +func sendToSlack(webhookURL string, nodeURLs []string, witnesses []Witness, prevVotes map[string]int64, prevSRs []string) error { + var summary strings.Builder + summary.WriteString("*TRON SR Status Update (Maintenance Period)*") + summary.WriteString(fmt.Sprintf(" Time: %s\n", time.Now().UTC().Format(time.RFC1123))) + + // 1. Calculate SR changes for the summary + if len(prevSRs) > 0 { + currentTop27 := make(map[string]bool) + for i := 0; i < 27 && i < len(witnesses); i++ { + currentTop27[witnesses[i].Address] = true + } + + prevTop27 := make(map[string]bool) + for _, addr := range prevSRs { + prevTop27[addr] = true + } + + var entered []string + var left []string + + // Who enter + for i := 0; i < 27 && i < len(witnesses); i++ { + w := witnesses[i] + if !prevTop27[w.Address] { + name := w.DisplayName + if name == "" { + name = w.Address + } + entered = append(entered, name) + } + } + + // Who left + for _, addr := range prevSRs { + if !currentTop27[addr] { + name := addr + foundInCurrentList := false + for _, w := range witnesses { + if w.Address == addr { + foundInCurrentList = true + name = w.DisplayName + if name == "" { + name = w.Address + } + break + } + } + if !foundInCurrentList { + if accountName := getAccountNameFromNodes(nodeURLs, addr); accountName != "" { + name = accountName + } + } + left = append(left, name) + } + } + + // Build rank map for previous SRs + prevRankMap := make(map[string]int) + for i, addr := range prevSRs { + prevRankMap[addr] = i + 1 // Rank is 1-based + } + + // Detect rank changes within top 27 + var rankChanges []string + for i := 0; i < 27 && i < len(witnesses); i++ { + w := witnesses[i] + if prevRank, ok := prevRankMap[w.Address]; ok { + change := prevRank - (i + 1) // Positive = rank up, negative = rank down + if change != 0 { + direction := "↑" + if change < 0 { + direction = "↓" + change = -change + } + name := w.DisplayName + if name == "" { + name = w.Address + } + rankChanges = append(rankChanges, + fmt.Sprintf("%s: %d → %d (%s%d)", name, prevRank, i+1, direction, change)) + } + } + } + + // Output all changes (entering, leaving, and rank changes) in one block + if len(entered) > 0 || len(left) > 0 || len(rankChanges) > 0 { + summary.WriteString("*SR Changes Detected:*\n") + if len(entered) > 0 { + summary.WriteString(fmt.Sprintf(">:inbox_tray: *Entered:* %s\n", strings.Join(entered, ", "))) + } + if len(left) > 0 { + summary.WriteString(fmt.Sprintf(">:outbox_tray: *Left:* %s\n", strings.Join(left, ", "))) + } + if len(rankChanges) > 0 { + summary.WriteString(">*Rank Changes:*\n") + for _, rc := range rankChanges { + summary.WriteString(fmt.Sprintf("> `%s`\n", rc)) + } + } + } else { + summary.WriteString("*Top 27 SRs remain unchanged.*\n") + } + } else { + summary.WriteString("*First check, initializing SR list*\n") + } + + // Calculate gap between 27th and 28th SR (always show this if we have enough witnesses) + if len(witnesses) >= 28 { + v27 := witnesses[26].VoteCount + v28 := witnesses[27].VoteCount + gap := v27 - v28 + summary.WriteString(fmt.Sprintf("*Gap (27 vs 28):* `%s` votes", formatComma(gap))) + } + + // 2. Build detailed list for attachment + var details strings.Builder + details.WriteString("```\n") + for i, w := range witnesses { + name := w.DisplayName + if name == "" { + name = w.Address + } + + prev := prevVotes[w.Address] + diff := w.VoteCount - prev + + diffStr := formatComma(diff) + if diff >= 0 { + diffStr = "+" + diffStr + } + if prev == 0 { + diffStr = "-" + } + + details.WriteString(fmt.Sprintf("%2d. %-25s Current: %15s Change: %10s\n", + i+1, name, formatComma(w.VoteCount), diffStr)) + } + details.WriteString("```") + + // 3. Construct Payload with Attachments + payload := map[string]interface{}{ + "text": summary.String(), + "attachments": []map[string]interface{}{ + { + "text": details.String(), + "color": "#36a64f", + }, + }, + } + + jsonPayload, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("failed to marshal slack payload: %v", err) + } + + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Post(webhookURL, "application/json", bytes.NewBuffer(jsonPayload)) + if err != nil { + return fmt.Errorf("failed to send to slack: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("slack returned status %d: %s", resp.StatusCode, string(body)) + } + + return nil +} + +func sendSlackText(webhookURL string, text string) error { + payload := map[string]interface{}{ + "text": text, + } + jsonPayload, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("failed to marshal slack payload: %v", err) + } + + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Post(webhookURL, "application/json", bytes.NewBuffer(jsonPayload)) + if err != nil { + return fmt.Errorf("failed to send to slack: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("slack returned status %d: %s", resp.StatusCode, string(body)) + } + + return nil +} + +func updateLastStatus(witnesses []Witness, lastVotes map[string]int64) []string { + var top27 []string + for i, w := range witnesses { + lastVotes[w.Address] = w.VoteCount + if i < 27 { + top27 = append(top27, w.Address) + } + } + return top27 +} + +func loadMonitorState() monitorState { + state := monitorState{ + Version: monitorStateVersion, + LastVotes: make(map[string]int64), + } + + body, err := os.ReadFile(monitorStatePath) + if err != nil { + if !os.IsNotExist(err) { + log.Printf("Failed to read monitor state: %v", err) + } + return state + } + + if err := json.Unmarshal(body, &state); err != nil { + log.Printf("Failed to parse monitor state: %v", err) + return monitorState{Version: monitorStateVersion, LastVotes: make(map[string]int64)} + } + if state.LastVotes == nil { + state.LastVotes = make(map[string]int64) + } + log.Printf("Loaded monitor state: votes=%d top27=%d updated_at=%s", len(state.LastVotes), len(state.LastTop27), state.UpdatedAt.UTC().Format(time.RFC3339)) + return state +} + +func saveMonitorState(lastVotes map[string]int64, lastTop27 []string) { + state := monitorState{ + Version: monitorStateVersion, + UpdatedAt: time.Now().UTC(), + LastVotes: lastVotes, + LastTop27: lastTop27, + } + body, err := json.MarshalIndent(state, "", " ") + if err != nil { + log.Printf("Failed to marshal monitor state: %v", err) + return + } + if err := os.MkdirAll(filepath.Dir(monitorStatePath), 0755); err != nil { + log.Printf("Failed to create monitor state directory: %v", err) + return + } + if err := os.WriteFile(monitorStatePath, body, 0644); err != nil { + log.Printf("Failed to write monitor state: %v", err) + } +} + +// The final path segment of a Slack webhook URL is the secret token; mask it before logging. +func maskWebhook(url string) string { + idx := strings.LastIndex(url, "/") + if idx < 0 || idx == len(url)-1 { + return "***" + } + return url[:idx+1] + "***" +} + +func runSRMonitor(tronNodes []string, slackWebhook string) { + log.Printf("Starting SR monitor.\nNodes: %s\nSlack Webhook: %s\n", strings.Join(tronNodes, ", "), maskWebhook(slackWebhook)) + + state := loadMonitorState() + lastVotes := state.LastVotes + lastTop27 := state.LastTop27 + + if len(lastVotes) > 0 && len(lastTop27) > 0 { + text := fmt.Sprintf("TRON SR monitor started with existing state. Previous state updated at `%s`. Next report will be sent after the next maintenance period.", state.UpdatedAt.UTC().Format(time.RFC3339)) + if err := sendSlackText(slackWebhook, text); err != nil { + log.Printf("Failed to send startup log to Slack: %v\n", err) + } else { + log.Println("Sent startup log to Slack; keeping existing monitor state until next maintenance report") + } + } else { + log.Println("No complete monitor state found. Performing initial check...") + witnesses, err := getWitnessListFromNodes(tronNodes) + if err != nil { + log.Printf("Initial check failed: %v\n", err) + } else { + log.Printf("Successfully fetched %d witnesses. Sending to Slack...\n", len(witnesses)) + if err := sendToSlack(slackWebhook, tronNodes, witnesses, lastVotes, lastTop27); err != nil { + log.Printf("Failed to send initial update to Slack: %v\n", err) + } else { + lastTop27 = updateLastStatus(witnesses, lastVotes) + saveMonitorState(lastVotes, lastTop27) + } + } + } + + log.Println("Monitoring for maintenance periods via getnextmaintenancetime...") + + for { + nextTime, err := getNextMaintenanceTimeFromNodes(tronNodes) + if err != nil { + log.Printf("Error fetching next maintenance time: %v, retrying in 1 minute...\n", err) + time.Sleep(1 * time.Minute) + continue + } + + // Calculate trigger time: next maintenance time + 1 minute + triggerTime := nextTime.Add(1 * time.Minute) + now := time.Now().UTC() + waitDuration := triggerTime.Sub(now) + + if waitDuration > 0 { + log.Printf("Next maintenance time: %s (UTC). Waiting %v until %s...\n", + nextTime.Format(time.RFC1123), waitDuration.Truncate(time.Second), triggerTime.Format(time.RFC1123)) + time.Sleep(waitDuration) + } else { + log.Printf("Maintenance time %s has already passed. Checking now...\n", nextTime.Format(time.RFC1123)) + } + + log.Printf("[%s] Maintenance period reached (+1m). Waiting for fresh node...\n", time.Now().UTC().Format(time.RFC3339)) + if err := waitForFreshNode(tronNodes); err != nil { + log.Printf("Skipping this maintenance report: %v\n", err) + time.Sleep(2 * time.Minute) + continue + } + + log.Printf("[%s] Fetching SR list...\n", time.Now().UTC().Format(time.RFC3339)) + witnesses, err := getWitnessListWithRetry(tronNodes) + if err != nil { + log.Printf("Skipping this maintenance report: %v\n", err) + } else { + if err := sendToSlack(slackWebhook, tronNodes, witnesses, lastVotes, lastTop27); err != nil { + log.Printf("Error sending to Slack: %v\n", err) + } else { + log.Println("Successfully sent SR list to Slack") + lastTop27 = updateLastStatus(witnesses, lastVotes) + saveMonitorState(lastVotes, lastTop27) + } + } + + // Wait a bit before checking for the NEXT maintenance time to avoid double-triggering + time.Sleep(2 * time.Minute) + } +} diff --git a/tools/stress_test/README.md b/tools/stress_test/README.md index 0a92b10b..a254556d 100644 --- a/tools/stress_test/README.md +++ b/tools/stress_test/README.md @@ -3,6 +3,16 @@ The stress testing tool is designed to evaluate the performance of the `java-tro It can generate a large volume of transactions, store them locally, and broadcast them to the test network. Finally, it provides a TPS (Transactions Per Second) report as the stress test result. +### ⚠️ Security Notice + +The `stress.conf` file contains placeholder values for configuration purposes only. + +**IMPORTANT:** +- Always replace the placeholder `privateKey` with your own test private key before running the tool +- Never commit your real private keys to version control +- Use test networks (Nile, Shasta) for stress testing, not mainnet with real funds +- Ensure your test private key has sufficient balance for the stress test (use DBFork tool if needed) + ### Build the stress test tool To build the stress test tool, we need to execute the following commands: ```shell script @@ -60,11 +70,11 @@ generateTx = { transferTrc20 = 30 } updateRefUrl = "127.0.0.1:50051" - // TRY18iTFy6p8yhWiCt1dhd2gz2c15ungq3 - privateKey = "aab926e86a17f0f46b4d22e61725edd5770a5b0fbdabb04b0f46ee499b1e34f2" + // IMPORTANT: Replace with your own private key + privateKey = "YOUR_PRIVATE_KEY_HERE_64_CHARACTERS_HEXADECIMAL_STRING_EXAMPLE" addressListFile = "/path/to/address-list.csv" trc10Id = 1000001 - trc20Address = "TR7NHqjeKQxGTCi8q8ZY4pL8otSzgjLj6t" + trc20Address = "TR7NHqjeKQxGTCi8q8ZY4pL8otSzgjLj6t" // Mainnet USDT - for reference only } ``` Here is the introduction for the configuration options: @@ -80,7 +90,7 @@ including `transfer`, `transferTrc10`, `transferTrc20`. The sum of all transacti `updateRefUrl`: configure the url which is used to update the `refBlockNum` and `refBlockNum` when generating the transactions; -`privateKey`: configure the private key used to sign the transactions; +`privateKey`: configure the private key used to sign the transactions. **IMPORTANT: Replace the placeholder with your own test private key. Never use keys with real funds.** `addressListFile`: configure the file path of receiver address list used to build the transactions; @@ -115,8 +125,6 @@ broadcastTx = { - `relayTx`: configure whether to broadcast the relayed transactions; -- `broadcastUrl`: configure the broadcast url list; - - `tpsLimit`: configure the maximum broadcast transactions per second; - `saveTxId`: configure whether to save the transaction id of the broadcast transactions. @@ -124,7 +132,7 @@ broadcastTx = { *Note*: we can use the [DBFork](../toolkit/DBFork.md) tool to get enough `TRX/TRC10/TRC20` balances of address corresponding to the `privateKey` for the stress test. -Then we can execute the following `generate` subcommand: +Then we can execute the following `broadcast` subcommand: ```shell # execute full command @@ -203,6 +211,9 @@ statistic = { - `startBlockNumber`: configure the start block number of range for the TPS statistic; - `endBlockNumber`: configure the end block number of range for the TPS statistic; +*Note*: if the `url` points to Lite full node, you should make sure +its configuration `openHistoryQueryWhenLiteFN = true` is set. + Then we can execute the `statistic` subcommand. ``` # execute full command diff --git a/tools/stress_test/build.gradle b/tools/stress_test/build.gradle index 894c81a8..0c8654a2 100644 --- a/tools/stress_test/build.gradle +++ b/tools/stress_test/build.gradle @@ -7,7 +7,7 @@ plugins { } sourceCompatibility = JavaVersion.VERSION_1_8 -targetCompatibility = JavaVersion.VERSION_1_8 +targetCompatibility = JavaVersion.current() [compileJava, compileTestJava]*.options*.encoding = 'UTF-8' repositories { @@ -44,18 +44,25 @@ dependencies { implementation "io.netty:netty-all:4.1.100.Final" implementation("io.github.317787106:trident:0.9.2.5") implementation group: 'com.typesafe', name: 'config', version: '1.3.2' -// implementation 'ch.qos.logback:logback-classic:1.2.6' implementation group: 'info.picocli', name: 'picocli', version: '4.6.3' - compileOnly 'org.projectlombok:lombok:1.18.12' - annotationProcessor 'org.projectlombok:lombok:1.18.12' - testCompileOnly 'org.projectlombok:lombok:1.18.12' - testAnnotationProcessor 'org.projectlombok:lombok:1.18.12' + compileOnly 'org.projectlombok:lombok:1.18.34' + annotationProcessor 'org.projectlombok:lombok:1.18.34' + testCompileOnly 'org.projectlombok:lombok:1.18.34' + testAnnotationProcessor 'org.projectlombok:lombok:1.18.34' - implementation('com.github.tronprotocol.java-tron:framework:GreatVoyage-v4.7.7') { - exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' - exclude group: "pull-parser", module: "pull-parser" - exclude group: "ch.qos.logback", module: "logback-classic" + if (rootProject.archInfo.isArm64) { + implementation('com.github.warku123.java-tron:framework:GreatVoyage-v4.8.1-arm') { + exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' + exclude group: "pull-parser", module: "pull-parser" + exclude group: "ch.qos.logback", module: "logback-classic" + } + } else { + implementation('com.github.federico2014.java-tron:framework:GreatVoyage-v4.8.1-x86_64') { + exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' + exclude group: "pull-parser", module: "pull-parser" + exclude group: "ch.qos.logback", module: "logback-classic" + } } implementation platform('com.fasterxml.jackson:jackson-bom:2.14.0') implementation(project(":toolkit")) { diff --git a/tools/stress_test/src/main/java/org/tron/BroadcastTx.java b/tools/stress_test/src/main/java/org/tron/BroadcastTx.java index 1c3381cc..c7c91bb6 100644 --- a/tools/stress_test/src/main/java/org/tron/BroadcastTx.java +++ b/tools/stress_test/src/main/java/org/tron/BroadcastTx.java @@ -14,7 +14,6 @@ import org.tron.core.config.args.Args; import org.tron.core.net.TronNetDelegate; import org.tron.core.net.TronNetService; -import org.tron.core.services.RpcApiService; import org.tron.trident.core.ApiWrapper; import org.tron.trident.core.exceptions.IllegalException; import org.tron.trident.proto.Chain.Block; @@ -104,10 +103,19 @@ public Integer call() throws IOException, InterruptedException, IllegalException } Args.getInstance().setNodeListenPort(nodeListenPort); logger.info("rpc.port: {}, node.listen.port {}", rpcPort, nodeListenPort); + Args.getInstance().setOpenHistoryQueryWhenLiteFN(true); + Args.getInstance().setRpcEnable(true); + Args.getInstance().setRpcSolidityEnable(false); + Args.getInstance().setRpcPBFTEnable(false); + Args.getInstance().setFullNodeHttpEnable(false); + Args.getInstance().setSolidityNodeHttpEnable(false); + Args.getInstance().setPBFTHttpEnable(false); + Args.getInstance().setJsonRpcHttpFullNodeEnable(false); + Args.getInstance().setJsonRpcHttpSolidityNodeEnable(false); + Args.getInstance().setJsonRpcHttpPBFTNodeEnable(false); context = new TronApplicationContext(DefaultConfig.class); app = ApplicationFactory.create(context); - app.addService(context.getBean(RpcApiService.class)); app.startup(); String url = String.format("%s:%d", "127.0.0.1", diff --git a/tools/stress_test/src/main/java/org/tron/CollectAddress.java b/tools/stress_test/src/main/java/org/tron/CollectAddress.java index 8f57d2c9..1db8019c 100644 --- a/tools/stress_test/src/main/java/org/tron/CollectAddress.java +++ b/tools/stress_test/src/main/java/org/tron/CollectAddress.java @@ -1,7 +1,5 @@ package org.tron; -import static org.tron.plugins.utils.Constant.ACCOUNT_STORE; - import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; import com.typesafe.config.Config; @@ -130,7 +128,7 @@ private Set getAddressListFromDB(String dbPath, int totalNumber) throws IOException, RocksDBException { Set addressList = new HashSet<>(); String srcDir = dbPath + File.separator + "database"; - DBInterface accountStore = DbTool.getDB(srcDir, ACCOUNT_STORE); + DBInterface accountStore = DbTool.getDB(srcDir, "account"); DBIterator iterator = accountStore.iterator(); for (iterator.seekToFirst(); iterator.valid(); iterator.next()) { addressList.add(ByteString.copyFrom(iterator.getKey())); diff --git a/tools/stress_test/src/main/java/org/tron/TpsStatistic.java b/tools/stress_test/src/main/java/org/tron/TpsStatistic.java index ec845e93..a79d52a9 100644 --- a/tools/stress_test/src/main/java/org/tron/TpsStatistic.java +++ b/tools/stress_test/src/main/java/org/tron/TpsStatistic.java @@ -84,6 +84,8 @@ public Integer call() throws Exception { Statistic.setApiWrapper( new ApiWrapper(config.getStatisticUrl(), config.getStatisticUrl(), config.getPrivateKey())); + Statistic.setBroadcastTpsLimit(config.getBroadcastTpsLimit()); + Statistic.setTotalGenerateTxCnt(config.getTotalTxCnt()); Statistic.result(startBlock, endBlock, output); return 0; } diff --git a/tools/stress_test/src/main/java/org/tron/trxs/BroadcastGenerate.java b/tools/stress_test/src/main/java/org/tron/trxs/BroadcastGenerate.java index 97eaf9cb..b2dc9376 100644 --- a/tools/stress_test/src/main/java/org/tron/trxs/BroadcastGenerate.java +++ b/tools/stress_test/src/main/java/org/tron/trxs/BroadcastGenerate.java @@ -93,7 +93,20 @@ public void broadcastTransactions() throws IOException, InterruptedException { long endTps; float currentTps; while ((transaction = Transaction.parseDelimitedFrom(fis)) != null) { + TransactionMessage message = new TransactionMessage(transaction); + int peerCnt = tronNetService.fastBroadcastTransaction(message); + while (peerCnt <= 0) { + logger.warn("broadcast task {}/{} has no available peers to broadcast, please wait", + index + 1, totalTask); + Thread.sleep(100); + peerCnt = tronNetService.fastBroadcastTransaction(message); + } + trxCount++; + if (trxCount % 10000 == 0) { + logger.info("total broadcast tx num: {}", trxCount); + } + if (cnt > TxConfig.getInstance().getBroadcastTpsLimit()) { endTps = System.currentTimeMillis(); if (endTps - startTps <= 1000) { @@ -107,19 +120,12 @@ public void broadcastTransactions() throws IOException, InterruptedException { cnt = 0; startTps = System.currentTimeMillis(); } else { - TransactionMessage message = new TransactionMessage(transaction); - int peerCnt = tronNetService.fastBroadcastTransaction(message); - while (peerCnt <= 0) { - logger.warn("broadcast task {}/{} has no available peers to broadcast, please wait", - index + 1, totalTask); - Thread.sleep(100); - peerCnt = tronNetService.fastBroadcastTransaction(message); - } - if (saveTrxId) { - transactionIDs.add(transaction); - } cnt++; } + + if (saveTrxId) { + transactionIDs.add(transaction); + } } isFinishSend = true; diff --git a/tools/stress_test/src/main/java/org/tron/trxs/BroadcastRelay.java b/tools/stress_test/src/main/java/org/tron/trxs/BroadcastRelay.java index ea756355..f6520f19 100644 --- a/tools/stress_test/src/main/java/org/tron/trxs/BroadcastRelay.java +++ b/tools/stress_test/src/main/java/org/tron/trxs/BroadcastRelay.java @@ -83,7 +83,18 @@ public void broadcastTransactions() { long startTps = System.currentTimeMillis(); long endTps; while ((transaction = Transaction.parseDelimitedFrom(fis)) != null) { + TransactionMessage message = new TransactionMessage(transaction); + int peerCnt = tronNetService.fastBroadcastTransaction(message); + while (peerCnt <= 0) { + logger.warn("broadcast relay task has no available peers to broadcast, please wait"); + Thread.sleep(100); + peerCnt = tronNetService.fastBroadcastTransaction(message); + } + trxCount++; + if (trxCount % 10000 == 0) { + logger.info("total broadcast tx num: {}", trxCount); + } if (cnt > TxConfig.getInstance().getBroadcastTpsLimit()) { endTps = System.currentTimeMillis(); if (endTps - startTps < 1000) { @@ -92,25 +103,12 @@ public void broadcastTransactions() { cnt = 0; startTps = System.currentTimeMillis(); } else { - try { - TransactionMessage message = new TransactionMessage(transaction); - int peerCnt = tronNetService.fastBroadcastTransaction(message); - while (peerCnt <= 0) { - logger.warn("broadcast relay task has no available peers to broadcast, please wait"); - Thread.sleep(100); - peerCnt = tronNetService.fastBroadcastTransaction(message); - } - if (trxCount % 1000 == 0) { - logger.info("total broadcast tx num: {}", trxCount); - } - } catch (Exception e) { - e.printStackTrace(); - } - if (saveTrxId) { - transactionIDs.add(transaction); - } cnt++; } + + if (saveTrxId) { + transactionIDs.add(transaction); + } } isFinishSend = true; diff --git a/tools/stress_test/src/main/java/org/tron/utils/Statistic.java b/tools/stress_test/src/main/java/org/tron/utils/Statistic.java index 892f1565..b855c6e1 100644 --- a/tools/stress_test/src/main/java/org/tron/utils/Statistic.java +++ b/tools/stress_test/src/main/java/org/tron/utils/Statistic.java @@ -53,6 +53,8 @@ public static void result(long startBlock, long endBlock, String output) throws int minTrxCntInOneBlock = 1000000; int trxCnt = 0; int totalTrxCnt = 0; + int minBlockNumber = 0; + int maxBlockNumber = 0; for (long i = startNumber; i < endNumber; i++) { block = apiWrapper.getBlockByNum(i); @@ -61,9 +63,11 @@ public static void result(long startBlock, long endBlock, String output) throws if (trxCnt > maxTrxCntInOneBlock) { maxTrxCntInOneBlock = trxCnt; + maxBlockNumber = (int) i; } if (trxCnt < minTrxCntInOneBlock) { minTrxCntInOneBlock = trxCnt; + minBlockNumber = (int) i; } } @@ -83,8 +87,8 @@ public static void result(long startBlock, long endBlock, String output) throws logger.info("total generate tx count: {}, total broadcast tx count: {}, tx on chain rate: {}", totalGenerateTxCnt, totalTrxCnt, 1.0 * totalTrxCnt / totalGenerateTxCnt); logger.info("cost time: {} minutes", 1.0 * actualTime / (60 * 1000)); - logger.info("max block size: {}", maxTrxCntInOneBlock); - logger.info("min block size: {}", minTrxCntInOneBlock); + logger.info("max block size: {} in block: {}", maxTrxCntInOneBlock, maxBlockNumber); + logger.info("min block size: {} in block: {}", minTrxCntInOneBlock, minBlockNumber); logger.info("tps: {}", tps); logger.info("miss block rate: {}", missBlockRate); @@ -102,13 +106,16 @@ public static void result(long startBlock, long endBlock, String output) throws writer.newLine(); writer.write(String.format("cost time: %f minutes", 1.0 * actualTime / (60 * 1000))); writer.newLine(); - writer.write(String.format("max block size: %d", maxTrxCntInOneBlock)); + writer.write( + String.format("max block size: %d in block: %d", maxTrxCntInOneBlock, maxBlockNumber)); writer.newLine(); - writer.write(String.format("min block size: %d", minTrxCntInOneBlock)); + writer.write( + String.format("min block size: %d in block: %d", minTrxCntInOneBlock, minBlockNumber)); writer.newLine(); writer.write(String.format("tps: %f", tps)); writer.newLine(); writer.write(String.format("miss block rate: %f", missBlockRate)); + writer.newLine(); } catch (IOException e) { e.printStackTrace(); } diff --git a/tools/stress_test/src/main/resources/stress.conf b/tools/stress_test/src/main/resources/stress.conf index e7c55d00..29f32394 100644 --- a/tools/stress_test/src/main/resources/stress.conf +++ b/tools/stress_test/src/main/resources/stress.conf @@ -16,11 +16,13 @@ generateTx = { transferTrc20 = 30 } updateRefUrl = "127.0.0.1:50051" - // TRY18iTFy6p8yhWiCt1dhd2gz2c15ungq3 - privateKey = "aab926e86a17f0f46b4d22e61725edd5770a5b0fbdabb04b0f46ee499b1e34f2" + // IMPORTANT: Replace with your own private key for testing + // Example format: 64-character hexadecimal string + // WARNING: Never use this key on mainnet or with real funds + privateKey = "YOUR_PRIVATE_KEY_HERE_64_CHARACTERS_HEXADECIMAL_STRING_EXAMPLE" addressListFile = "/path/to/address-list.csv" trc10Id = 1000001 - trc20Address = "TR7NHqjeKQxGTCi8q8ZY4pL8otSzgjLj6t" + trc20Address = "TR7NHqjeKQxGTCi8q8ZY4pL8otSzgjLj6t" // Mainnet USDT - for reference only } relayTx = { diff --git a/tools/toolkit/DBFork.md b/tools/toolkit/DBFork.md index d8e1fbb2..3fe3f06e 100644 --- a/tools/toolkit/DBFork.md +++ b/tools/toolkit/DBFork.md @@ -107,9 +107,15 @@ trc20Contracts = [ } ] -latestBlockHeaderTimestamp = 1735628883000 -maintenanceTimeInterval = 21600000 -nextMaintenanceTime = 1735628894000 +chainParameters = { + latestBlockHeaderTimestamp = 1747986162000 + maintenanceTimeInterval = 21600000 + nextMaintenanceTime = 1747996162000 + // accountUpgradeCost = 9999000000 + // createAccountFee = 100000 + // transactionFee = 1000 + // ...... +} ``` For the `witnesses`, we can configure the following properties: @@ -134,15 +140,20 @@ For the `trc20Contracts`, we can configure the following properties: - `address`: set the account address - `balance`: set the TRC20 balance + *Note*: the `balancesSlotPosition` sets slot position for `mapping(address account => uint256) private _balances` variable in the TRC20 contract. For most standard TRC20 contracts, the `balancesSlotPosition` is 0. For some special cases, you may need to change the `balancesSlotPosition` value. For more details about the variable slot position in the contract, please refer [layout_in_storage](https://docs.soliditylang.org/en/latest/internals/layout_in_storage.html). -Set `latestBlockHeaderTimestamp` as the current time in milliseconds. This timestamp is used before processing new blocks - setting it to the current time reduces debug log output and accelerates the production of the first new block. This setting does not impact the underlying block production logic. +For the `chainParameters`, we can configure the following parameters: + +- `latestBlockHeaderTimestamp`: set the current time in milliseconds. This timestamp is used before processing new blocks - setting it to the current time reduces debug log output and accelerates the production of the first new block. This setting does not impact the underlying block production logic. + +- `maintenanceTimeInterval` and `nextMaintenanceTime`: set optionally to facilitate testing. -Set `maintenanceTimeInterval` and `nextMaintenanceTime` optionally to facilitate testing. +- For more proposal parameters modification, please refer [fork.conf](./src/main/resources/fork.conf). Execute the fork command: ```shell script diff --git a/tools/toolkit/README.md b/tools/toolkit/README.md index 082fb589..ef76f53f 100644 --- a/tools/toolkit/README.md +++ b/tools/toolkit/README.md @@ -23,11 +23,11 @@ expenses. - `convert`: Covert leveldb to rocksdb. - `lite`: Split lite data for java-tron. - `cp, copy`: Quick copy leveldb or rocksdb data. -- `root`: compute merkle root for tiny db. NOTE: large db may GC overhead -limit exceeded. +- `root`: compute merkle root for tiny db. NOTE: large db may GC overhead limit exceeded. - `fork`: Modify the database of java-tron for shadow fork testing. +- `query`: Query the latest vote and reward information from the database. -## DB Archive +## DB Archive(Requires x86 + LevelDB) DB archive provides the ability to reformat the manifest according to the current `database`, parameters are compatible with the previous `ArchiveManifest`. @@ -50,8 +50,7 @@ DB archive provides the ability to reformat the manifest according to the curren java -jar Toolkit.jar db archive -m 128 #4. specify optimization only when Manifest exceeds 128M ``` - -## DB Convert +## DB Convert(Requires x86 + LevelDB) DB convert provides a helper which can convert LevelDB data to RocksDB data, parameters are compatible with previous `DBConvert`. @@ -59,15 +58,13 @@ DB convert provides a helper which can convert LevelDB data to RocksDB data, par - ``: Input path for leveldb, default: output-directory/database. - ``: Output path for rocksdb, default: output-directory-dst/database. -- `--safe`: In safe mode, read data from leveldb then put into rocksdb, it's a very time-consuming procedure. If not, just change engine.properties from leveldb to rocksdb, rocksdb - is compatible with leveldb for the current version. This may not be the case in the future, default: false. - `-h | --help`: Provide the help info. ### Examples: ```shell script # full command - java -jar Toolkit.jar db convert [-h] [--safe] + java -jar Toolkit.jar db convert [-h] # examples java -jar Toolkit.jar db convert output-directory/database /tmp/database ``` @@ -91,7 +88,7 @@ DB copy provides a helper which can copy LevelDB or RocksDB data quickly on the java -jar Toolkit.jar db cp output-directory/database /tmp/databse ``` -## DB Lite +## DB Lite(LevelDB unavailable on ARM) DB lite provides lite database, parameters are compatible with previous `LiteFullNodeTool`. @@ -159,7 +156,7 @@ Execute move command. java -jar Toolkit.jar db mv -c main_net_config.conf -d /data/tron/output-directory ``` -## DB Root +## DB Root(LevelDB unavailable on ARM) DB root provides a helper which can compute merkle root for tiny db. @@ -171,7 +168,6 @@ NOTE: large db may GC overhead limit exceeded. - `--db`: db name. - `-h | --help`: provide the help info - ## DB Fork DB fork tool can help launch a private java-tron FullNode or network based on the state of public chain database to support shadow fork testing. @@ -210,11 +206,26 @@ reward = [ "TLyqzVGLV1srkB7dToTAEqgDSfPtXRJZYH", "TJvaAeFb8Lykt9RQcVyyTFN2iDvGMuyD4M" ] + +voters = { + allWitnesses = false + allActiveWitnesses = true + witnessList = [ + "TQopP5GM68QoqLzpz8YReDfSoCMkvwcZYd", + "TDpt9adA6QidL1B1sy3D8NC717C6L5JxFo", + ] + threshold = 10000 +} ``` - `vote.allWitnesses`: configure whether to query the vote information of all witnesses. - `vote.witnessList`: configure to query the vote information from specified witness list. The option is valid only when `vote.allWitnesses = false`. - `reward`: configure the address list you need to query the reward. +- `voters.allWitnesses`: configure whether to query the voter information of all witnesses. +- `voters.allActiveWitnesses`: configure whether to query the voter information of all active + witnesses. +- `voters.witnessList`: configure to query the voter information from specified witness list. +- `voters.threshold`: configure the votes count threshold to filter the voters. Execute query command. ```shell script diff --git a/tools/toolkit/build.gradle b/tools/toolkit/build.gradle index 450a14ca..4ba8b768 100644 --- a/tools/toolkit/build.gradle +++ b/tools/toolkit/build.gradle @@ -4,10 +4,13 @@ plugins { id 'application' id 'checkstyle' id 'com.github.johnrengelman.shadow' version '7.1.2' + id "org.sonarqube" version "2.6" } +def springVersion = "5.3.39" + sourceCompatibility = JavaVersion.VERSION_1_8 -targetCompatibility = JavaVersion.VERSION_1_8 +targetCompatibility = JavaVersion.current() [compileJava, compileTestJava]*.options*.encoding = 'UTF-8' repositories { @@ -17,6 +20,13 @@ repositories { maven { url 'https://jitpack.io' } } +// Force RocksDB version based on architecture (same as java-tron/platform module) +configurations.all { + resolutionStrategy { + force "org.rocksdb:rocksdbjni:${rootProject.archInfo.requires.RocksdbVersion}" + } +} + tasks.named('jar') { enabled = false dependsOn shadowJar @@ -29,35 +39,67 @@ shadowJar { mergeServiceFiles() } +sourceSets { + main { + java.srcDirs = rootProject.archInfo.sourceSets.main.java.srcDirs + } + test { + java.srcDirs = rootProject.archInfo.sourceSets.test.java.srcDirs + } +} + + dependencies { testImplementation 'junit:junit:4.13.2' - testImplementation('com.github.tronprotocol.java-tron:framework:GreatVoyage-v4.7.7') { - exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' - exclude group: "pull-parser", module: "pull-parser" - } - testImplementation platform('com.fasterxml.jackson:jackson-bom:2.14.0') + implementation group: 'org.springframework', name: 'spring-context', version: "${springVersion}" implementation group: 'info.picocli', name: 'picocli', version: '4.6.3' implementation group: 'com.typesafe', name: 'config', version: '1.3.2' implementation group: 'me.tongfei', name: 'progressbar', version: '0.9.3' - implementation group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.69' - implementation group: 'org.rocksdb', name: 'rocksdbjni', version: '5.15.10' - implementation 'io.github.tronprotocol:leveldbjni-all:1.18.2' - implementation 'io.github.tronprotocol:leveldb:1.18.2' - - implementation('com.github.tronprotocol.java-tron:chainbase:GreatVoyage-v4.7.7') { - exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' - exclude group: "pull-parser", module: "pull-parser" - } - implementation platform('com.fasterxml.jackson:jackson-bom:2.14.0') - implementation group: 'info.picocli', name: 'picocli', version: '4.6.3' + implementation group: 'org.bouncycastle', name: 'bcprov-jdk18on', version: '1.79' - compileOnly 'org.projectlombok:lombok:1.18.12' - annotationProcessor 'org.projectlombok:lombok:1.18.12' - testCompileOnly 'org.projectlombok:lombok:1.18.12' - testAnnotationProcessor 'org.projectlombok:lombok:1.18.12' + compileOnly 'org.projectlombok:lombok:1.18.34' + annotationProcessor 'org.projectlombok:lombok:1.18.34' + testCompileOnly 'org.projectlombok:lombok:1.18.34' + testAnnotationProcessor 'org.projectlombok:lombok:1.18.34' implementation 'com.alibaba:fastjson:1.2.83' + implementation group: 'org.slf4j', name: 'slf4j-api', version: '1.7.25' + implementation group: 'org.slf4j', name: 'jcl-over-slf4j', version: '1.7.25' + implementation "org.apache.commons:commons-lang3:3.4" + implementation "org.apache.commons:commons-collections4:4.1" + implementation platform('com.fasterxml.jackson:jackson-bom:2.14.0') + + if (rootProject.archInfo.isArm64) { + // ARM64 + testImplementation('com.github.warku123.java-tron:framework:GreatVoyage-v4.8.1-arm') { + exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' + exclude group: "pull-parser", module: "pull-parser" + } + implementation('com.github.warku123.java-tron:chainbase:GreatVoyage-v4.8.1-arm') { + exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' + exclude group: "pull-parser", module: "pull-parser" + } + implementation 'com.github.warku123.java-tron:platform:GreatVoyage-v4.8.1-arm' + } else { + // x86 + testImplementation('com.github.federico2014.java-tron:framework:GreatVoyage-v4.8.1-x86_64') { + exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' + exclude group: "pull-parser", module: "pull-parser" + } + implementation('com.github.federico2014.java-tron:chainbase:GreatVoyage-v4.8.1-x86_64') { + exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' + exclude group: "pull-parser", module: "pull-parser" + } + implementation('com.github.federico2014.java-tron:platform:GreatVoyage-v4.8.1-x86_64') { + exclude(group: 'org.fusesource.leveldbjni', module: 'leveldbjni-all') + exclude(group: 'io.github.tronprotocol', module: 'zksnark-java-sdk') + exclude(group: 'commons-io', module: 'commons-io') + } + implementation 'io.github.tronprotocol:leveldbjni-all:1.18.2' + implementation 'io.github.tronprotocol:leveldb:1.18.2' + } + } application { @@ -95,3 +137,20 @@ task checkstyleToolkitTest(type: Checkstyle) { } check.dependsOn checkstyleToolkitMain, checkstyleToolkitTest + +test { + testLogging { + exceptionFormat = 'full' + } + + if (rootProject.archInfo.isArm64) { + exclude 'org/tron/plugins/leveldb/**' + filter { + excludeTestsMatching '*.*leveldb*' + excludeTestsMatching '*.*Leveldb*' + excludeTestsMatching '*.*LevelDB*' + excludeTestsMatching '*.*LevelDb*' + excludeTestsMatching '*.*Archive*' + } + } +} diff --git a/tools/toolkit/src/main/java/arm/org/tron/plugins/ArchiveManifest.java b/tools/toolkit/src/main/java/arm/org/tron/plugins/ArchiveManifest.java new file mode 100644 index 00000000..b7848cf4 --- /dev/null +++ b/tools/toolkit/src/main/java/arm/org/tron/plugins/ArchiveManifest.java @@ -0,0 +1,29 @@ +package org.tron.plugins; + +import lombok.extern.slf4j.Slf4j; +import org.tron.common.arch.Arch; + +/** + * ARM architecture only supports RocksDB, + * which does not require manifest rebuilding (manifest rebuilding is a LevelDB-only feature). + * This command is not supported but retained for compatibility. + **/ +@Slf4j(topic = "archive") +public class ArchiveManifest { + + public static void main(String[] args) { + int exitCode = run(args); + System.exit(exitCode); + } + + public static int run(String[] args) { + String tips = String.format( + "%s architecture only supports RocksDB, which does not require manifest rebuilding " + + "(manifest rebuilding is a LevelDB-only feature).", + Arch.getOsArch()); + System.out.println(tips); + logger.warn(tips); + return 0; + } + +} diff --git a/tools/toolkit/src/main/java/arm/org/tron/plugins/DbArchive.java b/tools/toolkit/src/main/java/arm/org/tron/plugins/DbArchive.java new file mode 100644 index 00000000..03c52f33 --- /dev/null +++ b/tools/toolkit/src/main/java/arm/org/tron/plugins/DbArchive.java @@ -0,0 +1,50 @@ +package org.tron.plugins; + +import java.util.concurrent.Callable; +import lombok.extern.slf4j.Slf4j; +import org.tron.common.arch.Arch; +import picocli.CommandLine; +import picocli.CommandLine.Option; + +/** + * ARM architecture only supports RocksDB, + * which does not require manifest rebuilding (manifest rebuilding is a LevelDB-only feature). + * This command is not supported but retained for compatibility. + **/ +@Slf4j(topic = "archive") +@CommandLine.Command(name = "archive", description = "A helper to rewrite leveldb manifest.") +public class DbArchive implements Callable { + + @CommandLine.Spec + CommandLine.Model.CommandSpec spec; + @Option(names = {"-d", "--database-directory"}, + defaultValue = "output-directory/database", + description = "java-tron database directory. Default: ${DEFAULT-VALUE}") + private String databaseDirectory; + + @Option(names = {"-b", "--batch-size"}, + defaultValue = "80000", + description = "deal manifest batch size. Default: ${DEFAULT-VALUE}") + private int maxBatchSize; + + @Option(names = {"-m", "--manifest-size"}, + defaultValue = "0", + description = "manifest min size(M) to archive. Default: ${DEFAULT-VALUE}") + private int maxManifestSize; + + @Option(names = {"-h", "--help"}) + private boolean help; + + + @Override + public Integer call() throws Exception { + String tips = String.format( + "%s architecture only supports RocksDB, which does not require manifest rebuilding " + + "(manifest rebuilding is a LevelDB-only feature).", + Arch.getOsArch()); + spec.commandLine().getErr().println(spec.commandLine().getColorScheme().errorText(tips)); + logger.warn(tips); + return 0; + } + +} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/Db.java b/tools/toolkit/src/main/java/common/org/tron/plugins/Db.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/Db.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/Db.java diff --git a/tools/toolkit/src/main/java/org/tron/plugins/DbConvert.java b/tools/toolkit/src/main/java/common/org/tron/plugins/DbConvert.java similarity index 89% rename from tools/toolkit/src/main/java/org/tron/plugins/DbConvert.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/DbConvert.java index a75b235b..bcf6e1e7 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/DbConvert.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/DbConvert.java @@ -20,6 +20,7 @@ import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; import org.rocksdb.Status; +import org.tron.common.arch.Arch; import org.tron.plugins.utils.DBUtils; import org.tron.plugins.utils.FileUtils; import picocli.CommandLine; @@ -49,20 +50,19 @@ public class DbConvert implements Callable { description = "Output path for rocksdb. Default: ${DEFAULT-VALUE}") private File dest; - @CommandLine.Option(names = {"--safe"}, - description = "In safe mode, read data from leveldb then put rocksdb." - + "If not, just change engine.properties from leveldb to rocksdb," - + "rocksdb is compatible with leveldb for current version." - + "This may not be the case in the future." - + "Default: ${DEFAULT-VALUE}") - private boolean safe; - @CommandLine.Option(names = {"-h", "--help"}) private boolean help; @Override public Integer call() throws Exception { + if (Arch.isArm64()) { + String tips = String.format("This command is not supported on %s architecture.", + Arch.getOsArch()); + spec.commandLine().getErr().println(spec.commandLine().getColorScheme().errorText(tips)); + logger.error(tips); + return 0; + } if (help) { spec.commandLine().usage(System.out); return 0; @@ -95,12 +95,12 @@ public Integer call() throws Exception { final long time = System.currentTimeMillis(); List services = new ArrayList<>(); files.forEach(f -> services.add( - new DbConverter(src.getPath(), dest.getPath(), f.getName(), safe))); + new DbConverter(src.getPath(), dest.getPath(), f.getName()))); cpList.forEach(f -> services.add( new DbConverter( Paths.get(src.getPath(), DBUtils.CHECKPOINT_DB_V2).toString(), Paths.get(dest.getPath(), DBUtils.CHECKPOINT_DB_V2).toString(), - f.getName(), safe))); + f.getName()))); List fails = ProgressBar.wrap(services.stream(), "convert task").parallel().map( dbConverter -> { try { @@ -140,15 +140,12 @@ static class DbConverter implements Converter { private long srcDbValueSum = 0L; private long dstDbValueSum = 0L; - private boolean safe; - - public DbConverter(String srcDir, String dstDir, String name, boolean safe) { + public DbConverter(String srcDir, String dstDir, String name) { this.srcDir = srcDir; this.dstDir = dstDir; this.dbName = name; this.srcDbPath = Paths.get(this.srcDir, name); this.dstDbPath = Paths.get(this.dstDir, name); - this.safe = safe; } @Override @@ -178,23 +175,15 @@ public boolean doConvert() throws Exception { FileUtils.createDirIfNotExists(dstDir); logger.info("Convert database {} start", this.dbName); - if (safe) { - convertLevelToRocks(); - compact(); - } else { - FileUtils.copyDir(Paths.get(srcDir), Paths.get(dstDir), dbName); - } + convertLevelToRocks(); + compact(); + boolean result = check() && createEngine(dstDbPath.toString()); long etime = System.currentTimeMillis(); if (result) { - if (safe) { - logger.info("Convert database {} successful end with {} key-value {} minutes", - this.dbName, this.srcDbKeyCount, (etime - startTime) / 1000.0 / 60); - } else { - logger.info("Convert database {} successful end {} minutes", - this.dbName, (etime - startTime) / 1000.0 / 60); - } + logger.info("Convert database {} successful end with {} key-value {} minutes", + this.dbName, this.srcDbKeyCount, (etime - startTime) / 1000.0 / 60); } else { logger.info("Convert database {} failure", this.dbName); @@ -234,8 +223,8 @@ private void batchInsert(RocksDB rocks, List keys, List values) * @throws Exception RocksDBException */ private void write(RocksDB rocks, org.rocksdb.WriteBatch batch) throws Exception { - try { - rocks.write(new org.rocksdb.WriteOptions(), batch); + try (org.rocksdb.WriteOptions writeOptions = new org.rocksdb.WriteOptions()) { + rocks.write(writeOptions, batch); } catch (RocksDBException e) { // retry if (maybeRetry(e)) { @@ -270,7 +259,8 @@ public void convertLevelToRocks() throws Exception { JniDBFactory.pushMemoryPool(1024 * 1024); try ( DB level = DBUtils.newLevelDb(srcDbPath); - RocksDB rocks = DBUtils.newRocksDbForBulkLoad(dstDbPath); + org.rocksdb.Options options = DBUtils.newDefaultRocksDbOptions(true, dbName); + RocksDB rocks = RocksDB.open(options, this.dstDbPath.toString()); DBIterator levelIterator = level.iterator( new org.iq80.leveldb.ReadOptions().fillCache(false))) { @@ -302,7 +292,8 @@ private void compact() throws RocksDBException { if (DBUtils.MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(this.dbName)) { return; } - try (RocksDB rocks = DBUtils.newRocksDb(this.dstDbPath)) { + try (org.rocksdb.Options options = DBUtils.newDefaultRocksDbOptions(false, dbName); + RocksDB rocks = RocksDB.open(options, this.dstDbPath.toString())) { logger.info("compact database {} start", this.dbName); rocks.compactRange(); logger.info("compact database {} end", this.dbName); @@ -310,11 +301,9 @@ private void compact() throws RocksDBException { } private boolean check() throws RocksDBException { - if (!safe) { - return true; - } try ( - RocksDB rocks = DBUtils.newRocksDbReadOnly(this.dstDbPath); + org.rocksdb.Options options = DBUtils.newDefaultRocksDbOptions(false, dbName); + RocksDB rocks = RocksDB.openReadOnly(options, this.dstDbPath.toString()); org.rocksdb.ReadOptions r = new org.rocksdb.ReadOptions().setFillCache(false); RocksIterator rocksIterator = rocks.newIterator(r)) { diff --git a/tools/toolkit/src/main/java/org/tron/plugins/DbCopy.java b/tools/toolkit/src/main/java/common/org/tron/plugins/DbCopy.java similarity index 98% rename from tools/toolkit/src/main/java/org/tron/plugins/DbCopy.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/DbCopy.java index 5a4e5969..7558d764 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/DbCopy.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/DbCopy.java @@ -22,7 +22,7 @@ exitCodeListHeading = "Exit Codes:%n", exitCodeList = { "0:Successful", - "n:Internal error: exception occurred, please check toolkit.log"}) + "n:Internal error: exception occurred,please check toolkit.log"}) public class DbCopy implements Callable { diff --git a/tools/toolkit/src/main/java/common/org/tron/plugins/DbFork.java b/tools/toolkit/src/main/java/common/org/tron/plugins/DbFork.java new file mode 100644 index 00000000..27e13705 --- /dev/null +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/DbFork.java @@ -0,0 +1,1256 @@ +package org.tron.plugins; + +import static java.lang.System.arraycopy; +import static org.tron.core.Constant.CREATE_ACCOUNT_TRANSACTION_MAX_BYTE_SIZE; +import static org.tron.core.Constant.CREATE_ACCOUNT_TRANSACTION_MIN_BYTE_SIZE; +import static org.tron.core.Constant.DYNAMIC_ENERGY_INCREASE_FACTOR_RANGE; +import static org.tron.core.config.Parameter.ChainConstant.ONE_YEAR_BLOCK_NUMBERS; +import static org.tron.plugins.utils.ChainParameters.CHAIN_PARAMETERS; +import static org.tron.plugins.utils.Constant.ACCOUNTS_KEY; +import static org.tron.plugins.utils.Constant.ACCOUNT_ADDRESS; +import static org.tron.plugins.utils.Constant.ACCOUNT_ASSET; +import static org.tron.plugins.utils.Constant.ACCOUNT_BALANCE; +import static org.tron.plugins.utils.Constant.ACCOUNT_NAME; +import static org.tron.plugins.utils.Constant.ACCOUNT_OWNER; +import static org.tron.plugins.utils.Constant.ACCOUNT_STORE; +import static org.tron.plugins.utils.Constant.ACCOUNT_TRC10_BALANCE; +import static org.tron.plugins.utils.Constant.ACCOUNT_TRC10_ID; +import static org.tron.plugins.utils.Constant.ACCOUNT_TYPE; +import static org.tron.plugins.utils.Constant.ACTIVE_WITNESSES; +import static org.tron.plugins.utils.Constant.ASSET_ISSUE_V2; +import static org.tron.plugins.utils.Constant.CONTRACT_STORE; +import static org.tron.plugins.utils.Constant.DYNAMIC_PROPERTY_STORE; +import static org.tron.plugins.utils.Constant.LATEST_BLOCK_HEADER_TIMESTAMP; +import static org.tron.plugins.utils.Constant.MAINTENANCE_TIME; +import static org.tron.plugins.utils.Constant.MAX_ACTIVE_WITNESS_NUM; +import static org.tron.plugins.utils.Constant.STORAGE_ROW_STORE; +import static org.tron.plugins.utils.Constant.TRC20_ACCOUNT; +import static org.tron.plugins.utils.Constant.TRC20_BALANCE; +import static org.tron.plugins.utils.Constant.TRC20_BALANCES_POSITION; +import static org.tron.plugins.utils.Constant.TRC20_CONTRACTS_KEY; +import static org.tron.plugins.utils.Constant.TRC20_CONTRACT_ADDRESS; +import static org.tron.plugins.utils.Constant.WITNESS_ADDRESS; +import static org.tron.plugins.utils.Constant.WITNESS_KEY; +import static org.tron.plugins.utils.Constant.WITNESS_SCHEDULE_STORE; +import static org.tron.plugins.utils.Constant.WITNESS_STORE; +import static org.tron.plugins.utils.Constant.WITNESS_URL; +import static org.tron.plugins.utils.Constant.WITNESS_VOTE; + +import com.google.common.primitives.Bytes; +import com.google.common.primitives.Longs; +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import java.io.File; +import java.io.IOException; +import java.math.BigInteger; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.Callable; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.ArrayUtils; +import org.rocksdb.RocksDBException; +import org.tron.common.crypto.Hash; +import org.tron.common.utils.ByteArray; +import org.tron.common.utils.ByteUtil; +import org.tron.common.utils.Commons; +import org.tron.core.capsule.AccountCapsule; +import org.tron.core.capsule.StorageRowCapsule; +import org.tron.core.capsule.WitnessCapsule; +import org.tron.core.exception.ContractValidateException; +import org.tron.plugins.utils.ChainParameters; +import org.tron.plugins.utils.Constant; +import org.tron.plugins.utils.FileUtils; +import org.tron.plugins.utils.db.DBInterface; +import org.tron.plugins.utils.db.DBIterator; +import org.tron.plugins.utils.db.DbTool; +import org.tron.protos.Protocol.Account; +import org.tron.protos.Protocol.AccountType; +import org.tron.protos.Protocol.Key; +import org.tron.protos.Protocol.Permission; +import org.tron.protos.Protocol.Permission.PermissionType; +import org.tron.protos.contract.SmartContractOuterClass.SmartContract; +import picocli.CommandLine; +import picocli.CommandLine.Command; + +@Slf4j(topic = "fork") +@Command(name = "fork", + description = "Modify the database of java-tron for shadow fork testing.", + exitCodeListHeading = "Exit Codes:%n", + exitCodeList = { + "0:Successful", + "n:Internal error: exception occurred, please check logs/toolkit.log"}) +public class DbFork implements Callable { + + private DBInterface witnessStore; + private DBInterface witnessScheduleStore; + private DBInterface accountStore; + private DBInterface dynamicPropertiesStore; + private DBInterface accountAssetStore; + private DBInterface assetIssueV2Store; + private DBInterface contractStore; + private DBInterface storageRowStore; + + @CommandLine.Spec + CommandLine.Model.CommandSpec spec; + + @CommandLine.Option(names = {"-d", "--database-directory"}, + defaultValue = "output-directory", + description = "java-tron database directory path. Default: ${DEFAULT-VALUE}") + private String database; + + @CommandLine.Option(names = {"-c", "--config"}, + defaultValue = "fork.conf", + description = "config the new witnesses, balances, etc for shadow fork." + + " Default: ${DEFAULT-VALUE}") + private String config; + + @CommandLine.Option(names = {"-r", "--retain-witnesses"}, + description = "retain the previous witnesses and active witnesses.") + private boolean retain; + + @CommandLine.Option(names = {"-h", "--help"}) + private boolean help; + + private void initStore() throws IOException, RocksDBException { + String srcDir = database + File.separator + "database"; + witnessStore = DbTool.getDB(srcDir, WITNESS_STORE); + witnessScheduleStore = DbTool.getDB(srcDir, WITNESS_SCHEDULE_STORE); + accountStore = DbTool.getDB(srcDir, ACCOUNT_STORE); + dynamicPropertiesStore = DbTool.getDB(srcDir, DYNAMIC_PROPERTY_STORE); + accountAssetStore = DbTool.getDB(srcDir, ACCOUNT_ASSET); + assetIssueV2Store = DbTool.getDB(srcDir, ASSET_ISSUE_V2); + contractStore = DbTool.getDB(srcDir, CONTRACT_STORE); + storageRowStore = DbTool.getDB(srcDir, STORAGE_ROW_STORE); + } + + private AccountCapsule getOrCreateAccountCapsule(byte[] address) { + byte[] value = accountStore.get(address); + Account account = null; + try { + account = ArrayUtils.isEmpty(value) ? null : Account.parseFrom(value); + } catch (InvalidProtocolBufferException e) { + e.printStackTrace(); + System.exit(-1); + } + + if (Objects.isNull(account)) { + account = Account.newBuilder().setAddress(ByteString.copyFrom(address)).build(); + } + return new AccountCapsule(account); + } + + private boolean isMultiSignEnabled() { + byte[] allowMultiSign = dynamicPropertiesStore.get(Constant.ALLOW_MULTI_SIGN); + return allowMultiSign != null && ByteArray.toLong(allowMultiSign) == 1L; + } + + private Permission createDefaultActivePermission(ByteString address) { + byte[] operations = dynamicPropertiesStore.get(Constant.ACTIVE_DEFAULT_OPERATIONS); + if (operations == null) { + throw new IllegalStateException( + "ACTIVE_DEFAULT_OPERATIONS not found while ALLOW_MULTI_SIGN is enabled"); + } + + Key.Builder key = Key.newBuilder(); + key.setAddress(address); + key.setWeight(1); + + return Permission.newBuilder() + .setType(PermissionType.Active) + .setId(2) + .setPermissionName("active") + .setThreshold(1) + .setParentId(0) + .setOperations(ByteString.copyFrom(operations)) + .addKeys(key) + .build(); + } + + private void setDefaultWitnessPermission(AccountCapsule accountCapsule) { + if (accountCapsule.getInstance().hasWitnessPermission()) { + return; + } + Permission owner = accountCapsule.getInstance().hasOwnerPermission() + ? accountCapsule.getInstance().getOwnerPermission() + : AccountCapsule.createDefaultOwnerPermission(accountCapsule.getAddress()); + Permission witness = AccountCapsule.createDefaultWitnessPermission(accountCapsule.getAddress()); + List actives = accountCapsule.getInstance().getActivePermissionCount() > 0 + ? new ArrayList<>(accountCapsule.getInstance().getActivePermissionList()) + : Collections.singletonList(createDefaultActivePermission(accountCapsule.getAddress())); + accountCapsule.updatePermissions(owner, witness, actives); + } + + @Override + public Integer call() throws IOException, RocksDBException, ContractValidateException { + if (help) { + spec.commandLine().usage(System.out); + return 0; + } + + File dbFile = Paths.get(database).toFile(); + if (!dbFile.exists() || !dbFile.isDirectory()) { + logger.error("Database [" + database + "] not exists!"); + spec.commandLine().getErr().format("Database %s not exists!", database).println(); + System.exit(-1); + } + File tmp = Paths.get(database, "database", "tmp").toFile(); + if (tmp.exists()) { + FileUtils.deleteDir(tmp); + } + + Config forkConfig = ConfigFactory.load(); + File file = Paths.get(config).toFile(); + if (file.exists() && file.isFile()) { + forkConfig = ConfigFactory.parseFile(file); + } else { + logger.error("Fork config file [" + config + "] not exists!"); + spec.commandLine().getErr().format("Fork config file: %s not exists!", config).println(); + System.exit(-1); + } + + initStore(); + processWitnesses(forkConfig); + processAccounts(forkConfig); + processTrc20Contracts(forkConfig); + processDynamicProperties(forkConfig); + + DbTool.close(); + return 0; + } + + private void processWitnesses(Config forkConfig) throws RocksDBException, IOException { + if (!retain) { + logger.info("Erase the previous witnesses and active witnesses."); + spec.commandLine().getOut().println("Erase the previous witnesses and active witnesses."); + witnessScheduleStore.delete(ACTIVE_WITNESSES); + DBIterator iterator = witnessStore.iterator(); + for (iterator.seekToFirst(); iterator.valid(); iterator.next()) { + witnessStore.delete(iterator.getKey()); + } + } else { + logger.warn("Keep the previous witnesses and active witnesses."); + spec.commandLine().getOut().println("Keep the previous witnesses and active witnesses."); + } + + if (!forkConfig.hasPath(WITNESS_KEY)) { + return; + } + + List witnesses = forkConfig.getConfigList(WITNESS_KEY); + if (witnesses.isEmpty()) { + spec.commandLine().getOut().println("no witness listed in the config."); + } + witnesses = witnesses.stream() + .filter(c -> c.hasPath(WITNESS_ADDRESS)) + .collect(Collectors.toList()); + + if (witnesses.isEmpty()) { + spec.commandLine().getOut().println("no witness listed in the config."); + } + + List witnessList = new ArrayList<>(); + witnesses.stream().forEach( + w -> { + ByteString address = ByteString.copyFrom( + Commons.decodeFromBase58Check(w.getString(WITNESS_ADDRESS))); + WitnessCapsule witness = new WitnessCapsule(address); + AccountCapsule accountCapsule = getOrCreateAccountCapsule(address.toByteArray()); + witness.setIsJobs(true); + accountCapsule.setIsWitness(true); + if (isMultiSignEnabled()) { + setDefaultWitnessPermission(accountCapsule); + } + if (w.hasPath(WITNESS_VOTE) && w.getLong(WITNESS_VOTE) > 0) { + witness.setVoteCount(w.getLong(WITNESS_VOTE)); + } + if (w.hasPath(WITNESS_URL)) { + witness.setUrl(w.getString(WITNESS_URL)); + } + witnessStore.put(address.toByteArray(), witness.getData()); + accountStore.put(address.toByteArray(), accountCapsule.getData()); + witnessList.add(witness.getAddress()); + }); + + witnessList.sort(Comparator.comparingLong((ByteString b) -> + new WitnessCapsule(witnessStore.get(b.toByteArray())).getVoteCount()) + .reversed() + .thenComparing(Comparator.comparingInt(ByteString::hashCode).reversed())); + List activeWitnesses = witnessList.subList(0, + witnesses.size() >= MAX_ACTIVE_WITNESS_NUM ? MAX_ACTIVE_WITNESS_NUM : witnessList.size()); + witnessScheduleStore.put(ACTIVE_WITNESSES, getActiveWitness(activeWitnesses)); + logger.info("{} witnesses and {} active witnesses have been modified.", + witnesses.size(), activeWitnesses.size()); + spec.commandLine().getOut().format("%d witnesses and %d active witnesses have been modified.", + witnesses.size(), activeWitnesses.size()).println(); + } + + private void processAccounts(Config forkConfig) throws RocksDBException, IOException { + if (!forkConfig.hasPath(ACCOUNTS_KEY)) { + return; + } + + List accounts = forkConfig.getConfigList(ACCOUNTS_KEY); + if (accounts.isEmpty()) { + spec.commandLine().getOut().println("no account listed in the config."); + } + + accounts = accounts.stream() + .filter(c -> c.hasPath(ACCOUNT_ADDRESS)) + .collect(Collectors.toList()); + + if (accounts.isEmpty()) { + spec.commandLine().getOut().println("no account listed in the config."); + } + + accounts.stream().forEach( + a -> { + byte[] address = Commons.decodeFromBase58Check(a.getString(ACCOUNT_ADDRESS)); + AccountCapsule accountCapsule = getOrCreateAccountCapsule(address); + + if (a.hasPath(ACCOUNT_BALANCE) && a.getLong(ACCOUNT_BALANCE) > 0) { + accountCapsule.setBalance(a.getLong(ACCOUNT_BALANCE)); + } + if (a.hasPath(ACCOUNT_NAME)) { + accountCapsule.setAccountName( + ByteArray.fromString(a.getString(ACCOUNT_NAME))); + } + if (a.hasPath(ACCOUNT_TYPE)) { + accountCapsule.updateAccountType( + AccountType.valueOf(a.getString(ACCOUNT_TYPE))); + } + + if (a.hasPath(ACCOUNT_OWNER)) { + byte[] owner = Commons.decodeFromBase58Check(a.getString(ACCOUNT_OWNER)); + Permission ownerPermission = AccountCapsule + .createDefaultOwnerPermission(ByteString.copyFrom(owner)); + accountCapsule.updatePermissions(ownerPermission, null, null); + } + + if (a.hasPath(ACCOUNT_TRC10_ID) && a.hasPath(ACCOUNT_TRC10_BALANCE) + && a.getLong(ACCOUNT_TRC10_BALANCE) > 0) { + String trc10Id = a.getString(ACCOUNT_TRC10_ID); + if (assetIssueV2Store.get(ByteArray.fromString(trc10Id)) != null) { + if (accountCapsule.getAssetOptimized()) { + byte[] k = Bytes.concat(address, ByteArray.fromString(trc10Id)); + accountAssetStore.put(k, Longs.toByteArray(a.getLong(ACCOUNT_TRC10_BALANCE))); + } else { + Map assetMapV2 = new HashMap<>(accountCapsule.getAssetMapV2()); + assetMapV2.put(trc10Id, a.getLong(ACCOUNT_TRC10_BALANCE)); + accountCapsule.clearAssetV2(); + accountCapsule.addAssetMapV2(assetMapV2); + } + } else { + logger.info("TRC10: {} not exists in the database.", trc10Id); + spec.commandLine().getOut().format("TRC10: %s not exists in the database.", trc10Id) + .println(); + } + } + + accountStore.put(address, accountCapsule.getData()); + }); + logger.info("{} accounts have been modified.", accounts.size()); + spec.commandLine().getOut().format("%d accounts have been modified.", accounts.size()) + .println(); + } + + private void processTrc20Contracts(Config forkConfig) throws RocksDBException, IOException { + if (!forkConfig.hasPath(TRC20_CONTRACTS_KEY)) { + return; + } + List trc20Contracts = forkConfig.getConfigList(TRC20_CONTRACTS_KEY); + if (trc20Contracts.isEmpty()) { + spec.commandLine().getOut().println("no TRC20 contract listed in the config."); + } + + trc20Contracts = trc20Contracts.stream() + .filter(c -> c.hasPath(TRC20_CONTRACT_ADDRESS) && c.hasPath(TRC20_BALANCES_POSITION) && + c.hasPath(TRC20_ACCOUNT) && c.hasPath(TRC20_BALANCE)) + .collect(Collectors.toList()); + + if (trc20Contracts.isEmpty()) { + spec.commandLine().getOut().println("no TRC20 contract listed in the config."); + } + + AtomicInteger cnt = new AtomicInteger(); + + trc20Contracts.stream().forEach( + contract -> { + byte[] contractAddress = Commons + .decodeFromBase58Check(contract.getString(TRC20_CONTRACT_ADDRESS)); + if (contractStore.get(contractAddress) == null) { + spec.commandLine().getErr().format("TRC20 contract: %s not exists in the database.", + contract.getString(TRC20_CONTRACT_ADDRESS)).println(); + return; + } + + SmartContract smartContract; + try { + smartContract = SmartContract.parseFrom(contractStore.get(contractAddress)); + } catch (InvalidProtocolBufferException e) { + e.printStackTrace(); + return; + } + + int balancesSlotPosition = 0; + if (contract.getInt(TRC20_BALANCES_POSITION) > 0) { + balancesSlotPosition = contract.getInt(TRC20_BALANCES_POSITION); + } + byte[] addressWithPrefix = Commons + .decodeFromBase58Check(contract.getString(TRC20_ACCOUNT)); + byte[] address = ByteArray.subArray(addressWithPrefix, 1, 21); + String paddedAddress = String + .format("%064x", new BigInteger(ByteArray.toHexString(address), 16)); + String paddedSlot = String.format("%064x", balancesSlotPosition); + byte[] contractKey = Hash.sha3(ByteArray.fromHexString( + paddedAddress + paddedSlot)); + + byte[] addressHash; + byte[] trxHash = smartContract.getTrxHash().toByteArray(); + if (ByteUtil.isNullOrZeroArray(trxHash)) { + addressHash = Hash.sha3(contractAddress); + } else { + addressHash = Hash.sha3(ByteUtil.merge(contractAddress, trxHash)); + } + + int contractVersion = smartContract.getVersion(); + if (contractVersion == 1) { + contractKey = Hash.sha3(contractKey); + } + byte[] rowKey = new byte[contractKey.length]; + arraycopy(addressHash, 0, rowKey, 0, 16); + arraycopy(contractKey, 16, rowKey, 16, 16); + + String paddedBalance = String + .format("%064x", new BigInteger(contract.getString(TRC20_BALANCE), 10)); + byte[] rowValue = ByteArray.fromHexString(paddedBalance); + StorageRowCapsule storageRowCapsule = new StorageRowCapsule(rowKey, rowValue); + + storageRowStore.put(rowKey, storageRowCapsule.getData()); + cnt.getAndIncrement(); + }); + logger.info("{} TRC20 contracts have been modified.", cnt.get()); + spec.commandLine().getOut() + .format("%d TRC20 contracts have been modified.", cnt.get()) + .println(); + } + + private void processDynamicProperties(Config forkConfig) throws ContractValidateException { + if (!forkConfig.hasPath(CHAIN_PARAMETERS) || forkConfig.getConfig(CHAIN_PARAMETERS).isEmpty()) { + return; + } + + Config chainParamsConfig = forkConfig.getConfig(CHAIN_PARAMETERS); + + if (chainParamsConfig.hasPath(ChainParameters.LATEST_BLOCK_TIMESTAMP) + && chainParamsConfig.getLong(ChainParameters.LATEST_BLOCK_TIMESTAMP) > 0) { + long latestBlockHeaderTimestamp = chainParamsConfig.getLong( + ChainParameters.LATEST_BLOCK_TIMESTAMP); + dynamicPropertiesStore + .put(LATEST_BLOCK_HEADER_TIMESTAMP, ByteArray.fromLong(latestBlockHeaderTimestamp)); + logger.info("The latest block header timestamp has been modified as {}.", + latestBlockHeaderTimestamp); + spec.commandLine().getOut().format("The LATEST_BLOCK_HEADER_TIMESTAMP has been modified " + + "as %d.", latestBlockHeaderTimestamp).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.MAINTENANCE_INTERVAL) + && chainParamsConfig.getLong(ChainParameters.MAINTENANCE_INTERVAL) > 0) { + long maintenanceTimeInterval = chainParamsConfig.getLong( + ChainParameters.MAINTENANCE_INTERVAL); + if (maintenanceTimeInterval < 3 * 27 * 1000 || maintenanceTimeInterval > 24 * 3600 * 1000) { + throw new ContractValidateException( + "Bad chain parameter value, valid range is [3 * 27 * 1000,24 * 3600 * 1000]"); + } + dynamicPropertiesStore + .put("MAINTENANCE_TIME_INTERVAL".getBytes(), ByteArray.fromLong(maintenanceTimeInterval)); + logger.info("The maintenance time interval has been modified as {}.", + maintenanceTimeInterval); + spec.commandLine().getOut().format("The MAINTENANCE_TIME_INTERVAL has been modified as %d.", + maintenanceTimeInterval).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.NEXT_MAINTENANCE_TIME) + && chainParamsConfig.getLong(ChainParameters.NEXT_MAINTENANCE_TIME) > 0) { + long nextMaintenanceTime = chainParamsConfig.getLong(ChainParameters.NEXT_MAINTENANCE_TIME); + dynamicPropertiesStore.put(MAINTENANCE_TIME, ByteArray.fromLong(nextMaintenanceTime)); + logger.info("The NEXT_MAINTENANCE_TIME has been modified as {}.", + nextMaintenanceTime); + spec.commandLine().getOut().format("The NEXT_MAINTENANCE_TIME has been modified as %d.", + nextMaintenanceTime).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ACCOUNT_UPGRADE_COST)) { + long accountUpgradeCost = chainParamsConfig.getLong(ChainParameters.ACCOUNT_UPGRADE_COST); + checkLongRange(accountUpgradeCost); + dynamicPropertiesStore.put("ACCOUNT_UPGRADE_COST".getBytes(), + ByteArray.fromLong(accountUpgradeCost)); + + spec.commandLine().getOut().format("The ACCOUNT_UPGRADE_COST has been modified as %d.", + accountUpgradeCost).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.CREATE_ACCOUNT_FEE)) { + long createAccountFee = chainParamsConfig.getLong(ChainParameters.CREATE_ACCOUNT_FEE); + checkLongRange(createAccountFee); + dynamicPropertiesStore.put("CREATE_ACCOUNT_FEE".getBytes(), + ByteArray.fromLong(createAccountFee)); + + spec.commandLine().getOut().format("The CREATE_ACCOUNT_FEE has been modified as %d.", + createAccountFee).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.TRANSACTION_FEE)) { + long transactionFee = chainParamsConfig.getLong(ChainParameters.TRANSACTION_FEE); + checkLongRange(transactionFee); + dynamicPropertiesStore.put("TRANSACTION_FEE".getBytes(), + ByteArray.fromLong(transactionFee)); + spec.commandLine().getOut().format("The TRANSACTION_FEE has been modified as %d.", + transactionFee).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ASSET_ISSUE_FEE)) { + long assetIssueFee = chainParamsConfig.getLong(ChainParameters.ASSET_ISSUE_FEE); + checkLongRange(assetIssueFee); + dynamicPropertiesStore.put("ASSET_ISSUE_FEE".getBytes(), + ByteArray.fromLong(assetIssueFee)); + spec.commandLine().getOut().format("The ASSET_ISSUE_FEE has been modified as %d.", + assetIssueFee).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.WITNESS_PAY_PER_BLOCK)) { + long witnessPayPerBlock = chainParamsConfig.getLong(ChainParameters.WITNESS_PAY_PER_BLOCK); + checkLongRange(witnessPayPerBlock); + dynamicPropertiesStore.put("WITNESS_PAY_PER_BLOCK".getBytes(), + ByteArray.fromLong(witnessPayPerBlock)); + spec.commandLine().getOut().format("The WITNESS_PAY_PER_BLOCK has been modified as %d.", + witnessPayPerBlock).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.WITNESS_STAND_BY_ALLOWANCE)) { + long witnessStandByAllowance = chainParamsConfig.getLong( + ChainParameters.WITNESS_STAND_BY_ALLOWANCE); + checkLongRange(witnessStandByAllowance); + dynamicPropertiesStore.put("WITNESS_STANDBY_ALLOWANCE".getBytes(), + ByteArray.fromLong(witnessStandByAllowance)); + spec.commandLine().getOut().format("The WITNESS_STANDBY_ALLOWANCE has been modified as %d.", + witnessStandByAllowance).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.CREATE_NEW_ACCOUNT_FEE_IN_SYSTEM_CONTRACT)) { + long createNewAccountFeeInSystemContract = chainParamsConfig.getLong( + ChainParameters.CREATE_NEW_ACCOUNT_FEE_IN_SYSTEM_CONTRACT); + checkLongRange(createNewAccountFeeInSystemContract); + dynamicPropertiesStore.put("CREATE_NEW_ACCOUNT_FEE_IN_SYSTEM_CONTRACT".getBytes(), + ByteArray.fromLong(createNewAccountFeeInSystemContract)); + spec.commandLine().getOut() + .format("The CREATE_NEW_ACCOUNT_FEE_IN_SYSTEM_CONTRACT has been modified as %d.", + createNewAccountFeeInSystemContract).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.CREATE_NEW_ACCOUNT_BANDWIDTH_RATE)) { + long createNewAccountBandwidthRate = chainParamsConfig.getLong( + ChainParameters.CREATE_NEW_ACCOUNT_BANDWIDTH_RATE); + checkLongRange(createNewAccountBandwidthRate); + dynamicPropertiesStore.put("CREATE_NEW_ACCOUNT_BANDWIDTH_RATE".getBytes(), + ByteArray.fromLong(createNewAccountBandwidthRate)); + spec.commandLine().getOut() + .format("The CREATE_NEW_ACCOUNT_BANDWIDTH_RATE has been modified as %d.", + createNewAccountBandwidthRate).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_CREATION_OF_CONTRACTS)) { + long allowCreationOfContracts = chainParamsConfig.getLong( + ChainParameters.ALLOW_CREATION_OF_CONTRACTS); + checkLongBoolean(allowCreationOfContracts); + dynamicPropertiesStore.put("ALLOW_CREATION_OF_CONTRACTS".getBytes(), + ByteArray.fromLong(allowCreationOfContracts)); + spec.commandLine().getOut() + .format("The ALLOW_CREATION_OF_CONTRACTS has been modified as %d.", + allowCreationOfContracts).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.REMOVE_THE_POWER_OF_THE_GR)) { + long removeThePowerOfTheGr = chainParamsConfig.getLong( + ChainParameters.REMOVE_THE_POWER_OF_THE_GR); + checkLongNegative(removeThePowerOfTheGr); + dynamicPropertiesStore.put("REMOVE_THE_POWER_OF_THE_GR".getBytes(), + ByteArray.fromLong(removeThePowerOfTheGr)); + spec.commandLine().getOut() + .format("The REMOVE_THE_POWER_OF_THE_GR has been modified as %d.", + removeThePowerOfTheGr).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ENERGY_FEE)) { + long energyFee = chainParamsConfig.getLong( + ChainParameters.ENERGY_FEE); + dynamicPropertiesStore.put("ENERGY_FEE".getBytes(), + ByteArray.fromLong(energyFee)); + spec.commandLine().getOut().format("The ENERGY_FEE has been modified as %d.", + energyFee).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.EXCHANGE_CREATE_FEE)) { + long exchangeCreateFee = chainParamsConfig.getLong( + ChainParameters.EXCHANGE_CREATE_FEE); + dynamicPropertiesStore.put("EXCHANGE_CREATE_FEE".getBytes(), + ByteArray.fromLong(exchangeCreateFee)); + spec.commandLine().getOut().format("The EXCHANGE_CREATE_FEE has been modified as %d.", + exchangeCreateFee).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.MAX_CPU_TIME_OF_ONE_TX)) { + long maxCpuTimeOfOneTx = chainParamsConfig.getLong( + ChainParameters.MAX_CPU_TIME_OF_ONE_TX); + if (maxCpuTimeOfOneTx < 10 || maxCpuTimeOfOneTx > 400) { + throw new ContractValidateException( + "Bad chain parameter value, valid range is [10,400]"); + } + dynamicPropertiesStore.put("MAX_CPU_TIME_OF_ONE_TX".getBytes(), + ByteArray.fromLong(maxCpuTimeOfOneTx)); + spec.commandLine().getOut().format("The MAX_CPU_TIME_OF_ONE_TX has been modified as %d.", + maxCpuTimeOfOneTx).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_UPDATE_ACCOUNT_NAME)) { + long allowUpdateAccountName = chainParamsConfig.getLong( + ChainParameters.ALLOW_UPDATE_ACCOUNT_NAME); + checkLongBoolean(allowUpdateAccountName); + dynamicPropertiesStore.put("ALLOW_UPDATE_ACCOUNT_NAME".getBytes(), + ByteArray.fromLong(allowUpdateAccountName)); + spec.commandLine().getOut().format("The ALLOW_UPDATE_ACCOUNT_NAME has been modified as %d.", + allowUpdateAccountName).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_SAME_TOKEN_NAME)) { + long allowSameTokenName = chainParamsConfig.getLong( + ChainParameters.ALLOW_SAME_TOKEN_NAME); + checkLongBoolean(allowSameTokenName); + dynamicPropertiesStore.put("ALLOW_SAME_TOKEN_NAME".getBytes(), + ByteArray.fromLong(allowSameTokenName)); + spec.commandLine().getOut().format("The ALLOW_SAME_TOKEN_NAME has been modified as %d.", + allowSameTokenName).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_DELEGATE_RESOURCE)) { + long allowDelegateResource = chainParamsConfig.getLong( + ChainParameters.ALLOW_DELEGATE_RESOURCE); + checkLongBoolean(allowDelegateResource); + dynamicPropertiesStore.put("ALLOW_DELEGATE_RESOURCE".getBytes(), + ByteArray.fromLong(allowDelegateResource)); + spec.commandLine().getOut().format("The ALLOW_DELEGATE_RESOURCE has been modified as %d.", + allowDelegateResource).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.TOTAL_ENERGY_LIMIT)) { + long totalEnergyLimit = chainParamsConfig.getLong( + ChainParameters.TOTAL_ENERGY_LIMIT); + checkLongRange(totalEnergyLimit); + dynamicPropertiesStore.put("TOTAL_ENERGY_LIMIT".getBytes(), + ByteArray.fromLong(totalEnergyLimit)); + spec.commandLine().getOut().format("The TOTAL_ENERGY_LIMIT has been modified as %d.", + totalEnergyLimit).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_TVM_TRANSFER_TRC10)) { + long allowTvmTransferTrc10 = chainParamsConfig.getLong( + ChainParameters.ALLOW_TVM_TRANSFER_TRC10); + checkLongBoolean(allowTvmTransferTrc10); + dynamicPropertiesStore.put("ALLOW_TVM_TRANSFER_TRC10".getBytes(), + ByteArray.fromLong(allowTvmTransferTrc10)); + spec.commandLine().getOut().format("The ALLOW_TVM_TRANSFER_TRC10 has been modified as %d.", + allowTvmTransferTrc10).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.TOTAL_ENERGY_CURRENT_LIMIT)) { + long totalEnergyCurrentLimit = chainParamsConfig.getLong( + ChainParameters.TOTAL_ENERGY_CURRENT_LIMIT); + checkLongRange(totalEnergyCurrentLimit); + dynamicPropertiesStore.put("TOTAL_ENERGY_LIMIT".getBytes(), + ByteArray.fromLong(totalEnergyCurrentLimit)); + spec.commandLine().getOut().format("The TOTAL_ENERGY__CURRENT_LIMIT has been modified as %d.", + totalEnergyCurrentLimit).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_MULTI_SIGN)) { + long allowMultiSign = chainParamsConfig.getLong( + ChainParameters.ALLOW_MULTI_SIGN); + checkLongBoolean(allowMultiSign); + dynamicPropertiesStore.put("ALLOW_MULTI_SIGN".getBytes(), + ByteArray.fromLong(allowMultiSign)); + spec.commandLine().getOut().format("The ALLOW_MULTI_SIGN has been modified as %d.", + allowMultiSign).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_ADAPTIVE_ENERGY)) { + long allowAdaptiveEnergy = chainParamsConfig.getLong( + ChainParameters.ALLOW_ADAPTIVE_ENERGY); + checkLongBoolean(allowAdaptiveEnergy); + dynamicPropertiesStore.put("ALLOW_ADAPTIVE_ENERGY".getBytes(), + ByteArray.fromLong(allowAdaptiveEnergy)); + spec.commandLine().getOut().format("The ALLOW_ADAPTIVE_ENERGY has been modified as %d.", + allowAdaptiveEnergy).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.TOTAL_ENERGY_TARGET_LIMIT)) { + long totalEnergyTargetLimit = chainParamsConfig.getLong( + ChainParameters.TOTAL_ENERGY_TARGET_LIMIT); + dynamicPropertiesStore.put("TOTAL_ENERGY_TARGET_LIMIT".getBytes(), + ByteArray.fromLong(totalEnergyTargetLimit)); + spec.commandLine().getOut().format("The TOTAL_ENERGY_TARGET_LIMIT has been modified as %d.", + totalEnergyTargetLimit).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.TOTAL_ENERGY_AVERAGE_USAGE)) { + long totalEnergyAverageUsage = chainParamsConfig.getLong( + ChainParameters.TOTAL_ENERGY_AVERAGE_USAGE); + dynamicPropertiesStore.put("TOTAL_ENERGY_AVERAGE_USAGE".getBytes(), + ByteArray.fromLong(totalEnergyAverageUsage)); + spec.commandLine().getOut().format("The TOTAL_ENERGY_AVERAGE_USAGE has been modified as %d.", + totalEnergyAverageUsage).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.UPDATE_ACCOUNT_PERMISSION_FEE)) { + long allowUpdateEnergyLimit = chainParamsConfig.getLong( + ChainParameters.UPDATE_ACCOUNT_PERMISSION_FEE); + if (allowUpdateEnergyLimit < 0 || allowUpdateEnergyLimit > ChainParameters.MAX_SUPPLY) { + throw new ContractValidateException( + "Bad chain parameter value, valid range is [0," + ChainParameters.MAX_SUPPLY + "]"); + } + dynamicPropertiesStore.put("UPDATE_ACCOUNT_PERMISSION_FEE".getBytes(), + ByteArray.fromLong(allowUpdateEnergyLimit)); + spec.commandLine().getOut().format("The UPDATE_ACCOUNT_PERMISSION_FEE has been modified as %d.", + allowUpdateEnergyLimit).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.MULTI_SIGN_FEE)) { + long multiSignFee = chainParamsConfig.getLong( + ChainParameters.MULTI_SIGN_FEE); + if (multiSignFee < 0 || multiSignFee > ChainParameters.MAX_SUPPLY) { + throw new ContractValidateException( + "Bad chain parameter value, valid range is [0," + ChainParameters.MAX_SUPPLY + "]"); + } + dynamicPropertiesStore.put("MULTI_SIGN_FEE".getBytes(), ByteArray.fromLong(multiSignFee)); + spec.commandLine().getOut().format("The MULTI_SIGN_FEE has been modified as %d.", + multiSignFee).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_ACCOUNT_STATE_ROOT)) { + long allowAccountStateRoot = chainParamsConfig.getLong( + ChainParameters.ALLOW_ACCOUNT_STATE_ROOT); + checkLongBoolean(allowAccountStateRoot); + dynamicPropertiesStore.put("ALLOW_ACCOUNT_STATE_ROOT".getBytes(), + ByteArray.fromLong(allowAccountStateRoot)); + spec.commandLine().getOut().format("The ALLOW_ACCOUNT_STATE_ROOT has been modified as %d.", + allowAccountStateRoot).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_PROTO_FILTER_NUM)) { + long allowProtoFilterNum = chainParamsConfig.getLong( + ChainParameters.ALLOW_PROTO_FILTER_NUM); + checkLongBoolean(allowProtoFilterNum); + dynamicPropertiesStore.put("ALLOW_PROTO_FILTER_NUM".getBytes(), + ByteArray.fromLong(allowProtoFilterNum)); + spec.commandLine().getOut().format("The ALLOW_PROTO_FILTER_NUM has been modified as %d.", + allowProtoFilterNum).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_TVM_CONSTANTINOPLE)) { + long allowTVMConstantinople = chainParamsConfig.getLong( + ChainParameters.ALLOW_TVM_CONSTANTINOPLE); + checkLongBoolean(allowTVMConstantinople); + dynamicPropertiesStore.put("ALLOW_TVM_CONSTANTINOPLE".getBytes(), + ByteArray.fromLong(allowTVMConstantinople)); + spec.commandLine().getOut().format("The ALLOW_TVM_CONSTANTINOPLE has been modified as %d.", + allowTVMConstantinople).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_TVM_SOLIDITY059)) { + long allowTVMSolidity059 = chainParamsConfig.getLong( + ChainParameters.ALLOW_TVM_SOLIDITY059); + checkLongBoolean(allowTVMSolidity059); + dynamicPropertiesStore.put("ALLOW_TVM_SOLIDITY_059".getBytes(), + ByteArray.fromLong(allowTVMSolidity059)); + spec.commandLine().getOut().format("The ALLOW_TVM_SOLIDITY_059 has been modified as %d.", + allowTVMSolidity059).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_TVM_ISTANBUL)) { + long allowTVMIStanbul = chainParamsConfig.getLong( + ChainParameters.ALLOW_TVM_ISTANBUL); + checkLongBoolean(allowTVMIStanbul); + dynamicPropertiesStore.put("ALLOW_TVM_ISTANBUL".getBytes(), + ByteArray.fromLong(allowTVMIStanbul)); + spec.commandLine().getOut().format("The ALLOW_TVM_ISTANBUL has been modified as %d.", + allowTVMIStanbul).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_SHIELDED_TRC20_TRANSACTION)) { + long allowShieldedTRC20Transaction = chainParamsConfig.getLong( + ChainParameters.ALLOW_SHIELDED_TRC20_TRANSACTION); + checkLongBoolean(allowShieldedTRC20Transaction); + dynamicPropertiesStore.put("ALLOW_SHIELDED_TRC20_TRANSACTION".getBytes(), + ByteArray.fromLong(allowShieldedTRC20Transaction)); + spec.commandLine().getOut().format("The ALLOW_SHIELDED_TRC20_TRANSACTION has been modified as %d.", + allowShieldedTRC20Transaction).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.FORBID_TRANSFER_TO_CONTRACT)) { + long forbidTransferToContract = chainParamsConfig.getLong( + ChainParameters.FORBID_TRANSFER_TO_CONTRACT); + checkLongBoolean(forbidTransferToContract); + dynamicPropertiesStore.put("FORBID_TRANSFER_TO_CONTRACT".getBytes(), + ByteArray.fromLong(forbidTransferToContract)); + spec.commandLine().getOut().format("The FORBID_TRANSFER_TO_CONTRACT has been modified as %d.", + forbidTransferToContract).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ADAPTIVE_RESOURCE_LIMIT_TARGET_RATIO)) { + long adaptiveResourceLimitTargetRatio = chainParamsConfig.getLong( + ChainParameters.ADAPTIVE_RESOURCE_LIMIT_TARGET_RATIO); + if (adaptiveResourceLimitTargetRatio < 1 || adaptiveResourceLimitTargetRatio > 1_000) { + throw new ContractValidateException( + "Bad chain parameter value, valid range is [1,1_000]"); + } + dynamicPropertiesStore.put("ADAPTIVE_RESOURCE_LIMIT_TARGET_RATIO".getBytes(), + ByteArray.fromLong(adaptiveResourceLimitTargetRatio)); + spec.commandLine().getOut() + .format("The ADAPTIVE_RESOURCE_LIMIT_TARGET_RATIO has been modified as %d.", + adaptiveResourceLimitTargetRatio).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ADAPTIVE_RESOURCE_LIMIT_MULTIPLIER)) { + long adaptiveResourceLimitMultiplier = chainParamsConfig.getLong( + ChainParameters.ADAPTIVE_RESOURCE_LIMIT_MULTIPLIER); + if (adaptiveResourceLimitMultiplier < 1 || adaptiveResourceLimitMultiplier > 10_000L) { + throw new ContractValidateException( + "Bad chain parameter value, valid range is [1,10_000]"); + } + dynamicPropertiesStore.put("ADAPTIVE_RESOURCE_LIMIT_MULTIPLIER".getBytes(), + ByteArray.fromLong(adaptiveResourceLimitMultiplier)); + spec.commandLine().getOut() + .format("The ADAPTIVE_RESOURCE_LIMIT_MULTIPLIER has been modified as %d.", + adaptiveResourceLimitMultiplier).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.CHANGE_DELEGATION)) { + long changeDelegation = chainParamsConfig.getLong( + ChainParameters.CHANGE_DELEGATION); + checkLongBoolean(changeDelegation); + dynamicPropertiesStore.put("CHANGE_DELEGATION".getBytes(), + ByteArray.fromLong(changeDelegation)); + spec.commandLine().getOut().format("The CHANGE_DELEGATION has been modified as %d.", + changeDelegation).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.WITNESS127_PAY_PER_BLOCK)) { + long witness127PayPerBlock = chainParamsConfig.getLong( + ChainParameters.WITNESS127_PAY_PER_BLOCK); + checkLongRange(witness127PayPerBlock); + dynamicPropertiesStore.put("WITNESS_127_PAY_PER_BLOCK".getBytes(), + ByteArray.fromLong(witness127PayPerBlock)); + spec.commandLine().getOut().format("The WITNESS_127_PAY_PER_BLOCK has been modified as %d.", + witness127PayPerBlock).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_MARKET_TRANSACTION)) { + long allowMarketTransaction = chainParamsConfig.getLong( + ChainParameters.ALLOW_MARKET_TRANSACTION); + checkLongBoolean(allowMarketTransaction); + dynamicPropertiesStore.put("ALLOW_MARKET_TRANSACTION".getBytes(), + ByteArray.fromLong(allowMarketTransaction)); + spec.commandLine().getOut().format("The ALLOW_MARKET_TRANSACTION has been modified as %d.", + allowMarketTransaction).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.MARKET_SELL_FEE)) { + long marketSellFee = chainParamsConfig.getLong( + ChainParameters.MARKET_SELL_FEE); + if (marketSellFee < 0 || marketSellFee > 10_000_000_000L) { + throw new ContractValidateException( + "Bad MARKET_SELL_FEE parameter value, valid range is [0,10_000_000_000L]"); + } + dynamicPropertiesStore.put("MARKET_SELL_FEE".getBytes(), + ByteArray.fromLong(marketSellFee)); + spec.commandLine().getOut().format("The MARKET_SELL_FEE has been modified as %d.", + marketSellFee).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.MARKET_CANCEL_FEE)) { + long marketCancelFee = chainParamsConfig.getLong( + ChainParameters.MARKET_CANCEL_FEE); + if (marketCancelFee < 0 || marketCancelFee > 10_000_000_000L) { + throw new ContractValidateException( + "Bad MARKET_CANCEL_FEE parameter value, valid range is [0,10_000_000_000L]"); + } + dynamicPropertiesStore.put("MARKET_CANCEL_FEE".getBytes(), + ByteArray.fromLong(marketCancelFee)); + spec.commandLine().getOut().format("The MARKET_CANCEL_FEE has been modified as %d.", + marketCancelFee).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_PBFT)) { + long allowPbft = chainParamsConfig.getLong( + ChainParameters.ALLOW_PBFT); + checkLongBoolean(allowPbft); + dynamicPropertiesStore.put("ALLOW_PBFT".getBytes(), + ByteArray.fromLong(allowPbft)); + spec.commandLine().getOut().format("The ALLOW_PBFT has been modified as %d.", + allowPbft).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_TRANSACTION_FEE_POOL)) { + long allowTransactionFeePool = chainParamsConfig.getLong( + ChainParameters.ALLOW_TRANSACTION_FEE_POOL); + checkLongBoolean(allowTransactionFeePool); + dynamicPropertiesStore.put("ALLOW_TRANSACTION_FEE_POOL".getBytes(), + ByteArray.fromLong(allowTransactionFeePool)); + spec.commandLine().getOut().format("The ALLOW_TRANSACTION_FEE_POOL has been modified as %d.", + allowTransactionFeePool).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.MAX_FEE_LIMIT)) { + long maxFeeLimit = chainParamsConfig.getLong( + ChainParameters.MAX_FEE_LIMIT); + if (maxFeeLimit < 0 || maxFeeLimit > 10_000_000_000L) { + throw new ContractValidateException( + "Bad MAX_FEE_LIMIT parameter value, valid range is [0,10_000_000_000L]"); + } + dynamicPropertiesStore.put("MAX_FEE_LIMIT".getBytes(), + ByteArray.fromLong(maxFeeLimit)); + spec.commandLine().getOut().format("The MAX_FEE_LIMIT has been modified as %d.", + maxFeeLimit).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_OPTIMIZE_BLACK_HOLE)) { + long allowOptimizeBlackHole = chainParamsConfig.getLong( + ChainParameters.ALLOW_OPTIMIZE_BLACK_HOLE); + checkLongBoolean(allowOptimizeBlackHole); + dynamicPropertiesStore.put("ALLOW_OPTIMIZE_BLACK_HOLE".getBytes(), + ByteArray.fromLong(allowOptimizeBlackHole)); + spec.commandLine().getOut().format("The ALLOW_OPTIMIZE_BLACK_HOLE has been modified as %d.", + allowOptimizeBlackHole).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_NEW_RESOURCE_MODEL)) { + long allowNewResourceModel = chainParamsConfig.getLong( + ChainParameters.ALLOW_NEW_RESOURCE_MODEL); + checkLongBoolean(allowNewResourceModel); + dynamicPropertiesStore.put("ALLOW_NEW_RESOURCE_MODEL".getBytes(), + ByteArray.fromLong(allowNewResourceModel)); + spec.commandLine().getOut().format("The ALLOW_NEW_RESOURCE_MODEL has been modified as %d.", + allowNewResourceModel).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_TVM_FREEZE)) { + long allowTVMFreeze = chainParamsConfig.getLong( + ChainParameters.ALLOW_TVM_FREEZE); + checkLongBoolean(allowTVMFreeze); + dynamicPropertiesStore.put("ALLOW_TVM_FREEZE".getBytes(), + ByteArray.fromLong(allowTVMFreeze)); + spec.commandLine().getOut().format("The ALLOW_TVM_FREEZE has been modified as %d.", + allowTVMFreeze).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_TVM_VOTE)) { + long allowTVMVote = chainParamsConfig.getLong( + ChainParameters.ALLOW_TVM_VOTE); + checkLongBoolean(allowTVMVote); + dynamicPropertiesStore.put("ALLOW_TVM_VOTE".getBytes(), + ByteArray.fromLong(allowTVMVote)); + spec.commandLine().getOut().format("The ALLOW_TVM_VOTE has been modified as %d.", + allowTVMVote).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_TVM_LONDON)) { + long allowTVMLondon = chainParamsConfig.getLong(ChainParameters.ALLOW_TVM_LONDON); + checkLongBoolean(allowTVMLondon); + dynamicPropertiesStore.put("ALLOW_TVM_LONDON".getBytes(), ByteArray.fromLong(allowTVMLondon)); + spec.commandLine().getOut().format("The ALLOW_TVM_LONDON has been modified as %d.", + allowTVMLondon).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_TVM_COMPATIBLE_EVM)) { + long allowTVMCompatibleEVM = chainParamsConfig.getLong(ChainParameters.ALLOW_TVM_COMPATIBLE_EVM); + checkLongBoolean(allowTVMCompatibleEVM); + dynamicPropertiesStore.put("ALLOW_TVM_COMPATIBLE_EVM".getBytes(), + ByteArray.fromLong(allowTVMCompatibleEVM)); + spec.commandLine().getOut().format("The ALLOW_TVM_COMPATIBLE_EVM has been modified as %d.", + allowTVMCompatibleEVM).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_ACCOUNT_ASSET_OPTIMIZATION)) { + long allowAccountAssetOptimization = chainParamsConfig.getLong( + ChainParameters.ALLOW_ACCOUNT_ASSET_OPTIMIZATION); + checkLongBoolean(allowAccountAssetOptimization); + dynamicPropertiesStore.put("ALLOW_ACCOUNT_ASSET_OPTIMIZATION".getBytes(), + ByteArray.fromLong(allowAccountAssetOptimization)); + spec.commandLine().getOut() + .format("The ALLOW_ACCOUNT_ASSET_OPTIMIZATION has been modified as %d.", + allowAccountAssetOptimization).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.FREE_NET_LIMIT)) { + long freeNetLimit = chainParamsConfig.getLong(ChainParameters.FREE_NET_LIMIT); + if (freeNetLimit < 0 || freeNetLimit > 100_000L) { + throw new ContractValidateException( + "Bad chain parameter value, valid range is [0,100_000]"); + } + dynamicPropertiesStore.put("FREE_NET_LIMIT".getBytes(), + ByteArray.fromLong(freeNetLimit)); + spec.commandLine().getOut().format("The FREE_NET_LIMIT has been modified as %d.", + freeNetLimit).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.TOTAL_NET_LIMIT)) { + long totalNetLimit = chainParamsConfig.getLong(ChainParameters.TOTAL_NET_LIMIT); + if (totalNetLimit < 0 || totalNetLimit > 1_000_000_000_000L) { + throw new ContractValidateException( + "Bad chain parameter value, valid range is [0, 1_000_000_000_000L]"); + } + dynamicPropertiesStore.put("TOTAL_NET_LIMIT".getBytes(), + ByteArray.fromLong(totalNetLimit)); + spec.commandLine().getOut().format("The TOTAL_NET_LIMIT has been modified as %d.", + totalNetLimit).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_HIGHER_LIMIT_FOR_MAX_CPU_TIME_OF_ONE_TX)) { + long allowHigherLimitForMaxCpuTimeOfOneTx = chainParamsConfig.getLong( + ChainParameters.ALLOW_HIGHER_LIMIT_FOR_MAX_CPU_TIME_OF_ONE_TX); + checkLongBoolean(allowHigherLimitForMaxCpuTimeOfOneTx); + dynamicPropertiesStore.put("ALLOW_HIGHER_LIMIT_FOR_MAX_CPU_TIME_OF_ONE_TX".getBytes(), + ByteArray.fromLong(allowHigherLimitForMaxCpuTimeOfOneTx)); + spec.commandLine().getOut() + .format("The ALLOW_HIGHER_LIMIT_FOR_MAX_CPU_TIME_OF_ONE_TX has been modified as %d.", + allowHigherLimitForMaxCpuTimeOfOneTx).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_ASSET_OPTIMIZATION)) { + long allowAssetOptimization = chainParamsConfig.getLong( + ChainParameters.ALLOW_ASSET_OPTIMIZATION); + checkLongBoolean(allowAssetOptimization); + dynamicPropertiesStore.put("ALLOW_ASSET_OPTIMIZATION".getBytes(), + ByteArray.fromLong(allowAssetOptimization)); + spec.commandLine().getOut().format("The ALLOW_ASSET_OPTIMIZATION has been modified as %d.", + allowAssetOptimization).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_NEW_REWARD)) { + long allowNewReward = chainParamsConfig.getLong(ChainParameters.ALLOW_NEW_REWARD); + checkLongBoolean(allowNewReward); + dynamicPropertiesStore.put("ALLOW_UPDATE_ACCOUNT_NAME".getBytes(), + ByteArray.fromLong(allowNewReward)); + spec.commandLine().getOut().format("The ALLOW_NEW_REWARD has been modified as %d.", + allowNewReward).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.MEMO_FEE)) { + long memoFee = chainParamsConfig.getLong(ChainParameters.MEMO_FEE); + if (memoFee < 0 || memoFee > 1_000_000_000) { + throw new ContractValidateException( + "This value[MEMO_FEE] is only allowed to be in the range 0-1000_000_000"); + } + dynamicPropertiesStore.put("MEMO_FEE".getBytes(), ByteArray.fromLong(memoFee)); + spec.commandLine().getOut().format("The MEMO_FEE has been modified as %d.", + memoFee).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_DELEGATE_OPTIMIZATION)) { + long allowDelegateOptimization = chainParamsConfig.getLong( + ChainParameters.ALLOW_DELEGATE_OPTIMIZATION); + checkLongBoolean(allowDelegateOptimization); + dynamicPropertiesStore.put("ALLOW_DELEGATE_OPTIMIZATION".getBytes(), + ByteArray.fromLong(allowDelegateOptimization)); + spec.commandLine().getOut().format("The ALLOW_DELEGATE_OPTIMIZATION has been modified as %d.", + allowDelegateOptimization).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.UNFREEZE_DELAY_DAYS)) { + long unfreezeDelayDays = chainParamsConfig.getLong(ChainParameters.UNFREEZE_DELAY_DAYS); + if (unfreezeDelayDays < 1 || unfreezeDelayDays > 365) { + throw new ContractValidateException( + "This value[UNFREEZE_DELAY_DAYS] is only allowed to be in the range 1-365"); + } dynamicPropertiesStore.put("UNFREEZE_DELAY_DAYS".getBytes(), + ByteArray.fromLong(unfreezeDelayDays)); + spec.commandLine().getOut().format("The UNFREEZE_DELAY_DAYS has been modified as %d.", + unfreezeDelayDays).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_OPTIMIZED_RETURN_VALUE_OF_CHAIN_ID)) { + long value = chainParamsConfig.getLong( + ChainParameters.ALLOW_OPTIMIZED_RETURN_VALUE_OF_CHAIN_ID); + checkLongBoolean(value); + dynamicPropertiesStore.put("ALLOW_UPDATE_ACCOUNT_NAME".getBytes(), + ByteArray.fromLong(value)); + spec.commandLine().getOut() + .format("The ALLOW_OPTIMIZED_RETURN_VALUE_OF_CHAIN_ID has been modified as %d.", + value).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_DYNAMIC_ENERGY)) { + long value = chainParamsConfig.getLong(ChainParameters.ALLOW_DYNAMIC_ENERGY); + checkLongBoolean(value); + dynamicPropertiesStore.put("ALLOW_DYNAMIC_ENERGY".getBytes(), ByteArray.fromLong(value)); + spec.commandLine().getOut().format("The ALLOW_DYNAMIC_ENERGY has been modified as %d.", + value).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.DYNAMIC_ENERGY_THRESHOLD)) { + long value = chainParamsConfig.getLong(ChainParameters.DYNAMIC_ENERGY_THRESHOLD); + checkLongRange(value); + dynamicPropertiesStore.put("DYNAMIC_ENERGY_THRESHOLD".getBytes(), + ByteArray.fromLong(value)); + spec.commandLine().getOut().format("The DYNAMIC_ENERGY_THRESHOLD has been modified as %d.", + value).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.DYNAMIC_ENERGY_INCREASE_FACTOR)) { + long value = chainParamsConfig.getLong(ChainParameters.DYNAMIC_ENERGY_INCREASE_FACTOR); + if (value < 0 || value > 10_000L) { + throw new ContractValidateException( + "This value[DYNAMIC_ENERGY_INCREASE_FACTOR] " + + "is only allowed to be in the range 0-" + + DYNAMIC_ENERGY_INCREASE_FACTOR_RANGE + ); + } + dynamicPropertiesStore.put("DYNAMIC_ENERGY_INCREASE_FACTOR".getBytes(), + ByteArray.fromLong(value)); + spec.commandLine().getOut() + .format("The DYNAMIC_ENERGY_INCREASE_FACTOR has been modified as %d.", + value).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_TVM_SHANGHAI)) { + long value = chainParamsConfig.getLong(ChainParameters.ALLOW_TVM_SHANGHAI); + checkLongBoolean(value); + dynamicPropertiesStore.put("ALLOW_TVM_SHANGHAI".getBytes(), ByteArray.fromLong(value)); + spec.commandLine().getOut().format("The ALLOW_TVM_SHANGHAI has been modified as %d.", + value).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_CANCEL_ALL_UNFREEZE_V2)) { + long value = chainParamsConfig.getLong(ChainParameters.ALLOW_CANCEL_ALL_UNFREEZE_V2); + checkLongBoolean(value); + dynamicPropertiesStore.put("ALLOW_CANCEL_ALL_UNFREEZE_V2".getBytes(), + ByteArray.fromLong(value)); + spec.commandLine().getOut().format("The ALLOW_CANCEL_ALL_UNFREEZE_V2 has been modified as %d.", + value).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.MAX_DELEGATE_LOCK_PERIOD)) { + long value = chainParamsConfig.getLong(ChainParameters.MAX_DELEGATE_LOCK_PERIOD); + long maxDelegateLockPeriod = ByteArray.toLong( + dynamicPropertiesStore.get("MAX_DELEGATE_LOCK_PERIOD".getBytes())); + if (value <= maxDelegateLockPeriod || value > ONE_YEAR_BLOCK_NUMBERS) { + throw new ContractValidateException( + "This value[MAX_DELEGATE_LOCK_PERIOD] is only allowed to be greater than " + + maxDelegateLockPeriod + " and less than or equal to " + ONE_YEAR_BLOCK_NUMBERS + + " !"); + } + dynamicPropertiesStore.put("ALLOW_TVM_LIGHT_NODE".getBytes(), ByteArray.fromLong(value)); + spec.commandLine().getOut().format("The MAX_DELEGATE_LOCK_PERIOD has been modified as %d.", + value).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_ENERGY_ADJUSTMENT)) { + long value = chainParamsConfig.getLong(ChainParameters.ALLOW_ENERGY_ADJUSTMENT); + checkLongBoolean(value); + dynamicPropertiesStore.put("ALLOW_ENERGY_ADJUSTMENT".getBytes(), + ByteArray.fromLong(value)); + spec.commandLine().getOut() + .format("The ALLOW_ENERGY_ADJUSTMENT has been modified as %d.", value); + } + + if (chainParamsConfig.hasPath(ChainParameters.MAX_CREATE_ACCOUNT_TX_SIZE)) { + long value = chainParamsConfig.getLong(ChainParameters.MAX_CREATE_ACCOUNT_TX_SIZE); + if (value < CREATE_ACCOUNT_TRANSACTION_MIN_BYTE_SIZE + || value > CREATE_ACCOUNT_TRANSACTION_MAX_BYTE_SIZE) { + throw new ContractValidateException( + "This value[MAX_CREATE_ACCOUNT_TX_SIZE] is only allowed to be greater than or equal " + + "to " + CREATE_ACCOUNT_TRANSACTION_MIN_BYTE_SIZE + " and less than or equal to " + + CREATE_ACCOUNT_TRANSACTION_MAX_BYTE_SIZE + "!"); + } + dynamicPropertiesStore.put("MAX_CREATE_ACCOUNT_TX_SIZE".getBytes(), + ByteArray.fromLong(value)); + spec.commandLine().getOut() + .format("The MAX_CREATE_ACCOUNT_TX_SIZE has been modified as %d.", value); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_STRICT_MATH)) { + long value = chainParamsConfig.getLong(ChainParameters.ALLOW_STRICT_MATH); + checkLongBoolean(value); + dynamicPropertiesStore.put("ALLOW_STRICT_MATH".getBytes(), ByteArray.fromLong(value)); + spec.commandLine().getOut().format("The ALLOW_STRICT_MATH has been modified as %d.", + value).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.CONSENSUS_LOGIC_OPTIMIZATION)) { + long value = chainParamsConfig.getLong(ChainParameters.CONSENSUS_LOGIC_OPTIMIZATION); + checkLongBoolean(value); + dynamicPropertiesStore.put("CONSENSUS_LOGIC_OPTIMIZATION".getBytes(), + ByteArray.fromLong(value)); + spec.commandLine().getOut().format("The CONSENSUS_LOGIC_OPTIMIZATION has been modified as %d.", + value).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_TVM_CANCUN)) { + long value = chainParamsConfig.getLong(ChainParameters.ALLOW_TVM_CANCUN); + checkLongBoolean(value); + dynamicPropertiesStore.put("ALLOW_TVM_CANCUN".getBytes(), + ByteArray.fromLong(value)); + spec.commandLine().getOut().format("The ALLOW_TVM_CANCUN has been modified as %d.", + value).println(); + } + + if (chainParamsConfig.hasPath(ChainParameters.ALLOW_TVM_BLOB)) { + long value = chainParamsConfig.getLong(ChainParameters.ALLOW_TVM_BLOB); + checkLongBoolean(value); + dynamicPropertiesStore.put("ALLOW_TVM_BLOB".getBytes(), + ByteArray.fromLong(value)); + spec.commandLine().getOut().format("The ALLOW_TVM_BLOB has been modified as %d.", + value).println(); + } + } + + private void checkLongRange(long value) throws ContractValidateException { + if (value < 0 || value > ChainParameters.LONG_VALUE) { + throw new ContractValidateException( + "Bad chain parameter value, valid range is [0," + ChainParameters.LONG_VALUE + "]"); + } + } + + private void checkLongBoolean(long value) throws ContractValidateException { + if (value != 0 && value != 1) { + throw new ContractValidateException( + "Bad chain parameter value, valid value is 0 or 1"); + } + } + + private void checkLongNegative(long value) throws ContractValidateException { + if (value != 0 && value != 1 && value != -1) { + throw new ContractValidateException( + "Bad chain parameter value, valid value is -1, 0, 1."); + } + } + + + public static byte[] getActiveWitness(List witnesses) { + byte[] ba = new byte[witnesses.size() * Constant.ADDRESS_BYTE_ARRAY_LENGTH]; + int i = 0; + for (ByteString address : witnesses) { + System.arraycopy(address.toByteArray(), 0, + ba, i * Constant.ADDRESS_BYTE_ARRAY_LENGTH, Constant.ADDRESS_BYTE_ARRAY_LENGTH); + i++; + } + return ba; + } +} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/DbLite.java b/tools/toolkit/src/main/java/common/org/tron/plugins/DbLite.java similarity index 99% rename from tools/toolkit/src/main/java/org/tron/plugins/DbLite.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/DbLite.java index eb11883c..5ce2a445 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/DbLite.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/DbLite.java @@ -656,12 +656,13 @@ private boolean isLite(String databaseDir) throws RocksDBException, IOException private long getSecondBlock(String databaseDir) throws RocksDBException, IOException { long num = 0; DBInterface sourceBlockIndexDb = DbTool.getDB(databaseDir, BLOCK_INDEX_DB_NAME); - DBIterator iterator = sourceBlockIndexDb.iterator(); - iterator.seek(ByteArray.fromLong(1)); - if (iterator.hasNext()) { - num = Longs.fromByteArray(iterator.getKey()); + try (DBIterator iterator = sourceBlockIndexDb.iterator()) { + iterator.seek(ByteArray.fromLong(1)); + if (iterator.hasNext()) { + num = Longs.fromByteArray(iterator.getKey()); + } + return num; } - return num; } private DBInterface getCheckpointDb(String sourceDir) throws IOException, RocksDBException { diff --git a/tools/toolkit/src/main/java/org/tron/plugins/DbMove.java b/tools/toolkit/src/main/java/common/org/tron/plugins/DbMove.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/DbMove.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/DbMove.java diff --git a/tools/toolkit/src/main/java/org/tron/plugins/DbQuery.java b/tools/toolkit/src/main/java/common/org/tron/plugins/DbQuery.java similarity index 75% rename from tools/toolkit/src/main/java/org/tron/plugins/DbQuery.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/DbQuery.java index 16dc2ff3..279176a1 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/DbQuery.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/DbQuery.java @@ -14,9 +14,14 @@ import static org.tron.plugins.utils.Constant.MAINTENANCE_TIME_INTERVAL; import static org.tron.plugins.utils.Constant.NEW_REWARD_ALGORITHM_EFFECTIVE_CYCLE; import static org.tron.plugins.utils.Constant.REWARDS_KEY; +import static org.tron.plugins.utils.Constant.VOTERS_ALL_ACTIVE_WITNESSES; +import static org.tron.plugins.utils.Constant.VOTERS_ALL_WITNESSES; +import static org.tron.plugins.utils.Constant.VOTERS_WITNESS_LIST; +import static org.tron.plugins.utils.Constant.VOTERS_WITNESS_THRESHOLD; import static org.tron.plugins.utils.Constant.VOTES_ALL_WITNESSES; import static org.tron.plugins.utils.Constant.VOTES_STORE; import static org.tron.plugins.utils.Constant.VOTES_WITNESS_LIST; +import static org.tron.plugins.utils.Constant.WITNESS_SCHEDULE_STORE; import static org.tron.plugins.utils.Constant.WITNESS_STORE; import com.google.common.collect.Maps; @@ -29,6 +34,7 @@ import java.math.BigInteger; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -54,6 +60,7 @@ import org.tron.core.capsule.WitnessCapsule; import org.tron.core.exception.BadItemException; import org.tron.core.store.DelegationStore; +import org.tron.plugins.utils.Constant; import org.tron.plugins.utils.FileUtils; import org.tron.plugins.utils.JsonFormat; import org.tron.plugins.utils.db.DBInterface; @@ -74,7 +81,7 @@ public class DbQuery implements Callable { @CommandLine.Spec - CommandLine.Model.CommandSpec spec; + static CommandLine.Model.CommandSpec spec; @CommandLine.Option(names = {"-d", "--database-directory"}, defaultValue = "output-directory", @@ -97,6 +104,7 @@ public class DbQuery implements Callable { private DBInterface blockStore; private DBInterface accountStore; private DBInterface delegationStore; + private DBInterface witnessScheduleStore; boolean allWitness = false; List witnessList = new ArrayList<>(); @@ -106,6 +114,11 @@ public class DbQuery implements Callable { List rewardAddressList = new ArrayList<>(); Map latestWitnessVi = new HashMap<>(); + boolean allWitnessVoters = false; + boolean allActiveWitnesses = false; + List votersWitnessList = new ArrayList<>(); + long threshold = 0; + private void initStore() throws IOException, RocksDBException { String srcDir = database + File.separator + "database"; witnessStore = DbTool.getDB(srcDir, WITNESS_STORE); @@ -115,6 +128,7 @@ private void initStore() throws IOException, RocksDBException { blockStore = DbTool.getDB(srcDir, BLOCK_STORE); accountStore = DbTool.getDB(srcDir, ACCOUNT_STORE); delegationStore = DbTool.getDB(srcDir, DELEGATION_STORE); + witnessScheduleStore = DbTool.getDB(srcDir, WITNESS_SCHEDULE_STORE); } @@ -149,6 +163,7 @@ public Integer call() throws Exception { initStore(); processVotes(queryConfig); processRewards(queryConfig); + processVoters(queryConfig); DbTool.close(); return 0; @@ -283,6 +298,12 @@ private void processVotes(Config queryConfig) throws BadItemException { spec.commandLine().getOut().println(cnt.get() + " " + output); logger.info(cnt.get() + " " + output); }); + + try { + iterator.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } } private String formatWitness(WitnessCapsule witnessCapsule, long previousCnt) { @@ -413,6 +434,168 @@ private Boolean existInWitnessList(VotesCapsule votesCapsule, List witne return exist.get(); } + private void processVoters(Config config) { + if (config.hasPath(VOTERS_ALL_WITNESSES)) { + allWitnessVoters = config.getBoolean(VOTERS_ALL_WITNESSES); + } + if (config.hasPath(VOTERS_ALL_ACTIVE_WITNESSES)) { + allActiveWitnesses = config.getBoolean(VOTERS_ALL_ACTIVE_WITNESSES); + } + if (config.hasPath(VOTERS_WITNESS_LIST)) { + votersWitnessList = config.getStringList(VOTERS_WITNESS_LIST); + } + + if (!allWitnessVoters && !allActiveWitnesses && votersWitnessList.isEmpty()) { + spec.commandLine().getOut() + .println("skip the voters query."); + logger.info("skip the voters query."); + return; + } + + if (config.hasPath(VOTERS_WITNESS_THRESHOLD)) { + threshold = config.getLong(VOTERS_WITNESS_THRESHOLD); + } + spec.commandLine().getOut() + .println("start voters query."); + logger.info("start the voters query."); + long start = System.currentTimeMillis(); + Map> votersMap = new HashMap<>(); + DBIterator iterator = accountStore.iterator(); + spec.commandLine().getOut().format("start to query account store: ").println(); + logger.info("begin to query account store: "); + AccountCapsule accountCapsule; + long accountCnt = 0; + for (iterator.seekToFirst(); iterator.valid(); iterator.next()) { + accountCapsule = new AccountCapsule(iterator.getValue()); + ByteString owner = accountCapsule.getAddress(); + accountCapsule.getVotesList().forEach(vote -> { + votersMap.computeIfAbsent(vote.getVoteAddress(), k -> new HashMap<>()); + votersMap.get(vote.getVoteAddress()).put(owner, vote.getVoteCount()); + }); + + accountCnt++; + if (accountCnt % 10000000 == 0) { + spec.commandLine().getOut().format("processed %d accounts", accountCnt) + .println(); + logger.info("processed {} accounts", accountCnt); + } + } + + long end = System.currentTimeMillis(); + spec.commandLine().getOut() + .format("account db query finished, total time: %d ms.", end - start).println(); + logger.info("account db query finished, total time: {} ms.", end - start); + + Map witnessesMap = new HashMap<>(); + Map oldWitnessCnt = new HashMap<>(); + DBIterator witnessIterator = witnessStore.iterator(); + WitnessCapsule witnessCapsule; + for (witnessIterator.seekToFirst(); witnessIterator.valid(); witnessIterator.next()) { + witnessCapsule = new WitnessCapsule(witnessIterator.getValue()); + witnessesMap.put(ByteString.copyFrom(witnessIterator.getKey()), witnessCapsule); + oldWitnessCnt.put(witnessCapsule.getAddress(), witnessCapsule.getVoteCount()); + } + Map countWitness = countVote(); + countWitness.forEach((address, voteCount) -> { + WitnessCapsule witness = witnessesMap.get(address); + if (witness == null) { + throw new IllegalStateException("witness not exist"); + } + witness.setVoteCount(witness.getVoteCount() + voteCount); + witnessesMap.put(address, witness); + }); + + AtomicInteger cnt = new AtomicInteger(); + cnt.set(-1); + if (allWitnessVoters) { + votersMap.entrySet().stream() + .forEach(entry -> { + WitnessCapsule witnessTmp = witnessesMap.get(entry.getKey()); + if (witnessTmp == null) { + throw new IllegalStateException("witness not exist"); + } + cnt.getAndIncrement(); + String output = formatWitness(witnessTmp, oldWitnessCnt.get(entry.getKey())); + spec.commandLine().getOut().println(cnt.get() + " " + output); + + String witness = StringUtil.encode58Check(entry.getKey().toByteArray()); + spec.commandLine().getOut() + .format("List the voters of witness: %s with votes great than %d: %s.", witness, + threshold).println(); + printVoters(entry.getValue(), threshold); + }); + } else if (allActiveWitnesses) { + byte[] witnesses = witnessScheduleStore.get(Constant.ACTIVE_WITNESSES); + List activeWitnesses = decodeActiveWitness(witnesses); + spec.commandLine().getOut().format("There are %d active witnesses in db", + activeWitnesses.size()).println(); + logger.info("There are {} active witnesses in db", activeWitnesses.size()); + activeWitnesses.forEach(witness -> { + WitnessCapsule witnessTmp = witnessesMap.get(witness); + if (witnessTmp == null) { + throw new IllegalStateException("witness not exist"); + } + cnt.getAndIncrement(); + String output = formatWitness(witnessTmp, oldWitnessCnt.get(witness)); + spec.commandLine().getOut().println(cnt.get() + " " + output); + + String witnessBase58 = StringUtil.encode58Check(witness.toByteArray()); + spec.commandLine().getOut() + .format("List the voters of active witness: %s with votes great than %d.", + witnessBase58, threshold).println(); + logger.info("List the voters of active witness: {} with votes great than {}.", + witnessBase58, threshold); + printVoters(votersMap.get(witness), threshold); + }); + } else { + votersWitnessList.forEach(witness -> { + ByteString address = ByteString.copyFrom(Commons.decode58Check(witness)); + if (!witnessesMap.containsKey(address)) { + spec.commandLine().getErr().format("address: %s is not a witness", witness).println(); + logger.error("address: {} is not a witness", witness); + return; + } + + cnt.getAndIncrement(); + String output = formatWitness(witnessesMap.get(address), oldWitnessCnt.get(address)); + spec.commandLine().getOut().println(cnt.get() + " " + output); + + spec.commandLine().getOut() + .format("List the voters of witness: %s with votes great than %d.", witness, + threshold).println(); + logger.info("List the voters of witness: {} with votes great than {}.", witness, threshold); + Map votesMap = votersMap.get(address); + printVoters(votesMap, threshold); + }); + } + + spec.commandLine().getOut().println("End of voters query."); + logger.info("End of voters query."); + try { + iterator.close(); + witnessIterator.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private void printVoters(Map votersMap, long threshold) { + if (votersMap == null || votersMap.isEmpty()) { + return; + } + votersMap.entrySet().stream() + .filter(entry -> entry.getValue() >= threshold) + .sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())) + .forEach(entry -> { + String witness = StringUtil.encode58Check(entry.getKey().toByteArray()); + long voteCount = entry.getValue(); + spec.commandLine().getOut() + .format("voterAddress: %s, vote count: %d", witness, voteCount).println(); + logger.info("voterAddress: {}, vote count: {}", witness, voteCount); + }); + } + + private void processRewards(Config config) { if (config.hasPath(REWARDS_KEY)) { rewardAddressList = config.getStringList(REWARDS_KEY); @@ -699,4 +882,18 @@ private long getOldReward(long begin, long end, List> votes) return reward; } + public static List decodeActiveWitness(byte[] witnesses) { + if (ByteArray.isEmpty(witnesses) + || witnesses.length % Constant.ADDRESS_BYTE_ARRAY_LENGTH != 0) { + throw new IllegalArgumentException("witnesses is invalid"); + } + + List witnessList = new ArrayList<>(); + for (int i = 0; i < witnesses.length; i += Constant.ADDRESS_BYTE_ARRAY_LENGTH) { + witnessList.add(ByteString.copyFrom( + Arrays.copyOfRange(witnesses, i, i + Constant.ADDRESS_BYTE_ARRAY_LENGTH))); + } + return witnessList; + } + } diff --git a/tools/toolkit/src/main/java/org/tron/plugins/DbRoot.java b/tools/toolkit/src/main/java/common/org/tron/plugins/DbRoot.java similarity index 84% rename from tools/toolkit/src/main/java/org/tron/plugins/DbRoot.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/DbRoot.java index 7c33219e..45854bbe 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/DbRoot.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/DbRoot.java @@ -67,16 +67,24 @@ public Integer call() throws Exception { .errorText("Specify at least one exit database: --db dbName.")); return 404; } - List task = ProgressBar.wrap(dbs.stream(), "root task").parallel() - .map(this::calcMerkleRoot).collect(Collectors.toList()); - task.forEach(this::printInfo); - int code = (int) task.stream().filter(r -> r.code == 1).count(); - if (code > 0) { + try { + List task = ProgressBar.wrap(dbs.stream(), "root task").parallel() + .map(this::calcMerkleRoot).collect(Collectors.toList()); + task.forEach(this::printInfo); + int code = (int) task.stream().filter(r -> r.code == 1).count(); + if (code > 0) { + spec.commandLine().getErr().println(spec.commandLine().getColorScheme() + .errorText("There are some errors, please check toolkit.log for detail.")); + } + spec.commandLine().getOut().println("root task done."); + return code; + } catch (Exception e) { + logger.error("{}", e); spec.commandLine().getErr().println(spec.commandLine().getColorScheme() - .errorText("There are some errors, please check toolkit.log for detail.")); + .errorText(e.getMessage())); + spec.commandLine().usage(System.out); + return 1; } - spec.commandLine().getOut().println("root task done."); - return code; } private Ret calcMerkleRoot(String name) { diff --git a/tools/toolkit/src/main/java/org/tron/plugins/Toolkit.java b/tools/toolkit/src/main/java/common/org/tron/plugins/Toolkit.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/Toolkit.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/Toolkit.java diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/ByteArray.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/ByteArray.java similarity index 97% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/ByteArray.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/ByteArray.java index 3422c36c..2922d110 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/utils/ByteArray.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/ByteArray.java @@ -59,7 +59,7 @@ public static int compareUnsigned(byte[] a, byte[] b) { if (b == null) { return 1; } - int minLen = Math.min(a.length, b.length); + int minLen = StrictMath.min(a.length, b.length); for (int i = 0; i < minLen; ++i) { int aVal = a[i] & 0xFF; int bVal = b[i] & 0xFF; diff --git a/tools/toolkit/src/main/java/common/org/tron/plugins/utils/ChainParameters.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/ChainParameters.java new file mode 100644 index 00000000..f8050766 --- /dev/null +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/ChainParameters.java @@ -0,0 +1,86 @@ +package org.tron.plugins.utils; + +public class ChainParameters { + + public static final long LONG_VALUE = 100_000_000_000_000_000L; + public static final long MAX_SUPPLY = 100_000_000_000L; + + public static final String CHAIN_PARAMETERS = "chainParameters"; + public static final String LATEST_BLOCK_TIMESTAMP = "latestBlockHeaderTimestamp"; + public static final String MAINTENANCE_INTERVAL = "maintenanceTimeInterval"; + public static final String NEXT_MAINTENANCE_TIME = "nextMaintenanceTime"; + public static final String ACCOUNT_UPGRADE_COST = "accountUpgradeCost"; + public static final String CREATE_ACCOUNT_FEE = "createAccountFee"; + public static final String TRANSACTION_FEE = "transactionFee"; + public static final String ASSET_ISSUE_FEE = "assetIssueFee"; + public static final String WITNESS_PAY_PER_BLOCK = "witnessPayPerBlock"; + public static final String WITNESS_STAND_BY_ALLOWANCE = "witnessStandbyAllowance"; + public static final String CREATE_NEW_ACCOUNT_FEE_IN_SYSTEM_CONTRACT = + "createNewAccountFeeInSystemContract"; + public static final String CREATE_NEW_ACCOUNT_BANDWIDTH_RATE = "createNewAccountBandwidthRate"; + public static final String ALLOW_CREATION_OF_CONTRACTS = "allowCreationOfContracts"; + public static final String REMOVE_THE_POWER_OF_THE_GR = "removeThePowerOfTheGr"; + public static final String ENERGY_FEE = "energyFee"; + public static final String EXCHANGE_CREATE_FEE = "exchangeCreateFee"; + public static final String MAX_CPU_TIME_OF_ONE_TX = "maxCpuTimeOfOneTx"; + public static final String ALLOW_UPDATE_ACCOUNT_NAME = "allowUpdateAccountName"; + public static final String ALLOW_SAME_TOKEN_NAME = "allowSameTokenName"; + public static final String ALLOW_DELEGATE_RESOURCE = "allowDelegateResource"; + public static final String TOTAL_ENERGY_LIMIT = "totalEnergyLimit"; + public static final String ALLOW_TVM_TRANSFER_TRC10 = "allowTvmTransferTrc10"; + public static final String TOTAL_ENERGY_CURRENT_LIMIT = "totalEnergyCurrentLimit"; + public static final String ALLOW_MULTI_SIGN = "allowMultiSign"; + public static final String ALLOW_ADAPTIVE_ENERGY = "allowAdaptiveEnergy"; + public static final String TOTAL_ENERGY_TARGET_LIMIT = "totalEnergyTargetLimit"; + public static final String TOTAL_ENERGY_AVERAGE_USAGE = "totalEnergyAverageUsage"; + public static final String UPDATE_ACCOUNT_PERMISSION_FEE = "updateAccountPermissionFee"; + public static final String MULTI_SIGN_FEE = "multiSignFee"; + public static final String ALLOW_ACCOUNT_STATE_ROOT = "allowAccountStateRoot"; + public static final String ALLOW_PROTO_FILTER_NUM = "allowProtoFilterNum"; + public static final String ALLOW_TVM_CONSTANTINOPLE = "allowTvmConstantinople"; + public static final String ALLOW_TVM_SOLIDITY059 = "allowTvmSolidity059"; + public static final String ALLOW_TVM_ISTANBUL = "allowTvmIstanbul"; + public static final String ALLOW_SHIELDED_TRC20_TRANSACTION = "allowShieldedTRC20Transaction"; + public static final String FORBID_TRANSFER_TO_CONTRACT = "forbidTransferToContract"; + public static final String ADAPTIVE_RESOURCE_LIMIT_TARGET_RATIO = + "adaptiveResourceLimitTargetRatio"; + public static final String ADAPTIVE_RESOURCE_LIMIT_MULTIPLIER = "adaptiveResourceLimitMultiplier"; + public static final String CHANGE_DELEGATION = "changeDelegation"; + public static final String WITNESS127_PAY_PER_BLOCK = "witness127PayPerBlock"; + public static final String ALLOW_MARKET_TRANSACTION = "allowMarketTransaction"; + public static final String MARKET_SELL_FEE = "marketSellFee"; + public static final String MARKET_CANCEL_FEE = "marketCancelFee"; + public static final String ALLOW_PBFT = "allowPBFT"; + public static final String ALLOW_TRANSACTION_FEE_POOL = "allowTransactionFeePool"; + public static final String MAX_FEE_LIMIT = "maxFeeLimit"; + public static final String ALLOW_OPTIMIZE_BLACK_HOLE = "allowOptimizeBlackHole"; + public static final String ALLOW_NEW_RESOURCE_MODEL = "allowNewResourceModel"; + public static final String ALLOW_TVM_FREEZE = "allowTvmFreeze"; + public static final String ALLOW_TVM_VOTE = "allowTvmVote"; + public static final String ALLOW_TVM_LONDON = "allowTvmLondon"; + public static final String ALLOW_TVM_COMPATIBLE_EVM = "allowTvmCompatibleEvm"; + public static final String ALLOW_ACCOUNT_ASSET_OPTIMIZATION = "allowAccountAssetOptimization"; + public static final String FREE_NET_LIMIT = "freeNetLimit"; + public static final String TOTAL_NET_LIMIT = "totalNetLimit"; + public static final String ALLOW_HIGHER_LIMIT_FOR_MAX_CPU_TIME_OF_ONE_TX = + "allowHigherLimitForMaxCpuTimeOfOneTx"; + public static final String ALLOW_ASSET_OPTIMIZATION = "allowAssetOptimization"; + public static final String ALLOW_NEW_REWARD = "allowNewReward"; + public static final String MEMO_FEE = "memoFee"; + public static final String ALLOW_DELEGATE_OPTIMIZATION = "allowDelegateOptimization"; + public static final String UNFREEZE_DELAY_DAYS = "unfreezeDelayDays"; + public static final String ALLOW_OPTIMIZED_RETURN_VALUE_OF_CHAIN_ID = + "allowOptimizedReturnValueOfChainId"; + public static final String ALLOW_DYNAMIC_ENERGY = "allowDynamicEnergy"; + public static final String DYNAMIC_ENERGY_THRESHOLD = "dynamicEnergyThreshold"; + public static final String DYNAMIC_ENERGY_INCREASE_FACTOR = "dynamicEnergyIncreaseFactor"; + public static final String ALLOW_TVM_SHANGHAI = "allowTvmShangHai"; + public static final String ALLOW_CANCEL_ALL_UNFREEZE_V2 = "allowCancelAllUnfreezeV2"; + public static final String MAX_DELEGATE_LOCK_PERIOD = "maxDelegateLockPeriod"; + public static final String ALLOW_ENERGY_ADJUSTMENT = "allowEnergyAdjustment"; + public static final String MAX_CREATE_ACCOUNT_TX_SIZE = "maxCreateAccountTxSize"; + public static final String ALLOW_STRICT_MATH = "allowStrictMath"; + public static final String CONSENSUS_LOGIC_OPTIMIZATION = "consensusLogicOptimization"; + public static final String ALLOW_TVM_CANCUN = "allowTvmCancun"; + public static final String ALLOW_TVM_BLOB = "allowTvmBlob"; +} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/Constant.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/Constant.java similarity index 86% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/Constant.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/Constant.java index 7fcc380c..8eb61a7f 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/utils/Constant.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/Constant.java @@ -23,9 +23,6 @@ public class Constant { public static final String TRC20_ACCOUNT = "address"; public static final String TRC20_BALANCE = "balance"; - public static final String LATEST_BLOCK_TIMESTAMP = "latestBlockHeaderTimestamp"; - public static final String MAINTENANCE_INTERVAL = "maintenanceTimeInterval"; - public static final String NEXT_MAINTENANCE_TIME = "nextMaintenanceTime"; public static final int MAX_ACTIVE_WITNESS_NUM = 27; public static final String WITNESS_STORE = "witness"; @@ -47,12 +44,21 @@ public class Constant { public static final byte[] LATEST_BLOCK_HEADER_NUMBER = "latest_block_header_number".getBytes(); public static final byte[] MAINTENANCE_TIME = "NEXT_MAINTENANCE_TIME".getBytes(); public static final byte[] ACTIVE_WITNESSES = "active_witnesses".getBytes(); + public static final byte[] ALLOW_MULTI_SIGN = "ALLOW_MULTI_SIGN".getBytes(); + public static final byte[] ACTIVE_DEFAULT_OPERATIONS = "ACTIVE_DEFAULT_OPERATIONS".getBytes(); public static final int ADDRESS_BYTE_ARRAY_LENGTH = 21; public static final String VOTES_ALL_WITNESSES = "vote.allWitnesses"; public static final String VOTES_WITNESS_LIST = "vote.witnessList"; public static final int BLOCK_PRODUCED_INTERVAL = 3000; + public static final String VOTERS_ALL_WITNESSES = "voters.allWitnesses"; + public static final String VOTERS_ALL_ACTIVE_WITNESSES = "voters.allActiveWitnesses"; + + public static final String VOTERS_WITNESS_LIST = "voters.witnessList"; + public static final String VOTERS_WITNESS_THRESHOLD = "voters.threshold"; + + public static final String REWARDS_KEY = "reward"; public static final byte[] CHANGE_DELEGATION = "CHANGE_DELEGATION".getBytes(); public static final byte[] CURRENT_CYCLE_NUMBER = "CURRENT_CYCLE_NUMBER".getBytes(); @@ -60,4 +66,7 @@ public class Constant { "NEW_REWARD_ALGORITHM_EFFECTIVE_CYCLE".getBytes(); public static final byte[] ALLOW_OLD_REWARD_OPT = "ALLOW_OLD_REWARD_OPT".getBytes(); public static final BigInteger DECIMAL_OF_VI_REWARD = BigInteger.valueOf(10).pow(18); + + + } diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/CryptoUitls.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/CryptoUitls.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/CryptoUitls.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/CryptoUitls.java diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/DBUtils.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/DBUtils.java similarity index 70% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/DBUtils.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/DBUtils.java index 19547e2b..6eb097cb 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/utils/DBUtils.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/DBUtils.java @@ -14,10 +14,9 @@ import org.rocksdb.BloomFilter; import org.rocksdb.ComparatorOptions; import org.rocksdb.Options; -import org.rocksdb.RocksDB; -import org.rocksdb.RocksDBException; -import org.tron.plugins.comparator.MarketOrderPriceComparatorForLevelDB; -import org.tron.plugins.comparator.MarketOrderPriceComparatorForRockDB; +import org.tron.common.arch.Arch; +import org.tron.common.utils.MarketOrderPriceComparatorForLevelDB; +import org.tron.common.utils.MarketOrderPriceComparatorForRocksDB; import org.tron.protos.Protocol; public class DBUtils { @@ -65,6 +64,7 @@ static Operator valueOf(byte b) { public static final String ROCKSDB = "ROCKSDB"; public static DB newLevelDb(Path db) throws IOException { + Arch.throwIfUnsupportedArm64Exception(LEVELDB); File file = db.toFile(); org.iq80.leveldb.Options dbOptions = newDefaultLevelDbOptions(); if (MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(file.getName())) { @@ -86,7 +86,23 @@ public static org.iq80.leveldb.Options newDefaultLevelDbOptions() { return dbOptions; } - private static Options newDefaultRocksDbOptions(boolean forBulkLoad) { + /** + * Creates a new RocksDB Options. + * + *

CRITICAL: Must be closed after use to prevent native memory leaks. + * Use try-with-resources. + * + * 

{@code
+   * try (Options options = newDefaultRocksDbOptions(false, name)) {
+   *     // do something
+   * }
+   * }
+ * + * @param forBulkLoad if true, optimizes for bulk loading + * @param name db name + * @return a new Options instance that must be closed + */ + public static Options newDefaultRocksDbOptions(boolean forBulkLoad, String name) { Options options = new Options(); options.setCreateIfMissing(true); options.setIncreaseParallelism(1); @@ -95,7 +111,8 @@ private static Options newDefaultRocksDbOptions(boolean forBulkLoad) { options.setTargetFileSizeBase(64 * 1024 * 1024); options.setTargetFileSizeMultiplier(1); options.setMaxBytesForLevelBase(512 * 1024 * 1024); - options.setMaxBackgroundCompactions(Math.max(1, Runtime.getRuntime().availableProcessors())); + options.setMaxBackgroundCompactions(StrictMath.max( + 1, Runtime.getRuntime().availableProcessors())); options.setLevel0FileNumCompactionTrigger(4); options.setLevelCompactionDynamicLevelBytes(true); final BlockBasedTableConfig tableCfg; @@ -108,35 +125,10 @@ private static Options newDefaultRocksDbOptions(boolean forBulkLoad) { if (forBulkLoad) { options.prepareForBulkLoad(); } - return options; - } - - public static RocksDB newRocksDb(Path db) throws RocksDBException { - try (Options options = newDefaultRocksDbOptions(false)) { - if (MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(db.getFileName().toString())) { - options.setComparator(new MarketOrderPriceComparatorForRockDB(new ComparatorOptions())); - } - return RocksDB.open(options, db.toString()); - } - } - - public static RocksDB newRocksDbForBulkLoad(Path db) throws RocksDBException { - try (Options options = newDefaultRocksDbOptions(true)) { - if (MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(db.getFileName().toString())) { - options.setComparator(new MarketOrderPriceComparatorForRockDB(new ComparatorOptions())); - } - return RocksDB.open(options, db.toString()); - } - } - - - public static RocksDB newRocksDbReadOnly(Path db) throws RocksDBException { - try (Options options = newDefaultRocksDbOptions(false)) { - if (MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(db.getFileName().toString())) { - options.setComparator(new MarketOrderPriceComparatorForRockDB(new ComparatorOptions())); - } - return RocksDB.openReadOnly(options, db.toString()); + if (MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(name)) { + options.setComparator(new MarketOrderPriceComparatorForRocksDB(new ComparatorOptions())); } + return options; } public static String simpleDecode(byte[] bytes) { diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/FileUtils.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/FileUtils.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/FileUtils.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/FileUtils.java diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/HttpSelfFormatFieldName.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/HttpSelfFormatFieldName.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/HttpSelfFormatFieldName.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/HttpSelfFormatFieldName.java diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/JsonFormat.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/JsonFormat.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/JsonFormat.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/JsonFormat.java diff --git a/tools/toolkit/src/main/java/common/org/tron/plugins/utils/MarketUtils.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/MarketUtils.java new file mode 100644 index 00000000..9bcfd5e7 --- /dev/null +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/MarketUtils.java @@ -0,0 +1,68 @@ +package org.tron.plugins.utils; + +public class MarketUtils { + + public static final int TOKEN_ID_LENGTH = ByteArray + .fromString(Long.toString(Long.MAX_VALUE)).length; // 19 + + + + /** + * In order to avoid the difference between the data of same key stored and fetched by hashMap and + * levelDB, when creating the price key, we will find the GCD (Greatest Common Divisor) of + * sellTokenQuantity and buyTokenQuantity. + */ + public static byte[] createPairPriceKey(byte[] sellTokenId, byte[] buyTokenId, + long sellTokenQuantity, long buyTokenQuantity) { + + byte[] sellTokenQuantityBytes; + byte[] buyTokenQuantityBytes; + + // cal the GCD + long gcd = findGCD(sellTokenQuantity, buyTokenQuantity); + if (gcd == 0) { + sellTokenQuantityBytes = ByteArray.fromLong(sellTokenQuantity); + buyTokenQuantityBytes = ByteArray.fromLong(buyTokenQuantity); + } else { + sellTokenQuantityBytes = ByteArray.fromLong(sellTokenQuantity / gcd); + buyTokenQuantityBytes = ByteArray.fromLong(buyTokenQuantity / gcd); + } + + return doCreatePairPriceKey(sellTokenId, buyTokenId, + sellTokenQuantityBytes, buyTokenQuantityBytes); + } + + public static long findGCD(long number1, long number2) { + if (number1 == 0 || number2 == 0) { + return 0; + } + return calGCD(number1, number2); + } + + private static long calGCD(long number1, long number2) { + if (number2 == 0) { + return number1; + } + return calGCD(number2, number1 % number2); + } + + + private static byte[] doCreatePairPriceKey(byte[] sellTokenId, byte[] buyTokenId, + byte[] sellTokenQuantity, byte[] buyTokenQuantity) { + byte[] result = new byte[TOKEN_ID_LENGTH + TOKEN_ID_LENGTH + + sellTokenQuantity.length + buyTokenQuantity.length]; + + System.arraycopy(sellTokenId, 0, result, 0, sellTokenId.length); + System.arraycopy(buyTokenId, 0, result, TOKEN_ID_LENGTH, buyTokenId.length); + System.arraycopy(sellTokenQuantity, 0, result, + TOKEN_ID_LENGTH + TOKEN_ID_LENGTH, + sellTokenQuantity.length); + System.arraycopy(buyTokenQuantity, 0, result, + TOKEN_ID_LENGTH + TOKEN_ID_LENGTH + buyTokenQuantity.length, + buyTokenQuantity.length); + + return result; + } + + +} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/MerkleRoot.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/MerkleRoot.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/MerkleRoot.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/MerkleRoot.java diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/Sha256Hash.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/Sha256Hash.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/Sha256Hash.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/Sha256Hash.java diff --git a/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/DBInterface.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/DBInterface.java new file mode 100644 index 00000000..13a195f9 --- /dev/null +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/DBInterface.java @@ -0,0 +1,39 @@ +package org.tron.plugins.utils.db; + +import java.io.Closeable; +import java.io.IOException; + + +public interface DBInterface extends Closeable { + + byte[] get(byte[] key); + + void put(byte[] key, byte[] value); + + void delete(byte[] key); + + /** + * Returns an iterator over the database. + * + *

CRITICAL: The returned iterator holds native resources and MUST be closed + * after use to prevent memory leaks. It is strongly recommended to use a try-with-resources + * statement. + * + *

Example of correct usage: + * 

{@code
+   * try (DBIterator iterator = db.iterator()) {
+   *  // do something
+   * }
+   * }
+ * + * @return a new database iterator that must be closed. + */ + DBIterator iterator(); + + long size() throws IOException; + + void close() throws IOException; + + String getName(); + +} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/DBIterator.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/DBIterator.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/db/DBIterator.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/DBIterator.java diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/DbTool.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/DbTool.java similarity index 83% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/db/DbTool.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/DbTool.java index 429025e8..d9962394 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/DbTool.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/DbTool.java @@ -9,6 +9,7 @@ import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.rocksdb.RocksDBException; +import org.tron.common.arch.Arch; import org.tron.plugins.utils.DBUtils; import org.tron.plugins.utils.FileUtils; @@ -18,8 +19,8 @@ public class DbTool { private static final String KEY_ENGINE = "ENGINE"; private static final String ENGINE_FILE = "engine.properties"; - private static final String FILE_SEPARATOR = File.separator; private static final String ROCKSDB = "ROCKSDB"; + private static final String LEVELDB = "LEVELDB"; private static final Map dbMap = Maps.newConcurrentMap(); @@ -162,10 +163,10 @@ public static void close() { } private static DbType getDbType(String sourceDir, String dbName) { - String engineFile = String.format("%s%s%s%s%s", sourceDir, FILE_SEPARATOR, - dbName, FILE_SEPARATOR, ENGINE_FILE); + String engineFile = Paths.get(sourceDir, dbName, ENGINE_FILE).toString(); if (!new File(engineFile).exists()) { - return DbType.LevelDB; + // For arm64, only support RocksDB + return Arch.isArm64() ? DbType.RocksDB : DbType.LevelDB; } String engine = FileUtils.readProperty(engineFile, KEY_ENGINE); if (engine.equalsIgnoreCase(ROCKSDB)) { @@ -175,13 +176,22 @@ private static DbType getDbType(String sourceDir, String dbName) { } } - private static LevelDBImpl openLevelDb(Path db, String name) throws IOException { - return new LevelDBImpl(DBUtils.newLevelDb(db), name); + public static LevelDBImpl openLevelDb(Path db, String name) throws IOException { + LevelDBImpl leveldb = new LevelDBImpl(DBUtils.newLevelDb(db), name); + tryInitEngineFile(db, LEVELDB); + return leveldb; } - private static RocksDBImpl openRocksDb(Path db, String name) throws RocksDBException { - return new RocksDBImpl(DBUtils.newRocksDb(db), name); + public static RocksDBImpl openRocksDb(Path db, String name) throws RocksDBException { + RocksDBImpl rocksdb = new RocksDBImpl(db, name); + tryInitEngineFile(db, ROCKSDB); + return rocksdb; } - + private static void tryInitEngineFile(Path db, String engine) { + String engineFile = Paths.get(db.toString(), ENGINE_FILE).toString(); + if (FileUtils.createFileIfNotExists(engineFile)) { + FileUtils.writeProperty(engineFile, KEY_ENGINE, engine); + } + } } diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/LevelDBImpl.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/LevelDBImpl.java similarity index 83% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/db/LevelDBImpl.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/LevelDBImpl.java index 511f4dfd..1c7f22ef 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/LevelDBImpl.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/LevelDBImpl.java @@ -40,8 +40,11 @@ public DBIterator iterator() { } @Override - public long size() { - return Streams.stream(leveldb.iterator()).count(); + public long size() throws IOException { + try (DBIterator iterator = this.iterator()) { + iterator.seekToFirst(); + return Streams.stream(iterator).count(); + } } @Override diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/LevelDBIterator.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/LevelDBIterator.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/db/LevelDBIterator.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/LevelDBIterator.java diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/RockDBIterator.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/RockDBIterator.java similarity index 76% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/db/RockDBIterator.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/RockDBIterator.java index d3e17d91..17ecca4a 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/RockDBIterator.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/RockDBIterator.java @@ -2,14 +2,19 @@ import java.io.IOException; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import org.rocksdb.ReadOptions; import org.rocksdb.RocksIterator; public class RockDBIterator implements DBIterator { private final RocksIterator iterator; + private final ReadOptions readOptions; + private final AtomicBoolean closed = new AtomicBoolean(false); - public RockDBIterator(RocksIterator iterator) { + public RockDBIterator(RocksIterator iterator, ReadOptions readOptions) { this.iterator = iterator; + this.readOptions = readOptions; } @Override @@ -72,6 +77,9 @@ public byte[] setValue(byte[] value) { @Override public void close() throws IOException { - iterator.close(); + if (closed.compareAndSet(false, true)) { + readOptions.close(); + iterator.close(); + } } } diff --git a/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/RocksDBImpl.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/RocksDBImpl.java new file mode 100644 index 00000000..236d0a84 --- /dev/null +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/RocksDBImpl.java @@ -0,0 +1,97 @@ +package org.tron.plugins.utils.db; + +import com.google.common.collect.Streams; +import java.io.IOException; +import java.nio.file.Path; +import java.util.concurrent.atomic.AtomicBoolean; +import lombok.Getter; +import org.rocksdb.Options; +import org.rocksdb.ReadOptions; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.tron.plugins.utils.DBUtils; + +public class RocksDBImpl implements DBInterface { + + private final RocksDB rocksDB; + + @Getter + private final String name; + private final AtomicBoolean closed = new AtomicBoolean(false); + private Options options = null; + + public RocksDBImpl(Path path, String name) throws RocksDBException { + try { + this.options = DBUtils.newDefaultRocksDbOptions(false, name); + this.name = name; + this.rocksDB = RocksDB.open(options, path.toString()); + } catch (RocksDBException e) { + if (this.options != null) { + this.options.close(); + } + throw e; + } + } + + @Override + public byte[] get(byte[] key) { + throwIfClosed(); + try { + return rocksDB.get(key); + } catch (RocksDBException e) { + throw new RuntimeException(name, e); + } + } + + @Override + public void put(byte[] key, byte[] value) { + throwIfClosed(); + try { + rocksDB.put(key, value); + } catch (RocksDBException e) { + throw new RuntimeException(name, e); + } + } + + @Override + public void delete(byte[] key) { + throwIfClosed(); + try { + rocksDB.delete(key); + } catch (RocksDBException e) { + throw new RuntimeException(name, e); + } + } + + @Override + public DBIterator iterator() { + throwIfClosed(); + ReadOptions readOptions = new ReadOptions().setFillCache(false); + return new RockDBIterator(rocksDB.newIterator(readOptions), readOptions); + } + + @Override + public long size() throws IOException { + throwIfClosed(); + try (DBIterator iterator = this.iterator()) { + iterator.seekToFirst(); + return Streams.stream(iterator).count(); + } + } + + @Override + public void close() throws IOException { + if (closed.compareAndSet(false, true)) { + if (this.options != null) { + this.options.close(); + } + rocksDB.close(); + } + } + + private void throwIfClosed() { + if (closed.get()) { + throw new IllegalStateException("db " + name + " has been closed"); + } + } +} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/DbFork.java b/tools/toolkit/src/main/java/org/tron/plugins/DbFork.java deleted file mode 100644 index 0e6919f7..00000000 --- a/tools/toolkit/src/main/java/org/tron/plugins/DbFork.java +++ /dev/null @@ -1,419 +0,0 @@ -package org.tron.plugins; - -import static java.lang.System.arraycopy; -import static org.tron.plugins.utils.Constant.ACCOUNTS_KEY; -import static org.tron.plugins.utils.Constant.ACCOUNT_ADDRESS; -import static org.tron.plugins.utils.Constant.ACCOUNT_ASSET; -import static org.tron.plugins.utils.Constant.ACCOUNT_BALANCE; -import static org.tron.plugins.utils.Constant.ACCOUNT_NAME; -import static org.tron.plugins.utils.Constant.ACCOUNT_OWNER; -import static org.tron.plugins.utils.Constant.ACCOUNT_STORE; -import static org.tron.plugins.utils.Constant.ACCOUNT_TRC10_BALANCE; -import static org.tron.plugins.utils.Constant.ACCOUNT_TRC10_ID; -import static org.tron.plugins.utils.Constant.ACCOUNT_TYPE; -import static org.tron.plugins.utils.Constant.ACTIVE_WITNESSES; -import static org.tron.plugins.utils.Constant.ASSET_ISSUE_V2; -import static org.tron.plugins.utils.Constant.CONTRACT_STORE; -import static org.tron.plugins.utils.Constant.DYNAMIC_PROPERTY_STORE; -import static org.tron.plugins.utils.Constant.LATEST_BLOCK_HEADER_TIMESTAMP; -import static org.tron.plugins.utils.Constant.LATEST_BLOCK_TIMESTAMP; -import static org.tron.plugins.utils.Constant.MAINTENANCE_INTERVAL; -import static org.tron.plugins.utils.Constant.MAINTENANCE_TIME; -import static org.tron.plugins.utils.Constant.MAINTENANCE_TIME_INTERVAL; -import static org.tron.plugins.utils.Constant.MAX_ACTIVE_WITNESS_NUM; -import static org.tron.plugins.utils.Constant.NEXT_MAINTENANCE_TIME; -import static org.tron.plugins.utils.Constant.STORAGE_ROW_STORE; -import static org.tron.plugins.utils.Constant.TRC20_ACCOUNT; -import static org.tron.plugins.utils.Constant.TRC20_BALANCE; -import static org.tron.plugins.utils.Constant.TRC20_BALANCES_POSITION; -import static org.tron.plugins.utils.Constant.TRC20_CONTRACTS_KEY; -import static org.tron.plugins.utils.Constant.TRC20_CONTRACT_ADDRESS; -import static org.tron.plugins.utils.Constant.WITNESS_ADDRESS; -import static org.tron.plugins.utils.Constant.WITNESS_KEY; -import static org.tron.plugins.utils.Constant.WITNESS_SCHEDULE_STORE; -import static org.tron.plugins.utils.Constant.WITNESS_STORE; -import static org.tron.plugins.utils.Constant.WITNESS_URL; -import static org.tron.plugins.utils.Constant.WITNESS_VOTE; - -import com.google.common.primitives.Bytes; -import com.google.common.primitives.Longs; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; -import com.typesafe.config.Config; -import com.typesafe.config.ConfigFactory; -import java.io.File; -import java.io.IOException; -import java.math.BigInteger; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.Callable; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.ArrayUtils; -import org.rocksdb.RocksDBException; -import org.tron.common.crypto.Hash; -import org.tron.common.utils.ByteArray; -import org.tron.common.utils.ByteUtil; -import org.tron.common.utils.Commons; -import org.tron.core.capsule.AccountCapsule; -import org.tron.core.capsule.StorageRowCapsule; -import org.tron.core.capsule.WitnessCapsule; -import org.tron.plugins.utils.Constant; -import org.tron.plugins.utils.FileUtils; -import org.tron.plugins.utils.db.DBInterface; -import org.tron.plugins.utils.db.DBIterator; -import org.tron.plugins.utils.db.DbTool; -import org.tron.protos.Protocol.Account; -import org.tron.protos.Protocol.AccountType; -import org.tron.protos.Protocol.Permission; -import org.tron.protos.contract.SmartContractOuterClass.SmartContract; -import picocli.CommandLine; -import picocli.CommandLine.Command; - -@Slf4j(topic = "fork") -@Command(name = "fork", - description = "Modify the database of java-tron for shadow fork testing.", - exitCodeListHeading = "Exit Codes:%n", - exitCodeList = { - "0:Successful", - "n:Internal error: exception occurred, please check logs/toolkit.log"}) -public class DbFork implements Callable { - - private DBInterface witnessStore; - private DBInterface witnessScheduleStore; - private DBInterface accountStore; - private DBInterface dynamicPropertiesStore; - private DBInterface accountAssetStore; - private DBInterface assetIssueV2Store; - private DBInterface contractStore; - private DBInterface storageRowStore; - - @CommandLine.Spec - CommandLine.Model.CommandSpec spec; - - @CommandLine.Option(names = {"-d", "--database-directory"}, - defaultValue = "output-directory", - description = "java-tron database directory path. Default: ${DEFAULT-VALUE}") - private String database; - - @CommandLine.Option(names = {"-c", "--config"}, - defaultValue = "fork.conf", - description = "config the new witnesses, balances, etc for shadow fork." - + " Default: ${DEFAULT-VALUE}") - private String config; - - @CommandLine.Option(names = {"-r", "--retain-witnesses"}, - description = "retain the previous witnesses and active witnesses.") - private boolean retain; - - @CommandLine.Option(names = {"-h", "--help"}) - private boolean help; - - private void initStore() throws IOException, RocksDBException { - String srcDir = database + File.separator + "database"; - witnessStore = DbTool.getDB(srcDir, WITNESS_STORE); - witnessScheduleStore = DbTool.getDB(srcDir, WITNESS_SCHEDULE_STORE); - accountStore = DbTool.getDB(srcDir, ACCOUNT_STORE); - dynamicPropertiesStore = DbTool.getDB(srcDir, DYNAMIC_PROPERTY_STORE); - accountAssetStore = DbTool.getDB(srcDir, ACCOUNT_ASSET); - assetIssueV2Store = DbTool.getDB(srcDir, ASSET_ISSUE_V2); - contractStore = DbTool.getDB(srcDir, CONTRACT_STORE); - storageRowStore = DbTool.getDB(srcDir, STORAGE_ROW_STORE); - } - - @Override - public Integer call() throws IOException, RocksDBException { - if (help) { - spec.commandLine().usage(System.out); - return 0; - } - - File dbFile = Paths.get(database).toFile(); - if (!dbFile.exists() || !dbFile.isDirectory()) { - logger.error("Database [" + database + "] not exists!"); - spec.commandLine().getErr().format("Database %s not exists!", database).println(); - System.exit(-1); - } - File tmp = Paths.get(database, "database", "tmp").toFile(); - if (tmp.exists()) { - FileUtils.deleteDir(tmp); - } - - Config forkConfig = ConfigFactory.load(); - File file = Paths.get(config).toFile(); - if (file.exists() && file.isFile()) { - forkConfig = ConfigFactory.parseFile(file); - } else { - logger.error("Fork config file [" + config + "] not exists!"); - spec.commandLine().getErr().format("Fork config file: %s not exists!", config).println(); - System.exit(-1); - } - - initStore(); - - if (!retain) { - logger.info("Erase the previous witnesses and active witnesses."); - spec.commandLine().getOut().println("Erase the previous witnesses and active witnesses."); - witnessScheduleStore.delete(ACTIVE_WITNESSES); - DBIterator iterator = witnessStore.iterator(); - for (iterator.seekToFirst(); iterator.valid(); iterator.next()) { - witnessStore.delete(iterator.getKey()); - } - } else { - logger.warn("Keep the previous witnesses and active witnesses."); - spec.commandLine().getOut().println("Keep the previous witnesses and active witnesses."); - } - - if (forkConfig.hasPath(WITNESS_KEY)) { - List witnesses = forkConfig.getConfigList(WITNESS_KEY); - if (witnesses.isEmpty()) { - spec.commandLine().getOut().println("no witness listed in the config."); - } - witnesses = witnesses.stream() - .filter(c -> c.hasPath(WITNESS_ADDRESS)) - .collect(Collectors.toList()); - - if (witnesses.isEmpty()) { - spec.commandLine().getOut().println("no witness listed in the config."); - } - - List witnessList = new ArrayList<>(); - witnesses.stream().forEach( - w -> { - ByteString address = ByteString.copyFrom( - Commons.decodeFromBase58Check(w.getString(WITNESS_ADDRESS))); - WitnessCapsule witness = new WitnessCapsule(address); - witness.setIsJobs(true); - if (w.hasPath(WITNESS_VOTE) && w.getLong(WITNESS_VOTE) > 0) { - witness.setVoteCount(w.getLong(WITNESS_VOTE)); - } - if (w.hasPath(WITNESS_URL)) { - witness.setUrl(w.getString(WITNESS_URL)); - } - witnessStore.put(address.toByteArray(), witness.getData()); - witnessList.add(witness.getAddress()); - }); - - witnessList.sort(Comparator.comparingLong((ByteString b) -> - new WitnessCapsule(witnessStore.get(b.toByteArray())).getVoteCount()) - .reversed() - .thenComparing(Comparator.comparingInt(ByteString::hashCode).reversed())); - List activeWitnesses = witnessList.subList(0, - witnesses.size() >= MAX_ACTIVE_WITNESS_NUM ? MAX_ACTIVE_WITNESS_NUM : witnessList.size()); - witnessScheduleStore.put(ACTIVE_WITNESSES, getActiveWitness(activeWitnesses)); - logger.info("{} witnesses and {} active witnesses have been modified.", - witnesses.size(), activeWitnesses.size()); - spec.commandLine().getOut().format("%d witnesses and %d active witnesses have been modified.", - witnesses.size(), activeWitnesses.size()).println(); - } - - if (forkConfig.hasPath(ACCOUNTS_KEY)) { - List accounts = forkConfig.getConfigList(ACCOUNTS_KEY); - if (accounts.isEmpty()) { - spec.commandLine().getOut().println("no account listed in the config."); - } - - accounts = accounts.stream() - .filter(c -> c.hasPath(ACCOUNT_ADDRESS)) - .collect(Collectors.toList()); - - if (accounts.isEmpty()) { - spec.commandLine().getOut().println("no account listed in the config."); - } - - accounts.stream().forEach( - a -> { - byte[] address = Commons.decodeFromBase58Check(a.getString(ACCOUNT_ADDRESS)); - byte[] value = accountStore.get(address); - Account account = null; - try { - account = ArrayUtils.isEmpty(value) ? null : Account.parseFrom(value); - } catch (InvalidProtocolBufferException e) { - e.printStackTrace(); - System.exit(-1); - } - - if (Objects.isNull(account)) { - ByteString byteAddress = ByteString.copyFrom( - Commons.decodeFromBase58Check(a.getString(ACCOUNT_ADDRESS))); - account = Account.newBuilder().setAddress(byteAddress).build(); - } - AccountCapsule accountCapsule = new AccountCapsule(account); - - if (a.hasPath(ACCOUNT_BALANCE) && a.getLong(ACCOUNT_BALANCE) > 0) { - accountCapsule.setBalance(a.getLong(ACCOUNT_BALANCE)); - } - if (a.hasPath(ACCOUNT_NAME)) { - accountCapsule.setAccountName( - ByteArray.fromString(a.getString(ACCOUNT_NAME))); - } - if (a.hasPath(ACCOUNT_TYPE)) { - accountCapsule.updateAccountType( - AccountType.valueOf(a.getString(ACCOUNT_TYPE))); - } - - if (a.hasPath(ACCOUNT_OWNER)) { - byte[] owner = Commons.decodeFromBase58Check(a.getString(ACCOUNT_OWNER)); - Permission ownerPermission = AccountCapsule - .createDefaultOwnerPermission(ByteString.copyFrom(owner)); - accountCapsule.updatePermissions(ownerPermission, null, null); - } - - if (a.hasPath(ACCOUNT_TRC10_ID) && a.hasPath(ACCOUNT_TRC10_BALANCE) - && a.getLong(ACCOUNT_TRC10_BALANCE) > 0) { - String trc10Id = a.getString(ACCOUNT_TRC10_ID); - if (assetIssueV2Store.get(ByteArray.fromString(trc10Id)) != null) { - if (accountCapsule.getAssetOptimized()) { - byte[] k = Bytes.concat(address, ByteArray.fromString(trc10Id)); - accountAssetStore.put(k, Longs.toByteArray(a.getLong(ACCOUNT_TRC10_BALANCE))); - } else { - Map assetMapV2 = new HashMap<>(account.getAssetV2Map()); - assetMapV2.put(trc10Id, a.getLong(ACCOUNT_TRC10_BALANCE)); - accountCapsule.clearAssetV2(); - accountCapsule.addAssetMapV2(assetMapV2); - } - } else { - logger.info("TRC10: {} not exists in the database.", trc10Id); - spec.commandLine().getOut().format("TRC10: %s not exists in the database.", trc10Id) - .println(); - } - } - - accountStore.put(address, accountCapsule.getData()); - }); - logger.info("{} accounts have been modified.", accounts.size()); - spec.commandLine().getOut().format("%d accounts have been modified.", accounts.size()) - .println(); - } - - if (forkConfig.hasPath(TRC20_CONTRACTS_KEY)) { - List trc20Contracts = forkConfig.getConfigList(TRC20_CONTRACTS_KEY); - if (trc20Contracts.isEmpty()) { - spec.commandLine().getOut().println("no TRC20 contract listed in the config."); - } - - trc20Contracts = trc20Contracts.stream() - .filter(c -> c.hasPath(TRC20_CONTRACT_ADDRESS) && c.hasPath(TRC20_BALANCES_POSITION) && - c.hasPath(TRC20_ACCOUNT) && c.hasPath(TRC20_BALANCE)) - .collect(Collectors.toList()); - - if (trc20Contracts.isEmpty()) { - spec.commandLine().getOut().println("no TRC20 contract listed in the config."); - } - - AtomicInteger cnt = new AtomicInteger(); - - trc20Contracts.stream().forEach( - contract -> { - byte[] contractAddress = Commons - .decodeFromBase58Check(contract.getString(TRC20_CONTRACT_ADDRESS)); - if (contractStore.get(contractAddress) == null) { - spec.commandLine().getErr().format("TRC20 contract: %s not exists in the database.", - contract.getString(TRC20_CONTRACT_ADDRESS)).println(); - return; - } - - SmartContract smartContract; - try { - smartContract = SmartContract.parseFrom(contractStore.get(contractAddress)); - } catch (InvalidProtocolBufferException e) { - e.printStackTrace(); - return; - } - - int balancesSlotPosition = 0; - if (contract.getInt(TRC20_BALANCES_POSITION) > 0) { - balancesSlotPosition = contract.getInt(TRC20_BALANCES_POSITION); - } - byte[] addressWithPrefix = Commons - .decodeFromBase58Check(contract.getString(TRC20_ACCOUNT)); - byte[] address = ByteArray.subArray(addressWithPrefix, 1, 21); - String paddedAddress = String - .format("%064x", new BigInteger(ByteArray.toHexString(address), 16)); - String paddedSlot = String.format("%064x", balancesSlotPosition); - byte[] contractKey = Hash.sha3(ByteArray.fromHexString( - paddedAddress + paddedSlot)); - - byte[] addressHash; - byte[] trxHash = smartContract.getTrxHash().toByteArray(); - if (ByteUtil.isNullOrZeroArray(trxHash)) { - addressHash = Hash.sha3(contractAddress); - } else { - addressHash = Hash.sha3(ByteUtil.merge(contractAddress, trxHash)); - } - - int contractVersion = smartContract.getVersion(); - if (contractVersion == 1) { - contractKey = Hash.sha3(contractKey); - } - byte[] rowKey = new byte[contractKey.length]; - arraycopy(addressHash, 0, rowKey, 0, 16); - arraycopy(contractKey, 16, rowKey, 16, 16); - - String paddedBalance = String - .format("%064x", new BigInteger(contract.getString(TRC20_BALANCE), 10)); - byte[] rowValue = ByteArray.fromHexString(paddedBalance); - StorageRowCapsule storageRowCapsule = new StorageRowCapsule(rowKey, rowValue); - - storageRowStore.put(rowKey, storageRowCapsule.getData()); - cnt.getAndIncrement(); - }); - logger.info("{} TRC20 contracts have been modified.", cnt.get()); - spec.commandLine().getOut() - .format("%d TRC20 contracts have been modified.", cnt.get()) - .println(); - } - - if (forkConfig.hasPath(LATEST_BLOCK_TIMESTAMP) - && forkConfig.getLong(LATEST_BLOCK_TIMESTAMP) > 0) { - long latestBlockHeaderTimestamp = forkConfig.getLong(LATEST_BLOCK_TIMESTAMP); - dynamicPropertiesStore - .put(LATEST_BLOCK_HEADER_TIMESTAMP, ByteArray.fromLong(latestBlockHeaderTimestamp)); - logger.info("The latest block header timestamp has been modified as {}.", - latestBlockHeaderTimestamp); - spec.commandLine().getOut().format("The latest block header timestamp has been modified " - + "as %d.", latestBlockHeaderTimestamp).println(); - } - - if (forkConfig.hasPath(MAINTENANCE_INTERVAL) - && forkConfig.getLong(MAINTENANCE_INTERVAL) > 0) { - long maintenanceTimeInterval = forkConfig.getLong(MAINTENANCE_INTERVAL); - dynamicPropertiesStore - .put(MAINTENANCE_TIME_INTERVAL, ByteArray.fromLong(maintenanceTimeInterval)); - logger.info("The maintenance time interval has been modified as {}.", - maintenanceTimeInterval); - spec.commandLine().getOut().format("The maintenance time interval has been modified as %d.", - maintenanceTimeInterval).println(); - } - - if (forkConfig.hasPath(NEXT_MAINTENANCE_TIME) - && forkConfig.getLong(NEXT_MAINTENANCE_TIME) > 0) { - long nextMaintenanceTime = forkConfig.getLong(NEXT_MAINTENANCE_TIME); - dynamicPropertiesStore.put(MAINTENANCE_TIME, ByteArray.fromLong(nextMaintenanceTime)); - logger.info("The next maintenance time has been modified as {}.", - nextMaintenanceTime); - spec.commandLine().getOut().format("The next maintenance time has been modified as %d.", - nextMaintenanceTime).println(); - } - - DbTool.close(); - return 0; - } - - public static byte[] getActiveWitness(List witnesses) { - byte[] ba = new byte[witnesses.size() * Constant.ADDRESS_BYTE_ARRAY_LENGTH]; - int i = 0; - for (ByteString address : witnesses) { - System.arraycopy(address.toByteArray(), 0, - ba, i * Constant.ADDRESS_BYTE_ARRAY_LENGTH, Constant.ADDRESS_BYTE_ARRAY_LENGTH); - i++; - } - return ba; - } -} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForLevelDB.java b/tools/toolkit/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForLevelDB.java deleted file mode 100644 index 9cb12b49..00000000 --- a/tools/toolkit/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForLevelDB.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.tron.plugins.comparator; - -import org.iq80.leveldb.DBComparator; -import org.tron.plugins.utils.MarketUtils; - -public class MarketOrderPriceComparatorForLevelDB implements DBComparator { - - @Override - public String name() { - return "MarketOrderPriceComparator"; - } - - @Override - public byte[] findShortestSeparator(byte[] start, byte[] limit) { - return new byte[0]; - } - - @Override - public byte[] findShortSuccessor(byte[] key) { - return new byte[0]; - } - - @Override - public int compare(byte[] o1, byte[] o2) { - return MarketUtils.comparePriceKey(o1, o2); - } - -} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForRockDB.java b/tools/toolkit/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForRockDB.java deleted file mode 100644 index 40d65516..00000000 --- a/tools/toolkit/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForRockDB.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.tron.plugins.comparator; - -import org.rocksdb.ComparatorOptions; -import org.rocksdb.DirectSlice; -import org.rocksdb.util.DirectBytewiseComparator; -import org.tron.plugins.utils.MarketUtils; - -public class MarketOrderPriceComparatorForRockDB extends DirectBytewiseComparator { - - public MarketOrderPriceComparatorForRockDB(final ComparatorOptions copt) { - super(copt); - } - - @Override - public String name() { - return "MarketOrderPriceComparator"; - } - - @Override - public int compare(final DirectSlice a, final DirectSlice b) { - return MarketUtils.comparePriceKey(convertDataToBytes(a), convertDataToBytes(b)); - } - - /** - * DirectSlice.data().array will throw UnsupportedOperationException. - * */ - public byte[] convertDataToBytes(DirectSlice directSlice) { - int capacity = directSlice.data().capacity(); - byte[] bytes = new byte[capacity]; - - for (int i = 0; i < capacity; i++) { - bytes[i] = directSlice.get(i); - } - - return bytes; - } - -} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/MarketUtils.java b/tools/toolkit/src/main/java/org/tron/plugins/utils/MarketUtils.java deleted file mode 100644 index a36fc8ad..00000000 --- a/tools/toolkit/src/main/java/org/tron/plugins/utils/MarketUtils.java +++ /dev/null @@ -1,148 +0,0 @@ -package org.tron.plugins.utils; - -import java.math.BigInteger; -import org.tron.plugins.utils.ByteArray; - -public class MarketUtils { - - public static final int TOKEN_ID_LENGTH = ByteArray - .fromString(Long.toString(Long.MAX_VALUE)).length; // 19 - - - - /** - * In order to avoid the difference between the data of same key stored and fetched by hashMap and - * levelDB, when creating the price key, we will find the GCD (Greatest Common Divisor) of - * sellTokenQuantity and buyTokenQuantity. - */ - public static byte[] createPairPriceKey(byte[] sellTokenId, byte[] buyTokenId, - long sellTokenQuantity, long buyTokenQuantity) { - - byte[] sellTokenQuantityBytes; - byte[] buyTokenQuantityBytes; - - // cal the GCD - long gcd = findGCD(sellTokenQuantity, buyTokenQuantity); - if (gcd == 0) { - sellTokenQuantityBytes = ByteArray.fromLong(sellTokenQuantity); - buyTokenQuantityBytes = ByteArray.fromLong(buyTokenQuantity); - } else { - sellTokenQuantityBytes = ByteArray.fromLong(sellTokenQuantity / gcd); - buyTokenQuantityBytes = ByteArray.fromLong(buyTokenQuantity / gcd); - } - - return doCreatePairPriceKey(sellTokenId, buyTokenId, - sellTokenQuantityBytes, buyTokenQuantityBytes); - } - - public static long findGCD(long number1, long number2) { - if (number1 == 0 || number2 == 0) { - return 0; - } - return calGCD(number1, number2); - } - - private static long calGCD(long number1, long number2) { - if (number2 == 0) { - return number1; - } - return calGCD(number2, number1 % number2); - } - - - private static byte[] doCreatePairPriceKey(byte[] sellTokenId, byte[] buyTokenId, - byte[] sellTokenQuantity, byte[] buyTokenQuantity) { - byte[] result = new byte[TOKEN_ID_LENGTH + TOKEN_ID_LENGTH - + sellTokenQuantity.length + buyTokenQuantity.length]; - - System.arraycopy(sellTokenId, 0, result, 0, sellTokenId.length); - System.arraycopy(buyTokenId, 0, result, TOKEN_ID_LENGTH, buyTokenId.length); - System.arraycopy(sellTokenQuantity, 0, result, - TOKEN_ID_LENGTH + TOKEN_ID_LENGTH, - sellTokenQuantity.length); - System.arraycopy(buyTokenQuantity, 0, result, - TOKEN_ID_LENGTH + TOKEN_ID_LENGTH + buyTokenQuantity.length, - buyTokenQuantity.length); - - return result; - } - - - public static int comparePriceKey(byte[] o1, byte[] o2) { - //compare pair - byte[] pair1 = new byte[TOKEN_ID_LENGTH * 2]; - byte[] pair2 = new byte[TOKEN_ID_LENGTH * 2]; - - System.arraycopy(o1, 0, pair1, 0, TOKEN_ID_LENGTH * 2); - System.arraycopy(o2, 0, pair2, 0, TOKEN_ID_LENGTH * 2); - - int pairResult = ByteArray.compareUnsigned(pair1, pair2); - if (pairResult != 0) { - return pairResult; - } - - //compare price - byte[] getSellTokenQuantity1 = new byte[8]; - byte[] getBuyTokenQuantity1 = new byte[8]; - - byte[] getSellTokenQuantity2 = new byte[8]; - byte[] getBuyTokenQuantity2 = new byte[8]; - - int longByteNum = 8; - - System.arraycopy(o1, TOKEN_ID_LENGTH + TOKEN_ID_LENGTH, - getSellTokenQuantity1, 0, longByteNum); - System.arraycopy(o1, TOKEN_ID_LENGTH + TOKEN_ID_LENGTH + longByteNum, - getBuyTokenQuantity1, 0, longByteNum); - - System.arraycopy(o2, TOKEN_ID_LENGTH + TOKEN_ID_LENGTH, - getSellTokenQuantity2, 0, longByteNum); - System.arraycopy(o2, TOKEN_ID_LENGTH + TOKEN_ID_LENGTH + longByteNum, - getBuyTokenQuantity2, 0, longByteNum); - - long sellTokenQuantity1 = ByteArray.toLong(getSellTokenQuantity1); - long buyTokenQuantity1 = ByteArray.toLong(getBuyTokenQuantity1); - long sellTokenQuantity2 = ByteArray.toLong(getSellTokenQuantity2); - long buyTokenQuantity2 = ByteArray.toLong(getBuyTokenQuantity2); - - if ((sellTokenQuantity1 == 0 || buyTokenQuantity1 == 0) - && (sellTokenQuantity2 == 0 || buyTokenQuantity2 == 0)) { - return 0; - } - - if (sellTokenQuantity1 == 0 || buyTokenQuantity1 == 0) { - return -1; - } - - if (sellTokenQuantity2 == 0 || buyTokenQuantity2 == 0) { - return 1; - } - - return comparePrice(sellTokenQuantity1, buyTokenQuantity1, - sellTokenQuantity2, buyTokenQuantity2); - - } - - /** - * Note: the params should be the same token pair, or you should change the order. - * All the quantity should be bigger than 0. - * */ - public static int comparePrice(long price1SellQuantity, long price1BuyQuantity, - long price2SellQuantity, long price2BuyQuantity) { - try { - return Long.compare(Math.multiplyExact(price1BuyQuantity, price2SellQuantity), - Math.multiplyExact(price2BuyQuantity, price1SellQuantity)); - - } catch (ArithmeticException ex) { - // do nothing here, because we will use BigInteger to compute again - } - - BigInteger price1BuyQuantityBI = BigInteger.valueOf(price1BuyQuantity); - BigInteger price1SellQuantityBI = BigInteger.valueOf(price1SellQuantity); - BigInteger price2BuyQuantityBI = BigInteger.valueOf(price2BuyQuantity); - BigInteger price2SellQuantityBI = BigInteger.valueOf(price2SellQuantity); - - return price1BuyQuantityBI.multiply(price2SellQuantityBI) - .compareTo(price2BuyQuantityBI.multiply(price1SellQuantityBI)); - } -} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/DBInterface.java b/tools/toolkit/src/main/java/org/tron/plugins/utils/db/DBInterface.java deleted file mode 100644 index 513e021c..00000000 --- a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/DBInterface.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.tron.plugins.utils.db; - -import java.io.Closeable; -import java.io.IOException; - - -public interface DBInterface extends Closeable { - - byte[] get(byte[] key); - - void put(byte[] key, byte[] value); - - void delete(byte[] key); - - DBIterator iterator(); - - long size(); - - void close() throws IOException; - - String getName(); - -} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/RocksDBImpl.java b/tools/toolkit/src/main/java/org/tron/plugins/utils/db/RocksDBImpl.java deleted file mode 100644 index 50957bbe..00000000 --- a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/RocksDBImpl.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.tron.plugins.utils.db; - -import java.io.IOException; -import lombok.Getter; -import org.rocksdb.RocksDBException; -import org.rocksdb.RocksIterator; - -public class RocksDBImpl implements DBInterface { - - private org.rocksdb.RocksDB rocksDB; - - @Getter - private final String name; - - public RocksDBImpl(org.rocksdb.RocksDB rocksDB, String name) { - this.rocksDB = rocksDB; - this.name = name; - } - - @Override - public byte[] get(byte[] key) { - try { - return rocksDB.get(key); - } catch (RocksDBException e) { - e.printStackTrace(); - } - return null; - } - - @Override - public void put(byte[] key, byte[] value) { - try { - rocksDB.put(key, value); - } catch (RocksDBException e) { - e.printStackTrace(); - } - } - - @Override - public void delete(byte[] key) { - try { - rocksDB.delete(key); - } catch (RocksDBException e) { - e.printStackTrace(); - } - } - - @Override - public DBIterator iterator() { - return new RockDBIterator(rocksDB.newIterator( - new org.rocksdb.ReadOptions().setFillCache(false))); - } - - @Override - public long size() { - RocksIterator iterator = rocksDB.newIterator(); - long size = 0; - for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { - size++; - } - iterator.close(); - return size; - } - - @Override - public void close() throws IOException { - rocksDB.close(); - } -} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/ArchiveManifest.java b/tools/toolkit/src/main/java/x86/org/tron/plugins/ArchiveManifest.java similarity index 98% rename from tools/toolkit/src/main/java/org/tron/plugins/ArchiveManifest.java rename to tools/toolkit/src/main/java/x86/org/tron/plugins/ArchiveManifest.java index 7d638bcd..115144be 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/ArchiveManifest.java +++ b/tools/toolkit/src/main/java/x86/org/tron/plugins/ArchiveManifest.java @@ -35,6 +35,7 @@ import org.iq80.leveldb.DB; import org.iq80.leveldb.Options; import org.iq80.leveldb.impl.Filename; +import org.tron.plugins.utils.DBUtils; import picocli.CommandLine; import picocli.CommandLine.Option; @@ -183,7 +184,7 @@ public boolean checkManifest(String dir) throws IOException { return false; } logger.info("CurrentName {}/{},size {} kb.", dir, currentName, current.length() / 1024); - if ("market_pair_price_to_order".equalsIgnoreCase(this.name)) { + if (DBUtils.MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(this.name)) { logger.info("Db {} ignored.", this.name); return false; } diff --git a/tools/toolkit/src/main/java/org/tron/plugins/DbArchive.java b/tools/toolkit/src/main/java/x86/org/tron/plugins/DbArchive.java similarity index 98% rename from tools/toolkit/src/main/java/org/tron/plugins/DbArchive.java rename to tools/toolkit/src/main/java/x86/org/tron/plugins/DbArchive.java index 2f117cd4..1088468f 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/DbArchive.java +++ b/tools/toolkit/src/main/java/x86/org/tron/plugins/DbArchive.java @@ -19,6 +19,7 @@ import org.iq80.leveldb.DB; import org.iq80.leveldb.Options; import org.iq80.leveldb.impl.Filename; +import org.tron.plugins.utils.DBUtils; import org.tron.plugins.utils.FileUtils; import picocli.CommandLine; import picocli.CommandLine.Option; @@ -163,7 +164,7 @@ public boolean checkManifest(String dir) throws IOException { return false; } logger.info("CurrentName {}/{},size {} kb.", dir, currentName, current.length() / 1024); - if ("market_pair_price_to_order".equalsIgnoreCase(this.name)) { + if (DBUtils.MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(this.name)) { logger.info("Db {} ignored.", this.name); return false; } @@ -194,5 +195,4 @@ public void doArchive() { } } } - } diff --git a/tools/toolkit/src/main/resources/fork.conf b/tools/toolkit/src/main/resources/fork.conf index f99896e3..705b29db 100644 --- a/tools/toolkit/src/main/resources/fork.conf +++ b/tools/toolkit/src/main/resources/fork.conf @@ -53,6 +53,78 @@ trc20Contracts = [ } ] -latestBlockHeaderTimestamp = 1747986162000 # MilliSecond. Used in init setup, setting to current timestamp reduces debug log output and speeds up initialization. This setting does not impact the underlying block production logic. -maintenanceTimeInterval = 21600000 -nextMaintenanceTime = 1747996162000 # MilliSecond. Set next maintenance time. It could be anytime. If you set an expired time, it will trigger maintenance logic immediately after node startup. +chainParameters = { + latestBlockHeaderTimestamp = 1747986162000 # MilliSecond. Used in init setup, setting to current timestamp reduces debug log output and speeds up initialization. This setting does not impact the underlying block production logic. + maintenanceTimeInterval = 21600000 + nextMaintenanceTime = 1747996162000 # MilliSecond. Set next maintenance time. It could be anytime. If you set an expired time, it will trigger maintenance logic immediately after node startup. + // accountUpgradeCost = 9999000000 + // createAccountFee = 100000 + // transactionFee = 1000 + // assetIssueFee = 1024000000 + // witnessPayPerBlock = 8000000 + // witnessStandbyAllowance = 115200000000 + // createNewAccountFeeInSystemContract = 1000000 + // createNewAccountBandwidthRate = 1 + // allowCreationOfContracts = 1 + // removeThePowerOfTheGr = -1 + // energyFee = 100 + // exchangeCreateFee = 1024000000 + // maxCpuTimeOfOneTx = 80 + // allowUpdateAccountName = 0 + // allowSameTokenName = 1 + // allowDelegateResource = 1 + // totalEnergyLimit = 180000000000 + // allowTvmTransferTrc10 = 1 + // totalEnergyCurrentLimit = 180000000000 + // allowMultiSign = 1 + // allowAdaptiveEnergy = 0 + // totalEnergyTargetLimit = 12500000 + // totalEnergyAverageUsage = 0 + // updateAccountPermissionFee = 100000000 + // multiSignFee = 1000000 + // allowAccountStateRoot = 0 + // allowProtoFilterNum = 0 + // allowTvmConstantinople = 1 + // allowTvmSolidity059 = 1 + // allowTvmIstanbul = 1 + // allowShieldedTRC20Transaction = 1 + // forbidTransferToContract = 0 + // adaptiveResourceLimitTargetRatio = 10 + // adaptiveResourceLimitMultiplier = 1000 + // changeDelegation = 1 + // witness127PayPerBlock = 128000000 + // allowMarketTransaction = 0 + // marketSellFee = 0 + // marketCancelFee = 0 + // allowPBFT = 0 + // allowTransactionFeePool = 0 + // maxFeeLimit = 15000000000 + // allowOptimizeBlackHole = 1 + // allowNewResourceModel = 0 + // allowTvmFreeze = 0 + // allowTvmVote = 1 + // allowTvmLondon = 1 + // allowTvmCompatibleEvm = 0 + // allowAccountAssetOptimization = 0 + // freeNetLimit = 600 + // totalNetLimit = 43200000000 + // allowHigherLimitForMaxCpuTimeOfOneTx = 1 + // allowAssetOptimization = 1 + // allowNewReward = 1 + // memoFee = 1000000 + // allowDelegateOptimization = 1 + // unfreezeDelayDays = 14 + // allowOptimizedReturnValueOfChainId = 1 + // allowDynamicEnergy = 1 + // dynamicEnergyThreshold = 5000000000 + // dynamicEnergyIncreaseFactor = 3400 + // allowTvmShangHai = 1 + // allowCancelAllUnfreezeV2 = 1 + // maxDelegateLockPeriod = 864000 + // allowEnergyAdjustment = 1 + // maxCreateAccountTxSize = 1000 + // allowStrictMath = 1 + // consensusLogicOptimization = 1 + // allowTvmCancun = 1 + // allowTvmBlob = 1 +} diff --git a/tools/toolkit/src/main/resources/logback.xml b/tools/toolkit/src/main/resources/logback.xml index dd2f2538..555edd23 100644 --- a/tools/toolkit/src/main/resources/logback.xml +++ b/tools/toolkit/src/main/resources/logback.xml @@ -53,5 +53,5 @@ - + diff --git a/tools/toolkit/src/main/resources/query.conf b/tools/toolkit/src/main/resources/query.conf index a1df2aee..4fcdb4cd 100644 --- a/tools/toolkit/src/main/resources/query.conf +++ b/tools/toolkit/src/main/resources/query.conf @@ -18,3 +18,13 @@ reward = [ "TDaQAfV7gZSPzx5FZ23kenPwLjr3AGxnmH", "TXwrVcWA2BPRMs9gGz5DciT6iwdDKagbpB" ] + +voters = { + allWitnesses = false + allActiveWitnesses = true + witnessList = [ + "TMEQ4hu7DtLTHPn6qPsYktk1QGdTbAyTEK", + "THKJYuUmMKKARNf7s2VT51g5uPY6KEqnat" + ] + threshold = 10000000 +} diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbCopyTest.java b/tools/toolkit/src/test/java/org/tron/plugins/DbCopyTest.java index 9e488a59..571fd8f5 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbCopyTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/DbCopyTest.java @@ -4,14 +4,25 @@ import java.util.UUID; import org.junit.Assert; import org.junit.Test; +import org.rocksdb.RocksDBException; +import org.tron.plugins.utils.db.DbTool; import picocli.CommandLine; public class DbCopyTest extends DbTest { @Test - public void testRun() { + public void testRunForLevelDB() throws RocksDBException, IOException { + init(DbTool.DbType.LevelDB); String[] args = new String[] { "db", "cp", INPUT_DIRECTORY, - genarateTmpDir()}; + generateTmpDir()}; + Assert.assertEquals(0, cli.execute(args)); + } + + @Test + public void testRunForRocksDB() throws RocksDBException, IOException { + init(DbTool.DbType.RocksDB); + String[] args = new String[] { "db", "cp", INPUT_DIRECTORY, + generateTmpDir()}; Assert.assertEquals(0, cli.execute(args)); } @@ -32,7 +43,7 @@ public void testNotExist() { @Test public void testEmpty() throws IOException { String[] args = new String[] {"db", "cp", temporaryFolder.newFolder().toString(), - genarateTmpDir()}; + generateTmpDir()}; Assert.assertEquals(0, cli.execute(args)); } @@ -46,7 +57,7 @@ public void testDestIsExist() throws IOException { @Test public void testSrcIsFile() throws IOException { String[] args = new String[] {"db", "cp", temporaryFolder.newFile().toString(), - genarateTmpDir()}; + generateTmpDir()}; Assert.assertEquals(403, cli.execute(args)); } diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbForkTest.java b/tools/toolkit/src/test/java/org/tron/plugins/DbForkTest.java index 1dea60b6..864ea232 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbForkTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/DbForkTest.java @@ -1,9 +1,35 @@ package org.tron.plugins; import static org.tron.plugins.DbFork.getActiveWitness; -import static org.tron.plugins.utils.Constant.*; +import static org.tron.plugins.utils.Constant.ACCOUNTS_KEY; +import static org.tron.plugins.utils.Constant.ACCOUNT_ADDRESS; +import static org.tron.plugins.utils.Constant.ACCOUNT_ASSET; +import static org.tron.plugins.utils.Constant.ACCOUNT_BALANCE; +import static org.tron.plugins.utils.Constant.ACCOUNT_NAME; +import static org.tron.plugins.utils.Constant.ACCOUNT_OWNER; +import static org.tron.plugins.utils.Constant.ACCOUNT_STORE; +import static org.tron.plugins.utils.Constant.ACCOUNT_TRC10_BALANCE; +import static org.tron.plugins.utils.Constant.ACCOUNT_TRC10_ID; +import static org.tron.plugins.utils.Constant.ACCOUNT_TYPE; +import static org.tron.plugins.utils.Constant.ACTIVE_DEFAULT_OPERATIONS; +import static org.tron.plugins.utils.Constant.ACTIVE_WITNESSES; +import static org.tron.plugins.utils.Constant.ALLOW_MULTI_SIGN; +import static org.tron.plugins.utils.Constant.ASSET_ISSUE_V2; +import static org.tron.plugins.utils.Constant.CONTRACT_STORE; +import static org.tron.plugins.utils.Constant.DYNAMIC_PROPERTY_STORE; +import static org.tron.plugins.utils.Constant.LATEST_BLOCK_HEADER_TIMESTAMP; +import static org.tron.plugins.utils.Constant.MAINTENANCE_TIME; +import static org.tron.plugins.utils.Constant.MAINTENANCE_TIME_INTERVAL; +import static org.tron.plugins.utils.Constant.STORAGE_ROW_STORE; +import static org.tron.plugins.utils.Constant.WITNESS_ADDRESS; +import static org.tron.plugins.utils.Constant.WITNESS_KEY; +import static org.tron.plugins.utils.Constant.WITNESS_SCHEDULE_STORE; +import static org.tron.plugins.utils.Constant.WITNESS_STORE; +import static org.tron.plugins.utils.Constant.WITNESS_URL; +import static org.tron.plugins.utils.Constant.WITNESS_VOTE; import com.google.common.primitives.Bytes; +import com.google.common.primitives.Longs; import com.google.protobuf.ByteString; import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; @@ -11,6 +37,7 @@ import java.io.IOException; import java.net.URL; import java.nio.file.Paths; +import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import org.junit.Assert; @@ -22,10 +49,15 @@ import org.tron.common.utils.Commons; import org.tron.core.capsule.AccountCapsule; import org.tron.core.capsule.WitnessCapsule; +import org.tron.plugins.utils.ChainParameters; import org.tron.plugins.utils.Constant; import org.tron.plugins.utils.FileUtils; import org.tron.plugins.utils.db.DBInterface; import org.tron.plugins.utils.db.DbTool; +import org.tron.protos.Protocol.Account; +import org.tron.protos.Protocol.Key; +import org.tron.protos.Protocol.Permission; +import org.tron.protos.Protocol.Permission.PermissionType; import picocli.CommandLine; public class DbForkTest { @@ -36,8 +68,6 @@ public class DbForkTest { private DBInterface dynamicPropertiesStore; private DBInterface accountAssetStore; private DBInterface assetIssueV2Store; - private DBInterface contractStore; - private DBInterface storageRowStore; @Rule public final TemporaryFolder folder = new TemporaryFolder(); @@ -67,8 +97,6 @@ public void init() throws IOException, RocksDBException { dynamicPropertiesStore = DbTool.getDB(srcDir, Constant.DYNAMIC_PROPERTY_STORE); accountAssetStore = DbTool.getDB(srcDir, Constant.ACCOUNT_ASSET); assetIssueV2Store = DbTool.getDB(srcDir, Constant.ASSET_ISSUE_V2); - contractStore = DbTool.getDB(srcDir, Constant.CONTRACT_STORE); - storageRowStore = DbTool.getDB(srcDir, Constant.STORAGE_ROW_STORE); } public void close() { @@ -78,7 +106,7 @@ public void close() { @Test public void testDbFork() throws IOException, RocksDBException { dbPath = folder.newFolder().toString(); - forkPath = getConfig("fork.conf"); + String forkPath = getConfig("fork.conf"); createDir(); String[] args = new String[]{"-d", @@ -122,12 +150,15 @@ public void testDbFork() throws IOException, RocksDBException { w -> { WitnessCapsule witnessCapsule = new WitnessCapsule(witnessStore.get( Commons.decodeFromBase58Check(w.getString(WITNESS_ADDRESS)))); + AccountCapsule accountCapsule = new AccountCapsule(accountStore.get( + Commons.decodeFromBase58Check(w.getString(WITNESS_ADDRESS)))); if (w.hasPath(WITNESS_VOTE)) { Assert.assertEquals(w.getLong(WITNESS_VOTE), witnessCapsule.getVoteCount()); } if (w.hasPath(WITNESS_URL)) { Assert.assertEquals(w.getString(WITNESS_URL), witnessCapsule.getUrl()); } + Assert.assertTrue(accountCapsule.getIsWitness()); } ); } @@ -176,29 +207,130 @@ public void testDbFork() throws IOException, RocksDBException { } } } + if (a.getString(ACCOUNT_ADDRESS).equals("TLLM21wteSPs4hKjbxgmH1L6poyMjeTbHm")) { + Assert.assertFalse(account.getIsWitness()); + } }); } - if (forkConfig.hasPath(LATEST_BLOCK_TIMESTAMP)) { - long latestBlockHeaderTimestamp = forkConfig.getLong(LATEST_BLOCK_TIMESTAMP); + if (!forkConfig.hasPath(ChainParameters.CHAIN_PARAMETERS)) { + return; + } + + Config chainParamsConfig = forkConfig.getConfig(ChainParameters.CHAIN_PARAMETERS); + if (chainParamsConfig.hasPath(ChainParameters.LATEST_BLOCK_TIMESTAMP)) { + long latestBlockHeaderTimestamp = chainParamsConfig.getLong(ChainParameters.LATEST_BLOCK_TIMESTAMP); Assert.assertEquals(latestBlockHeaderTimestamp, ByteArray.toLong(dynamicPropertiesStore.get(LATEST_BLOCK_HEADER_TIMESTAMP))); } - if (forkConfig.hasPath(MAINTENANCE_INTERVAL)) { - long maintenanceTimeInterval = forkConfig.getLong(MAINTENANCE_INTERVAL); + if (chainParamsConfig.hasPath(ChainParameters.MAINTENANCE_INTERVAL)) { + long maintenanceTimeInterval = chainParamsConfig.getLong(ChainParameters.MAINTENANCE_INTERVAL); Assert.assertEquals(maintenanceTimeInterval, ByteArray.toLong(dynamicPropertiesStore.get(MAINTENANCE_TIME_INTERVAL))); } - if (forkConfig.hasPath(NEXT_MAINTENANCE_TIME)) { - long nextMaintenanceTime = forkConfig.getLong(NEXT_MAINTENANCE_TIME); + if (chainParamsConfig.hasPath(ChainParameters.NEXT_MAINTENANCE_TIME)) { + long nextMaintenanceTime = chainParamsConfig.getLong(ChainParameters.NEXT_MAINTENANCE_TIME); Assert.assertEquals(nextMaintenanceTime, ByteArray.toLong(dynamicPropertiesStore.get(MAINTENANCE_TIME))); } close(); } + @Test + public void testDbForkAddsWitnessPermissionWhenMultiSignEnabled() + throws IOException, RocksDBException { + dbPath = folder.newFolder().toString(); + forkPath = getConfig("fork.conf"); + createDir(); + init(); + dynamicPropertiesStore.put(ALLOW_MULTI_SIGN, Longs.toByteArray(1L)); + byte[] activeDefaultOperations = new byte[32]; + Arrays.fill(activeDefaultOperations, (byte) 0x7f); + dynamicPropertiesStore.put(ACTIVE_DEFAULT_OPERATIONS, activeDefaultOperations); + close(); + + String[] args = new String[]{"-d", + dbPath, "-c", + forkPath}; + CommandLine cli = new CommandLine(new DbFork()); + Assert.assertEquals(0, cli.execute(args)); + + init(); + Config forkConfig = ConfigFactory.parseFile(Paths.get(forkPath).toFile()); + List witnesses = forkConfig.getConfigList(WITNESS_KEY).stream() + .filter(c -> c.hasPath(WITNESS_ADDRESS)) + .collect(Collectors.toList()); + + witnesses.forEach(w -> { + byte[] address = Commons.decodeFromBase58Check(w.getString(WITNESS_ADDRESS)); + AccountCapsule account = new AccountCapsule(accountStore.get(address)); + Assert.assertTrue(account.getInstance().hasOwnerPermission()); + Assert.assertTrue(account.getInstance().hasWitnessPermission()); + Assert.assertEquals(1, account.getInstance().getActivePermissionCount()); + Assert.assertArrayEquals(address, + account.getInstance().getOwnerPermission().getKeys(0).getAddress().toByteArray()); + Assert.assertArrayEquals(address, account.getWitnessPermissionAddress()); + Assert.assertArrayEquals(activeDefaultOperations, + account.getInstance().getActivePermission(0).getOperations().toByteArray()); + }); + + close(); + } + + @Test + public void testDbForkKeepsExistingWitnessPermissionWhenMultiSignEnabled() + throws IOException, RocksDBException { + dbPath = folder.newFolder().toString(); + forkPath = getConfig("fork.conf"); + createDir(); + init(); + dynamicPropertiesStore.put(ALLOW_MULTI_SIGN, Longs.toByteArray(1L)); + byte[] activeDefaultOperations = new byte[32]; + Arrays.fill(activeDefaultOperations, (byte) 0x7f); + dynamicPropertiesStore.put(ACTIVE_DEFAULT_OPERATIONS, activeDefaultOperations); + + byte[] witnessAddress = Commons.decodeFromBase58Check("TS1hu4ZCcwBFYpQqUGoWy1GWBzamqxiT5W"); + byte[] customWitnessAddress = Commons.decodeFromBase58Check("TLLM21wteSPs4hKjbxgmH1L6poyMjeTbHm"); + Permission ownerPermission = AccountCapsule.createDefaultOwnerPermission( + ByteString.copyFrom(witnessAddress)); + Permission activePermission = Permission.newBuilder() + .setType(PermissionType.Active) + .setId(2) + .setPermissionName("active") + .setThreshold(1) + .setParentId(0) + .setOperations(ByteString.copyFrom(activeDefaultOperations)) + .addKeys(Key.newBuilder() + .setAddress(ByteString.copyFrom(witnessAddress)) + .setWeight(1) + .build()) + .build(); + Permission witnessPermission = AccountCapsule.createDefaultWitnessPermission( + ByteString.copyFrom(customWitnessAddress)); + AccountCapsule existingAccount = new AccountCapsule(Account.newBuilder() + .setAddress(ByteString.copyFrom(witnessAddress)) + .setOwnerPermission(ownerPermission) + .addActivePermission(activePermission) + .setWitnessPermission(witnessPermission) + .build()); + accountStore.put(witnessAddress, existingAccount.getData()); + close(); + + String[] args = new String[]{"-d", + dbPath, "-c", + forkPath}; + CommandLine cli = new CommandLine(new DbFork()); + Assert.assertEquals(0, cli.execute(args)); + + init(); + AccountCapsule account = new AccountCapsule(accountStore.get(witnessAddress)); + Assert.assertTrue(account.getInstance().hasWitnessPermission()); + Assert.assertArrayEquals(customWitnessAddress, account.getWitnessPermissionAddress()); + close(); + } + private String getConfig(String config) { URL path = DbForkTest.class.getClassLoader().getResource(config); return path == null ? null : path.getPath(); diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbLiteTest.java b/tools/toolkit/src/test/java/org/tron/plugins/DbLiteTest.java index 3d3b9d5f..1d5771a0 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbLiteTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/DbLiteTest.java @@ -1,12 +1,12 @@ package org.tron.plugins; +import static org.tron.plugins.utils.PublicMethod.getRandomPrivateKey; + import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import java.io.File; import java.io.IOException; -import java.net.URL; import java.nio.file.Paths; -import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Rule; @@ -18,11 +18,10 @@ import org.tron.common.config.DbBackupConfig; import org.tron.common.crypto.ECKey; import org.tron.common.utils.FileUtil; +import org.tron.common.utils.TimeoutInterceptor; import org.tron.common.utils.Utils; import org.tron.core.config.DefaultConfig; import org.tron.core.config.args.Args; -import org.tron.core.services.RpcApiService; -import org.tron.core.services.interfaceOnSolidity.RpcApiServiceOnSolidity; import org.tron.plugins.utils.PublicMethod; import picocli.CommandLine; @@ -47,14 +46,13 @@ public class DbLiteTest { public void startApp() { context = new TronApplicationContext(DefaultConfig.class); appTest = ApplicationFactory.create(context); - appTest.addService(context.getBean(RpcApiService.class)); - appTest.addService(context.getBean(RpcApiServiceOnSolidity.class)); appTest.startup(); String fullNode = String.format("%s:%d", "127.0.0.1", Args.getInstance().getRpcPort()); channelFull = ManagedChannelBuilder.forTarget(fullNode) .usePlaintext() + .intercept(new TimeoutInterceptor(5000)) .build(); blockingStubFull = WalletGrpc.newBlockingStub(channelFull); } @@ -64,18 +62,20 @@ public void startApp() { */ public void shutdown() throws InterruptedException { if (channelFull != null) { - channelFull.shutdown().awaitTermination(5, TimeUnit.SECONDS); + channelFull.shutdownNow(); } context.close(); } - public void init() throws IOException { + public void init(String dbType) throws IOException { dbPath = folder.newFolder().toString(); - Args.setParam(new String[]{"-d", dbPath, "-w", "--p2p-disable", "true"}, - getConfig("config-localtest.conf")); + Args.setParam(new String[] { + "-d", dbPath, "-w", "--p2p-disable", "true", "--storage-db-engine", dbType}, + "config-localtest.conf"); // allow account root Args.getInstance().setAllowAccountStateRoot(1); Args.getInstance().setRpcPort(PublicMethod.chooseRandomPort()); + Args.getInstance().setRpcEnable(true); databaseDir = Args.getInstance().getStorage().getDbDirectory(); // init dbBackupConfig to avoid NPE Args.getInstance().dbBackupConfig = DbBackupConfig.getInstance(); @@ -86,23 +86,22 @@ public void clear() { Args.clearParam(); } - void testTools(String dbType, int checkpointVersion) + public void testTools(String dbType, int checkpointVersion) throws InterruptedException, IOException { logger.info("dbType {}, checkpointVersion {}", dbType, checkpointVersion); dbPath = String.format("%s_%s_%d", dbPath, dbType, System.currentTimeMillis()); - init(); + init(dbType); final String[] argsForSnapshot = - new String[]{"-o", "split", "-t", "snapshot", "--fn-data-path", + new String[] {"-o", "split", "-t", "snapshot", "--fn-data-path", dbPath + File.separator + databaseDir, "--dataset-path", dbPath}; final String[] argsForHistory = - new String[]{"-o", "split", "-t", "history", "--fn-data-path", + new String[] {"-o", "split", "-t", "history", "--fn-data-path", dbPath + File.separator + databaseDir, "--dataset-path", dbPath}; final String[] argsForMerge = - new String[]{"-o", "merge", "--fn-data-path", dbPath + File.separator + databaseDir, + new String[] {"-o", "merge", "--fn-data-path", dbPath + File.separator + databaseDir, "--dataset-path", dbPath + File.separator + "history"}; - Args.getInstance().getStorage().setDbEngine(dbType); Args.getInstance().getStorage().setCheckpointVersion(checkpointVersion); DbLite.setRecentBlks(3); // start fullNode @@ -127,15 +126,15 @@ void testTools(String dbType, int checkpointVersion) File database = new File(Paths.get(dbPath, databaseDir).toString()); if (!database.renameTo(new File(Paths.get(dbPath, databaseDir + "_bak").toString()))) { throw new RuntimeException( - String.format("rename %s to %s failed", database.getPath(), - Paths.get(dbPath, databaseDir))); + String.format("rename %s to %s failed", database.getPath(), + Paths.get(dbPath, databaseDir))); } // change snapshot to the new database File snapshot = new File(Paths.get(dbPath, "snapshot").toString()); if (!snapshot.renameTo(new File(Paths.get(dbPath, databaseDir).toString()))) { throw new RuntimeException( - String.format("rename snapshot to %s failed", - Paths.get(dbPath, databaseDir))); + String.format("rename snapshot to %s failed", + Paths.get(dbPath, databaseDir))); } // start and validate the snapshot startApp(); @@ -159,10 +158,10 @@ private void generateSomeTransactions(int during) { ECKey ecKey2 = new ECKey(Utils.getRandom()); byte[] address = ecKey2.getAddress(); - String sunPri = "cba92a516ea09f620a16ff7ee95ce0df1d56550a8babe9964981a7144c8a784a"; + String sunPri = getRandomPrivateKey(); byte[] sunAddress = PublicMethod.getFinalAddress(sunPri); PublicMethod.sendcoin(address, 1L, - sunAddress, sunPri, blockingStubFull); + sunAddress, sunPri, blockingStubFull); try { Thread.sleep(sleepOnce); } catch (InterruptedException e) { @@ -173,9 +172,4 @@ private void generateSomeTransactions(int during) { } } } - - private static String getConfig(String config) { - URL path = DbLiteTest.class.getClassLoader().getResource(config); - return path == null ? null : path.getPath(); - } } diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbMoveTest.java b/tools/toolkit/src/test/java/org/tron/plugins/DbMoveTest.java index c1bc6b47..5b25739f 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbMoveTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/DbMoveTest.java @@ -1,52 +1,41 @@ package org.tron.plugins; -import static org.iq80.leveldb.impl.Iq80DBFactory.factory; - import java.io.File; import java.io.IOException; import java.net.URL; import java.nio.file.Paths; -import java.util.UUID; import lombok.extern.slf4j.Slf4j; -import org.junit.AfterClass; +import org.junit.After; import org.junit.Assert; -import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; -import org.tron.plugins.utils.FileUtils; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.RocksDBException; +import org.tron.plugins.utils.DBUtils; +import org.tron.plugins.utils.db.DbTool; import picocli.CommandLine; @Slf4j public class DbMoveTest { private static final String OUTPUT_DIRECTORY = "output-directory-toolkit"; - private static final String OUTPUT_DIRECTORY_DATABASE = - Paths.get(OUTPUT_DIRECTORY,"ori","database").toString(); - private static final String ENGINE = "ENGINE"; - private static final String LEVELDB = "LEVELDB"; - private static final String ACCOUNT = "account"; - private static final String TRANS = "trans"; - private static final String MARKET = "market_pair_price_to_order"; - private static final String ENGINE_FILE = "engine.properties"; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); - @BeforeClass - public static void init() throws IOException { - File file = new File(OUTPUT_DIRECTORY_DATABASE, ACCOUNT); - factory.open(file, ArchiveManifest.newDefaultLevelDbOptions()).close(); - FileUtils.writeProperty(file + File.separator + ENGINE_FILE, ENGINE, LEVELDB); + private static final String ACCOUNT = "account"; + private static final String TRANS = "trans"; - file = new File(OUTPUT_DIRECTORY_DATABASE, MARKET); - factory.open(file, ArchiveManifest.newDefaultLevelDbOptions()).close(); - FileUtils.writeProperty(file + File.separator + ENGINE_FILE, ENGINE, LEVELDB); - file = new File(OUTPUT_DIRECTORY_DATABASE, TRANS); - factory.open(file, ArchiveManifest.newDefaultLevelDbOptions()).close(); - FileUtils.writeProperty(file + File.separator + ENGINE_FILE, ENGINE, LEVELDB); + private void init(DbTool.DbType dbType, String path) throws IOException, RocksDBException { + DbTool.getDB(path, ACCOUNT, dbType).close(); + DbTool.getDB(path, DBUtils.MARKET_PAIR_PRICE_TO_ORDER, dbType).close(); + DbTool.getDB(path, TRANS, dbType).close(); } - @AfterClass - public static void destroy() { + @After + public void destroy() { deleteDir(new File(OUTPUT_DIRECTORY)); } @@ -74,9 +63,23 @@ private static String getConfig(String config) { } @Test - public void testMv() { + public void testMvForLevelDB() throws RocksDBException, IOException { + File database = temporaryFolder.newFolder("database"); + init(DbTool.DbType.LevelDB, Paths.get(database.getPath()).toString()); + String[] args = new String[] {"db", "mv", "-d", + database.getParent(), "-c", + getConfig("config.conf")}; + CommandLine cli = new CommandLine(new Toolkit()); + Assert.assertEquals(0, cli.execute(args)); + Assert.assertEquals(2, cli.execute(args)); + } + + @Test + public void testMvForRocksDB() throws RocksDBException, IOException { + File database = temporaryFolder.newFolder("database"); + init(DbTool.DbType.RocksDB, Paths.get(database.getPath()).toString()); String[] args = new String[] {"db", "mv", "-d", - Paths.get(OUTPUT_DIRECTORY,"ori").toString(), "-c", + database.getParent(), "-c", getConfig("config.conf")}; CommandLine cli = new CommandLine(new Toolkit()); Assert.assertEquals(0, cli.execute(args)); @@ -84,9 +87,10 @@ public void testMv() { } @Test - public void testDuplicate() { + public void testDuplicate() throws IOException { + File output = temporaryFolder.newFolder(); String[] args = new String[] {"db", "mv", "-d", - Paths.get(OUTPUT_DIRECTORY,"ori").toString(), "-c", + output.getPath(), "-c", getConfig("config-duplicate.conf")}; CommandLine cli = new CommandLine(new Toolkit()); Assert.assertEquals(2, cli.execute(args)); @@ -107,20 +111,19 @@ public void testDicNotExist() { } @Test - public void testConfNotExist() { + public void testConfNotExist() throws IOException { + File output = temporaryFolder.newFolder(); String[] args = new String[] {"db", "mv", "-d", - Paths.get(OUTPUT_DIRECTORY,"ori").toString(), "-c", + output.getPath(), "-c", "config.conf"}; CommandLine cli = new CommandLine(new Toolkit()); Assert.assertEquals(2, cli.execute(args)); } @Test - public void testEmpty() { - File file = new File(OUTPUT_DIRECTORY_DATABASE + File.separator + UUID.randomUUID()); - file.mkdirs(); - file.deleteOnExit(); - String[] args = new String[] {"db", "mv", "-d", file.toString(), "-c", + public void testEmpty() throws IOException { + File output = temporaryFolder.newFolder(); + String[] args = new String[] {"db", "mv", "-d", output.getPath(), "-c", getConfig("config.conf")}; CommandLine cli = new CommandLine(new Toolkit()); diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbRootTest.java b/tools/toolkit/src/test/java/org/tron/plugins/DbRootTest.java index b86688f7..9d032a43 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbRootTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/DbRootTest.java @@ -9,10 +9,8 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.rocksdb.RocksDBException; -import org.tron.plugins.utils.DBUtils; import org.tron.plugins.utils.db.DBInterface; import org.tron.plugins.utils.db.DbTool; -import org.tron.plugins.utils.db.LevelDBImpl; import picocli.CommandLine; @Slf4j @@ -27,8 +25,18 @@ public class DbRootTest { private static final String EMPTY_DB = "empty"; private static final String ERROR_DB = "error"; + + @Test + public void testRootForLevelDB() throws RocksDBException, IOException { + testRoot(DbTool.DbType.LevelDB); + } + @Test - public void testRoot() throws IOException, RocksDBException { + public void testRootForRocksDB() throws RocksDBException, IOException { + testRoot(DbTool.DbType.RocksDB); + } + + public void testRoot(DbTool.DbType dbType) throws IOException, RocksDBException { File file = folder.newFolder(); @@ -36,8 +44,8 @@ public void testRoot() throws IOException, RocksDBException { Assert.assertTrue(database.mkdirs()); - try (DBInterface normal = DbTool.getDB(database.toString(), NORMAL_DB, DbTool.DbType.LevelDB); - DBInterface empty = DbTool.getDB(database.toString(), EMPTY_DB, DbTool.DbType.RocksDB)) { + try (DBInterface normal = DbTool.getDB(database.toString(), NORMAL_DB, dbType); + DBInterface empty = DbTool.getDB(database.toString(), EMPTY_DB, dbType)) { for (int i = 0; i < 10; i++) { normal.put(("" + i).getBytes(), (NORMAL_DB + "-" + i).getBytes()); } @@ -53,8 +61,7 @@ public void testRoot() throws IOException, RocksDBException { "--db", EMPTY_DB}; Assert.assertEquals(0, cli.execute(args)); - try (DBInterface errorDb = new LevelDBImpl( - DBUtils.newLevelDb(Paths.get(database.toString(), ERROR_DB)), ERROR_DB)) { + try (DBInterface errorDb = DbTool.getDB(database.toString(), ERROR_DB, dbType)) { for (int i = 0; i < 10; i++) { errorDb.put(("" + i).getBytes(), (ERROR_DB + "-" + i).getBytes()); } diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbTest.java b/tools/toolkit/src/test/java/org/tron/plugins/DbTest.java index 8605fa18..bbcc1a0b 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/DbTest.java @@ -5,42 +5,43 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Paths; import java.util.UUID; -import org.iq80.leveldb.DB; -import org.junit.Before; +import org.junit.Assert; import org.junit.Rule; import org.junit.rules.TemporaryFolder; +import org.rocksdb.RocksDBException; import org.tron.plugins.utils.ByteArray; import org.tron.plugins.utils.DBUtils; import org.tron.plugins.utils.MarketUtils; +import org.tron.plugins.utils.db.DBInterface; +import org.tron.plugins.utils.db.DbTool; import picocli.CommandLine; public class DbTest { - String INPUT_DIRECTORY; + public String INPUT_DIRECTORY; private static final String ACCOUNT = "account"; private static final String MARKET = DBUtils.MARKET_PAIR_PRICE_TO_ORDER; - CommandLine cli = new CommandLine(new Toolkit()); - String tmpDir = System.getProperty("java.io.tmpdir"); + public CommandLine cli = new CommandLine(new Toolkit()); @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder(); - @Before - public void init() throws IOException { + public void init(DbTool.DbType dbType) throws IOException, RocksDBException { INPUT_DIRECTORY = temporaryFolder.newFolder().toString(); - initDB(new File(INPUT_DIRECTORY, ACCOUNT)); - initDB(new File(INPUT_DIRECTORY, MARKET)); - initDB(new File(INPUT_DIRECTORY, DBUtils.CHECKPOINT_DB_V2)); + initDB(INPUT_DIRECTORY, ACCOUNT, dbType); + initDB(INPUT_DIRECTORY, MARKET, dbType); + initDB(INPUT_DIRECTORY, DBUtils.CHECKPOINT_DB_V2, dbType); } - private static void initDB(File file) throws IOException { - if (DBUtils.CHECKPOINT_DB_V2.equalsIgnoreCase(file.getName())) { - File dbFile = new File(file, DBUtils.CHECKPOINT_DB_V2); + private static void initDB(String sourceDir, String dbName, DbTool.DbType dbType) + throws IOException, RocksDBException { + if (DBUtils.CHECKPOINT_DB_V2.equalsIgnoreCase(dbName)) { + File dbFile = new File(sourceDir, DBUtils.CHECKPOINT_DB_V2); if (dbFile.mkdirs()) { for (int i = 0; i < 3; i++) { - try (DB db = DBUtils.newLevelDb(Paths.get(dbFile.getPath(), - System.currentTimeMillis() + ""))) { + try (DBInterface db = DbTool.getDB(dbFile.getPath(), + System.currentTimeMillis() + "", dbType)) { for (int j = 0; j < 100; j++) { byte[] bytes = UUID.randomUUID().toString().getBytes(); db.put(bytes, bytes); @@ -50,8 +51,8 @@ private static void initDB(File file) throws IOException { } return; } - try (DB db = DBUtils.newLevelDb(file.toPath())) { - if (MARKET.equalsIgnoreCase(file.getName())) { + try (DBInterface db = DbTool.getDB(sourceDir, dbName, dbType)) { + if (MARKET.equalsIgnoreCase(dbName)) { byte[] sellTokenID1 = ByteArray.fromString("100"); byte[] buyTokenID1 = ByteArray.fromString("200"); byte[] pairPriceKey1 = MarketUtils.createPairPriceKey( @@ -73,26 +74,29 @@ private static void initDB(File file) throws IOException { 2003L ); - //Use out-of-order insertion，key in store should be 1,2,3 db.put(pairPriceKey1, "1".getBytes(StandardCharsets.UTF_8)); db.put(pairPriceKey2, "2".getBytes(StandardCharsets.UTF_8)); db.put(pairPriceKey3, "3".getBytes(StandardCharsets.UTF_8)); + Assert.assertEquals(3, db.size()); } else { for (int i = 0; i < 100; i++) { byte[] bytes = UUID.randomUUID().toString().getBytes(); db.put(bytes, bytes); } + Assert.assertEquals(100, db.size()); } } } /** * Generate a not-exist temporary directory path. + * * @return temporary path */ - public String genarateTmpDir() { - File dir = Paths.get(tmpDir, UUID.randomUUID().toString()).toFile(); + public String generateTmpDir() throws IOException { + File dir = Paths.get(temporaryFolder.newFolder().toString(), UUID.randomUUID().toString()) + .toFile(); dir.deleteOnExit(); return dir.getPath(); } diff --git a/tools/toolkit/src/test/java/org/tron/plugins/ArchiveManifestTest.java b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/ArchiveManifestTest.java similarity index 69% rename from tools/toolkit/src/test/java/org/tron/plugins/ArchiveManifestTest.java rename to tools/toolkit/src/test/java/org/tron/plugins/leveldb/ArchiveManifestTest.java index 4fd9e537..f5880d82 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/ArchiveManifestTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/ArchiveManifestTest.java @@ -1,6 +1,4 @@ -package org.tron.plugins; - -import static org.iq80.leveldb.impl.Iq80DBFactory.factory; +package org.tron.plugins.leveldb; import java.io.BufferedReader; import java.io.BufferedWriter; @@ -14,47 +12,41 @@ import java.nio.charset.StandardCharsets; import java.util.Properties; import java.util.UUID; - import lombok.extern.slf4j.Slf4j; -import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.RocksDBException; +import org.tron.plugins.ArchiveManifest; +import org.tron.plugins.utils.DBUtils; +import org.tron.plugins.utils.db.DbTool; @Slf4j public class ArchiveManifestTest { - private static final String OUTPUT_DIRECTORY = "output-directory/database/archiveManifest"; + @ClassRule + public static final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private static String OUTPUT_DIRECTORY; - private static final String ENGINE = "ENGINE"; - private static final String LEVELDB = "LEVELDB"; - private static final String ROCKSDB = "ROCKSDB"; private static final String ACCOUNT = "account"; private static final String ACCOUNT_ROCKSDB = "account-rocksdb"; - private static final String MARKET = "market_pair_price_to_order"; - private static final String ENGINE_FILE = "engine.properties"; - @BeforeClass - public static void init() throws IOException { - File file = new File(OUTPUT_DIRECTORY,ACCOUNT); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,LEVELDB); - - file = new File(OUTPUT_DIRECTORY,MARKET); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,LEVELDB); + public static void init() throws IOException, RocksDBException { + OUTPUT_DIRECTORY = temporaryFolder.newFolder("database").toString(); + File file = new File(OUTPUT_DIRECTORY, ACCOUNT); + DbTool.openLevelDb(file.toPath(),ACCOUNT).close(); - file = new File(OUTPUT_DIRECTORY,ACCOUNT_ROCKSDB); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,ROCKSDB); + file = new File(OUTPUT_DIRECTORY, DBUtils.MARKET_PAIR_PRICE_TO_ORDER); + DbTool.openLevelDb(file.toPath(), DBUtils.MARKET_PAIR_PRICE_TO_ORDER).close(); - } + file = new File(OUTPUT_DIRECTORY, ACCOUNT_ROCKSDB); + DbTool.openRocksDb(file.toPath(), ACCOUNT_ROCKSDB).close(); - @AfterClass - public static void destroy() { - deleteDir(new File(OUTPUT_DIRECTORY)); } @Test diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbArchiveTest.java b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbArchiveTest.java similarity index 71% rename from tools/toolkit/src/test/java/org/tron/plugins/DbArchiveTest.java rename to tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbArchiveTest.java index 10bed418..69dca01e 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbArchiveTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbArchiveTest.java @@ -1,6 +1,4 @@ -package org.tron.plugins; - -import static org.iq80.leveldb.impl.Iq80DBFactory.factory; +package org.tron.plugins.leveldb; import java.io.BufferedReader; import java.io.BufferedWriter; @@ -12,48 +10,43 @@ import java.io.OutputStream; import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; -import java.nio.file.Paths; import java.util.Properties; import java.util.UUID; import lombok.extern.slf4j.Slf4j; -import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.RocksDBException; +import org.tron.plugins.Toolkit; +import org.tron.plugins.utils.DBUtils; +import org.tron.plugins.utils.db.DbTool; import picocli.CommandLine; @Slf4j public class DbArchiveTest { - private static final String OUTPUT_DIRECTORY = "output-directory/database/dbArchive"; + @ClassRule + public static final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private static String OUTPUT_DIRECTORY; - private static final String ENGINE = "ENGINE"; - private static final String LEVELDB = "LEVELDB"; - private static final String ROCKSDB = "ROCKSDB"; private static final String ACCOUNT = "account"; private static final String ACCOUNT_ROCKSDB = "account-rocksdb"; - private static final String MARKET = "market_pair_price_to_order"; - private static final String ENGINE_FILE = "engine.properties"; @BeforeClass - public static void init() throws IOException { - File file = new File(OUTPUT_DIRECTORY,ACCOUNT); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,LEVELDB); + public static void init() throws IOException, RocksDBException { + OUTPUT_DIRECTORY = temporaryFolder.newFolder("database").toString(); + File file = new File(OUTPUT_DIRECTORY, ACCOUNT); + DbTool.openLevelDb(file.toPath(),ACCOUNT).close(); - file = new File(OUTPUT_DIRECTORY,MARKET); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,LEVELDB); + file = new File(OUTPUT_DIRECTORY, DBUtils.MARKET_PAIR_PRICE_TO_ORDER); + DbTool.openLevelDb(file.toPath(), DBUtils.MARKET_PAIR_PRICE_TO_ORDER).close(); - file = new File(OUTPUT_DIRECTORY,ACCOUNT_ROCKSDB); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,ROCKSDB); - - } + file = new File(OUTPUT_DIRECTORY, ACCOUNT_ROCKSDB); + DbTool.openRocksDb(file.toPath(), ACCOUNT_ROCKSDB).close(); - @AfterClass - public static void destroy() { - deleteDir(new File(OUTPUT_DIRECTORY)); } @Test diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbConvertTest.java b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbConvertTest.java similarity index 76% rename from tools/toolkit/src/test/java/org/tron/plugins/DbConvertTest.java rename to tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbConvertTest.java index 150e47c9..d24604f0 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbConvertTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbConvertTest.java @@ -1,27 +1,25 @@ -package org.tron.plugins; +package org.tron.plugins.leveldb; import java.io.IOException; import java.util.UUID; import org.junit.Assert; import org.junit.Test; +import org.rocksdb.RocksDBException; +import org.tron.plugins.DbTest; +import org.tron.plugins.Toolkit; +import org.tron.plugins.utils.db.DbTool; import picocli.CommandLine; public class DbConvertTest extends DbTest { @Test - public void testRun() throws IOException { + public void testRun() throws IOException, RocksDBException { + init(DbTool.DbType.LevelDB); String[] args = new String[] { "db", "convert", INPUT_DIRECTORY, temporaryFolder.newFolder().toString() }; Assert.assertEquals(0, cli.execute(args)); } - @Test - public void testRunWithSafe() throws IOException { - String[] args = new String[] { "db", "convert", INPUT_DIRECTORY, - temporaryFolder.newFolder().toString(),"--safe" }; - Assert.assertEquals(0, cli.execute(args)); - } - @Test public void testHelp() { String[] args = new String[] {"db", "convert", "-h"}; diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbLiteLevelDbTest.java b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbTest.java similarity index 76% rename from tools/toolkit/src/test/java/org/tron/plugins/DbLiteLevelDbTest.java rename to tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbTest.java index 792f93ad..7666806e 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbLiteLevelDbTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbTest.java @@ -1,7 +1,8 @@ -package org.tron.plugins; +package org.tron.plugins.leveldb; import java.io.IOException; import org.junit.Test; +import org.tron.plugins.DbLiteTest; public class DbLiteLevelDbTest extends DbLiteTest { diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbLiteLevelDbV2Test.java b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbV2Test.java similarity index 76% rename from tools/toolkit/src/test/java/org/tron/plugins/DbLiteLevelDbV2Test.java rename to tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbV2Test.java index ae48e1d6..de32ae29 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbLiteLevelDbV2Test.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbV2Test.java @@ -1,7 +1,8 @@ -package org.tron.plugins; +package org.tron.plugins.leveldb; import java.io.IOException; import org.junit.Test; +import org.tron.plugins.DbLiteTest; public class DbLiteLevelDbV2Test extends DbLiteTest { diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbLiteRocksDbTest.java b/tools/toolkit/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbTest.java similarity index 76% rename from tools/toolkit/src/test/java/org/tron/plugins/DbLiteRocksDbTest.java rename to tools/toolkit/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbTest.java index e6910b11..2f9c92f9 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbLiteRocksDbTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbTest.java @@ -1,7 +1,8 @@ -package org.tron.plugins; +package org.tron.plugins.rocksdb; import java.io.IOException; import org.junit.Test; +import org.tron.plugins.DbLiteTest; public class DbLiteRocksDbTest extends DbLiteTest { diff --git a/tools/toolkit/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbV2Test.java b/tools/toolkit/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbV2Test.java new file mode 100644 index 00000000..ab1067fe --- /dev/null +++ b/tools/toolkit/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbV2Test.java @@ -0,0 +1,13 @@ +package org.tron.plugins.rocksdb; + +import java.io.IOException; +import org.junit.Test; +import org.tron.plugins.DbLiteTest; + +public class DbLiteRocksDbV2Test extends DbLiteTest { + + @Test + public void testToolsWithRocksDB() throws InterruptedException, IOException { + testTools("ROCKSDB", 2); + } +} diff --git a/tools/toolkit/src/test/java/org/tron/plugins/utils/PublicMethod.java b/tools/toolkit/src/test/java/org/tron/plugins/utils/PublicMethod.java index ce191c83..5e21a693 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/utils/PublicMethod.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/utils/PublicMethod.java @@ -6,22 +6,28 @@ import java.net.InetAddress; import java.net.Socket; import java.util.List; +import java.util.Objects; import java.util.Random; +import org.bouncycastle.util.encoders.Hex; import org.tron.api.GrpcAPI; import org.tron.api.WalletGrpc; import org.tron.common.crypto.ECKey; import org.tron.common.crypto.ECKey.ECDSASignature; import org.tron.common.parameter.CommonParameter; import org.tron.common.utils.Sha256Hash; +import org.tron.common.utils.Utils; import org.tron.core.Wallet; -import org.tron.protos.Protocol; import org.tron.protos.Protocol.Transaction; import org.tron.protos.Protocol.Transaction.Contract; -import org.tron.protos.contract.BalanceContract; import org.tron.protos.contract.BalanceContract.TransferContract; public class PublicMethod { + public static String getRandomPrivateKey() { + return Hex.toHexString(Objects + .requireNonNull(new ECKey(Utils.getRandom()).getPrivKeyBytes())); + } + /** * Transfer TRX. * @param to addr receives the asset @@ -47,15 +53,15 @@ public static Boolean sendcoin(byte[] to, long amount, byte[] owner, String priK while (times++ <= 2) { TransferContract.Builder builder = - BalanceContract.TransferContract.newBuilder(); - com.google.protobuf.ByteString bsTo = com.google.protobuf.ByteString.copyFrom(to); - com.google.protobuf.ByteString bsOwner = ByteString.copyFrom(owner); + TransferContract.newBuilder(); + ByteString bsTo = ByteString.copyFrom(to); + ByteString bsOwner = ByteString.copyFrom(owner); builder.setToAddress(bsTo); builder.setOwnerAddress(bsOwner); builder.setAmount(amount); TransferContract contract = builder.build(); - Protocol.Transaction transaction = blockingStubFull.createTransaction(contract); + Transaction transaction = blockingStubFull.createTransaction(contract); if (transaction == null || transaction.getRawData().getContractCount() == 0) { continue; } @@ -71,8 +77,8 @@ public static Boolean sendcoin(byte[] to, long amount, byte[] owner, String priK * @param ecKey ecKey of the private key * @param transaction transaction object */ - public static Protocol.Transaction signTransaction(ECKey ecKey, - Protocol.Transaction transaction) { + public static Transaction signTransaction(ECKey ecKey, + Transaction transaction) { if (ecKey == null || ecKey.getPrivKey() == null) { return null; } @@ -83,7 +89,7 @@ public static Protocol.Transaction signTransaction(ECKey ecKey, public static Transaction setTimestamp(Transaction transaction) { long currentTime = System.currentTimeMillis();//*1000000 + System.nanoTime()%1000000; Transaction.Builder builder = transaction.toBuilder(); - org.tron.protos.Protocol.Transaction.raw.Builder rowBuilder = transaction.getRawData() + Transaction.raw.Builder rowBuilder = transaction.getRawData() .toBuilder(); rowBuilder.setTimestamp(currentTime); builder.setRawData(rowBuilder.build()); @@ -113,7 +119,7 @@ public static Transaction sign(Transaction transaction, ECKey myKey) { * @param blockingStubFull Grpc interface */ public static GrpcAPI.Return broadcastTransaction( - Protocol.Transaction transaction, WalletGrpc.WalletBlockingStub blockingStubFull) { + Transaction transaction, WalletGrpc.WalletBlockingStub blockingStubFull) { int i = 10; GrpcAPI.Return response = blockingStubFull.broadcastTransaction(transaction); while (!response.getResult() && response.getCode() == GrpcAPI.Return.response_code.SERVER_BUSY diff --git a/tools/toolkit/src/test/resources/config-localtest.conf b/tools/toolkit/src/test/resources/config-localtest.conf index b280997c..8049ceb6 100644 --- a/tools/toolkit/src/test/resources/config-localtest.conf +++ b/tools/toolkit/src/test/resources/config-localtest.conf @@ -90,7 +90,7 @@ node { minParticipationRate = 0 - fullNodeAllowShieldedTransaction = true + allowShieldedTransactionApi = true zenTokenId = 1000001 @@ -117,14 +117,21 @@ node { ] http { + fullNodeEnable = false fullNodePort = 8090 + solidityEnable = false solidityPort = 8091 + PBFTEnable = false + PBFTPort = 8092 } rpc { + enable = false port = 50051 - # default value is 50061 - # solidityPort = 50061 + solidityEnable = false + solidityPort = 50061 + PBFTEnable = false + PBFTPort = 50071 # Number of gRPC thread, default availableProcessors / 2 # thread = 16 @@ -158,6 +165,9 @@ node { # httpSolidityPort = 8555 # httpPBFTEnable = true # httpPBFTPort = 8565 + # maxBlockRange = 5000 + # maxSubTopics = 1000 + # maxBlockFilterNum = 30000 } } @@ -209,7 +219,7 @@ genesis.block = { witnesses = [ { - address: "TN3zfjYUmMFK3ZsHSsrdJoNRtGkQmZLBLz" + address: TEDapYSVvAZ3aYH7w8N9tMEEFKaNKUD5Bp url = "http://Test.org", voteCount = 106 }, @@ -246,7 +256,7 @@ genesis.block = { //localWitnessAccountAddress = TN3zfjYUmMFK3ZsHSsrdJoNRtGkQmZLBLz localwitness = [ - f4df789d3210ac881cb900464dd30409453044d2777060a0c391cbdf4c6a4f57 + 1234567890123456789012345678901234567890123456789012345678901234 ] diff --git a/tools/toolkit/src/test/resources/config.conf b/tools/toolkit/src/test/resources/config.conf index a596d68e..df7b703a 100644 --- a/tools/toolkit/src/test/resources/config.conf +++ b/tools/toolkit/src/test/resources/config.conf @@ -1,11 +1,5 @@ storage { - # Directory for storing persistent data - db.engine = "LEVELDB", - db.sync = false, - db.directory = "database", - index.directory = "index", - transHistory.switch = "on", properties = [ { name = "account", diff --git a/tools/toolkit/src/test/resources/fork.conf b/tools/toolkit/src/test/resources/fork.conf index 602eb4d6..a2516dcd 100644 --- a/tools/toolkit/src/test/resources/fork.conf +++ b/tools/toolkit/src/test/resources/fork.conf @@ -53,6 +53,8 @@ trc20Contracts = [ } ] -latestBlockHeaderTimestamp = 1739269661000 -maintenanceTimeInterval = 21600000 -nextMaintenanceTime = 1735628894000 +chainParameters = { + latestBlockHeaderTimestamp = 1739269661000 + maintenanceTimeInterval = 21600000 + nextMaintenanceTime = 1735628894000 +} diff --git a/tools/trond/README.md b/tools/trond/README.md index 0802d61b..90ff01a0 100644 --- a/tools/trond/README.md +++ b/tools/trond/README.md @@ -99,6 +99,33 @@ Here is an example of stopping the node using `trond`: For more detailed usage instructions, refer to the help command or the [command documentation](./docs/trond.md). +## Security Configuration + +### SSH Host Key Verification + +When using `trond` for remote node deployment, SSH connections are made to remote servers. By default, host key verification is disabled for ease of testing and demonstration. + +For detailed security configuration and best practices, see [SECURITY.md](SECURITY.md). + +**Quick Start for Production:** + +```bash +# Enable strict host key verification +export TROND_STRICT_HOST_KEY_CHECK=true + +# Add remote servers to known_hosts +ssh-keyscan -H remote-server-ip >> ~/.ssh/known_hosts + +# Then run trond commands +./trond node env-multi +./trond node run-multi start +``` + +For more information, refer to the security documentation for each command: +- [trond node env-multi](docs/trond_node_env-multi.md#security-configuration) +- [trond node run-multi](docs/trond_node_run-multi.md#security-configuration) +- [trond node run-multi stop](docs/trond_node_run-multi_stop.md#security-configuration) + ## TroubleShooting If you have any issues starting a java-tron node, please refer to the corresponding [TroubleShooting](../../single_node/README.md#troubleshot) guide. diff --git a/tools/trond/cmd/docker/build.go b/tools/trond/cmd/docker/build.go index 82074870..92cc2e73 100644 --- a/tools/trond/cmd/docker/build.go +++ b/tools/trond/cmd/docker/build.go @@ -2,11 +2,9 @@ package docker import ( "fmt" - "strings" "github.com/MakeNowJust/heredoc/v2" "github.com/spf13/cobra" - "github.com/tronprotocol/tron-docker/utils" ) // buildCmd represents the snapshot source command @@ -22,39 +20,44 @@ var buildCmd = &cobra.Command{ # Using code from https://github.com/tronprotocol/java-tron.git $ ./trond docker build - # Build java-tron docker image with specified org, artifact and version + # Build java-tron docker image with specified org, artifact and version, targeting the amd64 platform # Using code from https://github.com/tronprotocol/java-tron.git $ ./trond docker build -o tronprotocol -a java-tron -v latest $ ./trond docker build -o tronprotocol -a java-tron -v latest -n mainnet - # Build java-tron docker image for nile testnet with specified org, artifact and version + # Build java-tron docker image for nile testnet with specified org, artifact and version, targeting the amd64 platform # Using code from https://github.com/tron-nile-testnet/nile-testnet.git $ ./trond docker build -o tronnile -a java-tron -v latest -n nile + + # Build java-tron docker image with specified org, artifact and version, targeting the arm64 platform + # Using code from https://github.com/tronprotocol/java-tron.git + $ ./trond docker build -o tronprotocol -a java-tron -v latest + $ ./trond docker build -o tronprotocol -a java-tron -v latest -n mainnet -p linux/arm64 + + # Build java-tron docker image for nile testnet with specified org, artifact and version, targeting the arm64 platform + # Using code from https://github.com/tron-nile-testnet/nile-testnet.git + $ ./trond docker build -o tronnile -a java-tron -v latest -n nile -p linux/arm64 `), Run: func(cmd *cobra.Command, args []string) { - if yes, err := utils.IsJDK1_8(); err != nil || !yes { + /* if yes, err := utils.IsJDK1_8(); err != nil || !yes { fmt.Println("Error: JDK version should be 1.8") return - } + } */ // Get the flag value org, _ := cmd.Flags().GetString("org") artifact, _ := cmd.Flags().GetString("artifact") version, _ := cmd.Flags().GetString("version") network, _ := cmd.Flags().GetString("network") + platform, _ := cmd.Flags().GetString("platform") fmt.Println("The building progress may take a long time, depending on your network speed.") fmt.Println("If you don't specify the flags for building, the default values will be used.") fmt.Println("The default result will be: tronprotocol/java-tron:latest") fmt.Println("Start building...") - cmd1 := fmt.Sprintf("./gradlew --no-daemon sourceDocker -PdockerOrgName=%s -PdockerArtifactName=%s -Prelease.releaseVersion=%s", org, artifact, version) - if len(network) > 0 { - cmd1 = fmt.Sprintf("./gradlew --no-daemon sourceDocker -PdockerOrgName=%s -PdockerArtifactName=%s -Prelease.releaseVersion=%s -Pnetwork=%s", org, artifact, version, network) - } - cmds := []string{cmd1} - if err := utils.RunMultipleCommands(strings.Join(cmds, " && "), "./tools/gradlew"); err != nil { + if err := runGradleDockerTask("sourceDocker", org, artifact, version, network, platform); err != nil { fmt.Println("Error: ", err) return } @@ -67,6 +70,7 @@ func init() { buildCmd.Flags().StringP("artifact", "a", "java-tron", "ArtifactName for the docker image") buildCmd.Flags().StringP("version", "v", "latest", "Release version for the docker image") buildCmd.Flags().StringP("network", "n", "mainnet", "Which code will be used for the docker image, mainnet or nile") + buildCmd.Flags().StringP("platform", "p", "linux/amd64", "Platform for the docker image, linux/amd64 or linux/arm64") DockerCmd.AddCommand(buildCmd) } diff --git a/tools/trond/cmd/docker/gradle.go b/tools/trond/cmd/docker/gradle.go new file mode 100644 index 00000000..5c21dba0 --- /dev/null +++ b/tools/trond/cmd/docker/gradle.go @@ -0,0 +1,63 @@ +package docker + +import ( + "fmt" + "os" + "os/exec" + "regexp" +) + +var ( + dockerNamePattern = regexp.MustCompile(`^[a-z0-9]+(?:[._-][a-z0-9]+)*$`) + versionTagPattern = regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$`) +) + +func validateDockerGradleFlags(org, artifact, version, network, platform string) error { + if !dockerNamePattern.MatchString(org) { + return fmt.Errorf("invalid org %q: only lowercase letters, digits, '.', '_' and '-' are allowed", org) + } + if !dockerNamePattern.MatchString(artifact) { + return fmt.Errorf("invalid artifact %q: only lowercase letters, digits, '.', '_' and '-' are allowed", artifact) + } + if !versionTagPattern.MatchString(version) { + return fmt.Errorf("invalid version %q: must match Docker tag format", version) + } + if network != "mainnet" && network != "nile" && network != "" { + return fmt.Errorf("invalid network %q: allowed values are mainnet or nile", network) + } + if platform != "linux/amd64" && platform != "linux/arm64" && platform != "" { + return fmt.Errorf("invalid platform %q: allowed values are linux/amd64 or linux/arm64", platform) + } + return nil +} + +func runGradleDockerTask(task, org, artifact, version, network, platform string) error { + if err := validateDockerGradleFlags(org, artifact, version, network, platform); err != nil { + return err + } + + args := []string{ + "--no-daemon", + task, + fmt.Sprintf("-PdockerOrgName=%s", org), + fmt.Sprintf("-PdockerArtifactName=%s", artifact), + fmt.Sprintf("-Prelease.releaseVersion=%s", version), + } + + if network != "" { + args = append(args, fmt.Sprintf("-Pnetwork=%s", network)) + } + if platform != "" { + args = append(args, fmt.Sprintf("-Pplatform=%s", platform)) + } + + cmd := exec.Command("./gradlew", args...) + cmd.Dir = "./tools/gradlew" + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if err := cmd.Run(); err != nil { + return fmt.Errorf("gradle task %s failed: %w", task, err) + } + return nil +} diff --git a/tools/trond/cmd/docker/test.go b/tools/trond/cmd/docker/test.go index 45f15c5b..56b1b1e5 100644 --- a/tools/trond/cmd/docker/test.go +++ b/tools/trond/cmd/docker/test.go @@ -2,11 +2,9 @@ package docker import ( "fmt" - "strings" "github.com/MakeNowJust/heredoc/v2" "github.com/spf13/cobra" - "github.com/tronprotocol/tron-docker/utils" ) // testCmd represents the snapshot source command @@ -25,36 +23,36 @@ var testCmd = &cobra.Command{ It is used to test TRON Mainnet block synchronization within the docker container. It has a timeout limit of 1000 seconds and may encounter the error “ERROR: goss_wait.yaml never passed” due to network issues or timeouts. If this happens, please check your network status and try again. `), Example: heredoc.Doc(` - # Build java-tron docker image, defualt: tronprotocol/java-tron:latest + # Test java-tron docker image, defualt: tronprotocol/java-tron:latest $ ./trond docker test - # Build java-tron docker image with specified org, artifact and version + # Test java-tron docker image for amd64 with specified org, artifact and version $ ./trond docker test -o tronprotocol -a java-tron -v latest $ ./trond docker test -o tronnile -a java-tron -v latest -n nile + + # Test java-tron docker image for arm64 with specified org, artifact and version + $ ./trond docker test -o tronprotocol -a java-tron -v latest -p linux/arm64 + $ ./trond docker test -o tronnile -a java-tron -v latest -n nile -p linux/arm64 `), Run: func(cmd *cobra.Command, args []string) { - if yes, err := utils.IsJDK1_8(); err != nil || !yes { + /* if yes, err := utils.IsJDK1_8(); err != nil || !yes { fmt.Println("Error: JDK version should be 1.8") return - } + } */ // Get the flag value org, _ := cmd.Flags().GetString("org") artifact, _ := cmd.Flags().GetString("artifact") version, _ := cmd.Flags().GetString("version") network, _ := cmd.Flags().GetString("network") + platform, _ := cmd.Flags().GetString("platform") fmt.Println("If you don't specify the flags for building, the default values will be used.") fmt.Println("The default result will be: tronprotocol/java-tron:latest") fmt.Println("Start testing...") - cmd1 := fmt.Sprintf("./gradlew --no-daemon testDocker -PdockerOrgName=%s -PdockerArtifactName=%s -Prelease.releaseVersion=%s", org, artifact, version) - if len(network) > 0 { - cmd1 = fmt.Sprintf("./gradlew --no-daemon testDocker -PdockerOrgName=%s -PdockerArtifactName=%s -Prelease.releaseVersion=%s -Pnetwork=%s", org, artifact, version, network) - } - cmds := []string{cmd1} - if err := utils.RunMultipleCommands(strings.Join(cmds, " && "), "./tools/gradlew"); err != nil { + if err := runGradleDockerTask("testDocker", org, artifact, version, network, platform); err != nil { fmt.Println("Error: ", err) } }, @@ -66,6 +64,7 @@ func init() { testCmd.Flags().StringP("artifact", "a", "java-tron", "ArtifactName for the docker image") testCmd.Flags().StringP("version", "v", "latest", "Release version for the docker image") testCmd.Flags().StringP("network", "n", "mainnet", "Which code will be used for the docker image, mainnet or nile") + testCmd.Flags().StringP("platform", "p", "linux/amd64", "Platform for the docker image, linux/amd64 or linux/arm64") DockerCmd.AddCommand(testCmd) } diff --git a/tools/trond/cmd/node/envMultiNodes.go b/tools/trond/cmd/node/envMultiNodes.go index 4f299f7b..b81ae437 100644 --- a/tools/trond/cmd/node/envMultiNodes.go +++ b/tools/trond/cmd/node/envMultiNodes.go @@ -24,9 +24,46 @@ var envMultiCmd = &cobra.Command{ - Configuration file for private network layout (Please refer to the example configuration file and rewrite it according to your needs) ./conf/private_net_layout.toml + + SECURITY CONFIGURATION: + + When deploying to remote nodes, SSH connections are used to transfer configuration files and manage the environment. By default, host key verification is disabled for ease of testing. + + For Production Environments: + + Enable strict host key checking to prevent man-in-the-middle attacks: + + # Enable strict host key verification + export TROND_STRICT_HOST_KEY_CHECK=true + + # Optional: Specify custom known_hosts file location + export TROND_KNOWN_HOSTS_FILE=/path/to/your/known_hosts + + # Then run the command + ./trond node env-multi + + Setting up known_hosts file: + + 1. First, manually SSH to each remote server to add it to known_hosts: + ssh user@remote-server-ip + + 2. Or use ssh-keyscan to add host keys: + ssh-keyscan -H remote-server-ip >> ~/.ssh/known_hosts + + For Testing/Development: + + Host key verification is disabled by default. You'll see a warning message: + ⚠️ WARNING: Host key verification is DISABLED (testing mode) + ⚠️ For production use, set TROND_STRICT_HOST_KEY_CHECK=true + + For more details, see SECURITY.md `), Example: heredoc.Doc(` - # Check and configure node local environment + # Check and configure node local environment (testing mode) + $ ./trond node env-multi + + # Check and configure with strict host key verification (production mode) + $ export TROND_STRICT_HOST_KEY_CHECK=true $ ./trond node env-multi # Use the scp command to copy files and synchronize databases between multiple nodes: diff --git a/tools/trond/cmd/node/runMulti.go b/tools/trond/cmd/node/runMulti.go index bd75d421..b0596e8f 100644 --- a/tools/trond/cmd/node/runMulti.go +++ b/tools/trond/cmd/node/runMulti.go @@ -19,9 +19,46 @@ var runMultiCmd = &cobra.Command{ - Configuration file for private network layout (Please refer to the example configuration file and rewrite it according to your needs) ./conf/private_net_layout.toml + SECURITY CONFIGURATION: + + When deploying to remote nodes, SSH connections are used to start and manage docker-compose services. By default, host key verification is disabled for ease of testing. + + For Production Environments: + + Enable strict host key checking to prevent man-in-the-middle attacks: + + # Enable strict host key verification + export TROND_STRICT_HOST_KEY_CHECK=true + + # Optional: Specify custom known_hosts file location + export TROND_KNOWN_HOSTS_FILE=/path/to/your/known_hosts + + # Then run the command + ./trond node run-multi + + Setting up known_hosts file: + + 1. First, manually SSH to each remote server to add it to known_hosts: + ssh user@remote-server-ip + + 2. Or use ssh-keyscan to add host keys: + ssh-keyscan -H remote-server-ip >> ~/.ssh/known_hosts + + For Testing/Development: + + Host key verification is disabled by default. You'll see a warning message: + ⚠️ WARNING: Host key verification is DISABLED (testing mode) + ⚠️ For production use, set TROND_STRICT_HOST_KEY_CHECK=true + + For more details, see SECURITY.md + `), Example: heredoc.Doc(` - # Run java-tron nodes according to ./conf/private_net_layout.toml + # Run java-tron nodes according to ./conf/private_net_layout.toml (testing mode) + $ ./trond node run-multi + + # Run with strict host key verification (production mode) + $ export TROND_STRICT_HOST_KEY_CHECK=true $ ./trond node run-multi `), Run: func(cmd *cobra.Command, args []string) { @@ -75,9 +112,38 @@ var runMultiStopCmd = &cobra.Command{ - Configuration file(by default, these exist in the current repository directory) ./conf/private_net_layout.toml + SECURITY CONFIGURATION: + + When stopping remote nodes, SSH connections are used to execute docker-compose commands. By default, host key verification is disabled for ease of testing. + + For Production Environments: + + Enable strict host key checking to prevent man-in-the-middle attacks: + + # Enable strict host key verification + export TROND_STRICT_HOST_KEY_CHECK=true + + # Optional: Specify custom known_hosts file location + export TROND_KNOWN_HOSTS_FILE=/path/to/your/known_hosts + + # Then run the command + ./trond node run-multi stop + + For Testing/Development: + + Host key verification is disabled by default. You'll see a warning message: + ⚠️ WARNING: Host key verification is DISABLED (testing mode) + ⚠️ For production use, set TROND_STRICT_HOST_KEY_CHECK=true + + For more details, see SECURITY.md + `), Example: heredoc.Doc(` - # Stop multi java-tron node + # Stop multi java-tron node (testing mode) + $ ./trond node run-multi stop + + # Stop with strict host key verification (production mode) + $ export TROND_STRICT_HOST_KEY_CHECK=true $ ./trond node run-multi stop `), Run: func(cmd *cobra.Command, args []string) { diff --git a/tools/trond/docs/trond_docker_build.md b/tools/trond/docs/trond_docker_build.md index 9292bc41..d85303c4 100644 --- a/tools/trond/docs/trond_docker_build.md +++ b/tools/trond/docs/trond_docker_build.md @@ -19,15 +19,24 @@ trond docker build [flags] # Using code from https://github.com/tronprotocol/java-tron.git $ ./trond docker build -# Build java-tron docker image with specified org, artifact and version +# Build java-tron docker image with specified org, artifact and version, targeting the amd64 platform # Using code from https://github.com/tronprotocol/java-tron.git $ ./trond docker build -o tronprotocol -a java-tron -v latest $ ./trond docker build -o tronprotocol -a java-tron -v latest -n mainnet -# Build java-tron docker image for nile testnet with specified org, artifact and version +# Build java-tron docker image for nile testnet with specified org, artifact and version, targeting the amd64 platform # Using code from https://github.com/tron-nile-testnet/nile-testnet.git $ ./trond docker build -o tronnile -a java-tron -v latest -n nile +# Build java-tron docker image with specified org, artifact and version, targeting the arm64 platform +# Using code from https://github.com/tronprotocol/java-tron.git +$ ./trond docker build -o tronprotocol -a java-tron -v latest +$ ./trond docker build -o tronprotocol -a java-tron -v latest -n mainnet -p linux/arm64 + +# Build java-tron docker image for nile testnet with specified org, artifact and version, targeting the arm64 platform +# Using code from https://github.com/tron-nile-testnet/nile-testnet.git +$ ./trond docker build -o tronnile -a java-tron -v latest -n nile -p linux/arm64 + ``` ### Options @@ -37,6 +46,7 @@ $ ./trond docker build -o tronnile -a java-tron -v latest -n nile -h, --help help for build -n, --network string Which code will be used for the docker image, mainnet or nile (default "mainnet") -o, --org string OrgName for the docker image (default "tronprotocol") + -p, --platform string Platform for the docker image, linux/amd64 or linux/arm64 (default "linux/amd64") -v, --version string Release version for the docker image (default "latest") ``` diff --git a/tools/trond/docs/trond_docker_test.md b/tools/trond/docs/trond_docker_test.md index d2b49d29..a88289a9 100644 --- a/tools/trond/docs/trond_docker_test.md +++ b/tools/trond/docs/trond_docker_test.md @@ -22,13 +22,17 @@ trond docker test [flags] ### Examples ``` -# Build java-tron docker image, defualt: tronprotocol/java-tron:latest +# Test java-tron docker image, defualt: tronprotocol/java-tron:latest $ ./trond docker test -# Build java-tron docker image with specified org, artifact and version +# Test java-tron docker image for amd64 with specified org, artifact and version $ ./trond docker test -o tronprotocol -a java-tron -v latest $ ./trond docker test -o tronnile -a java-tron -v latest -n nile +# Test java-tron docker image for arm64 with specified org, artifact and version +$ ./trond docker test -o tronprotocol -a java-tron -v latest -p linux/arm64 +$ ./trond docker test -o tronnile -a java-tron -v latest -n nile -p linux/arm64 + ``` ### Options @@ -38,6 +42,7 @@ $ ./trond docker test -o tronnile -a java-tron -v latest -n nile -h, --help help for test -n, --network string Which code will be used for the docker image, mainnet or nile (default "mainnet") -o, --org string OrgName for the docker image (default "tronprotocol") + -p, --platform string Platform for the docker image, linux/amd64 or linux/arm64 (default "linux/amd64") -v, --version string Release version for the docker image (default "latest") ``` diff --git a/tools/trond/docs/trond_node_env-multi.md b/tools/trond/docs/trond_node_env-multi.md index 42874a2f..750c7fcd 100644 --- a/tools/trond/docs/trond_node_env-multi.md +++ b/tools/trond/docs/trond_node_env-multi.md @@ -15,6 +15,39 @@ The following files are required: - Configuration file for private network layout (Please refer to the example configuration file and rewrite it according to your needs) ./conf/private_net_layout.toml +SECURITY CONFIGURATION: + +When deploying to remote nodes, SSH connections are used to transfer configuration files and manage the environment. By default, host key verification is disabled for ease of testing. + +For Production Environments: + +Enable strict host key checking to prevent man-in-the-middle attacks: + + # Enable strict host key verification + export TROND_STRICT_HOST_KEY_CHECK=true + + # Optional: Specify custom known_hosts file location + export TROND_KNOWN_HOSTS_FILE=/path/to/your/known_hosts + + # Then run the command + ./trond node env-multi + +Setting up known_hosts file: + +1. First, manually SSH to each remote server to add it to known_hosts: + ssh user@remote-server-ip + +2. Or use ssh-keyscan to add host keys: + ssh-keyscan -H remote-server-ip >> ~/.ssh/known_hosts + +For Testing/Development: + +Host key verification is disabled by default. You'll see a warning message: + ⚠️ WARNING: Host key verification is DISABLED (testing mode) + ⚠️ For production use, set TROND_STRICT_HOST_KEY_CHECK=true + +For more details, see SECURITY.md + ``` trond node env-multi [flags] @@ -23,7 +56,11 @@ trond node env-multi [flags] ### Examples ``` -# Check and configure node local environment +# Check and configure node local environment (testing mode) +$ ./trond node env-multi + +# Check and configure with strict host key verification (production mode) +$ export TROND_STRICT_HOST_KEY_CHECK=true $ ./trond node env-multi # Use the scp command to copy files and synchronize databases between multiple nodes: diff --git a/tools/trond/docs/trond_node_run-multi.md b/tools/trond/docs/trond_node_run-multi.md index edb163d2..5c28316a 100644 --- a/tools/trond/docs/trond_node_run-multi.md +++ b/tools/trond/docs/trond_node_run-multi.md @@ -11,6 +11,39 @@ The following files are required: - Configuration file for private network layout (Please refer to the example configuration file and rewrite it according to your needs) ./conf/private_net_layout.toml +SECURITY CONFIGURATION: + +When deploying to remote nodes, SSH connections are used to start and manage docker-compose services. By default, host key verification is disabled for ease of testing. + +For Production Environments: + +Enable strict host key checking to prevent man-in-the-middle attacks: + + # Enable strict host key verification + export TROND_STRICT_HOST_KEY_CHECK=true + + # Optional: Specify custom known_hosts file location + export TROND_KNOWN_HOSTS_FILE=/path/to/your/known_hosts + + # Then run the command + ./trond node run-multi + +Setting up known_hosts file: + +1. First, manually SSH to each remote server to add it to known_hosts: + ssh user@remote-server-ip + +2. Or use ssh-keyscan to add host keys: + ssh-keyscan -H remote-server-ip >> ~/.ssh/known_hosts + +For Testing/Development: + +Host key verification is disabled by default. You'll see a warning message: + ⚠️ WARNING: Host key verification is DISABLED (testing mode) + ⚠️ For production use, set TROND_STRICT_HOST_KEY_CHECK=true + +For more details, see SECURITY.md + ``` @@ -20,7 +53,11 @@ trond node run-multi [flags] ### Examples ``` -# Run java-tron nodes according to ./conf/private_net_layout.toml +# Run java-tron nodes according to ./conf/private_net_layout.toml (testing mode) +$ ./trond node run-multi + +# Run with strict host key verification (production mode) +$ export TROND_STRICT_HOST_KEY_CHECK=true $ ./trond node run-multi ``` diff --git a/tools/trond/docs/trond_node_run-multi_stop.md b/tools/trond/docs/trond_node_run-multi_stop.md index 275571d7..340a410d 100644 --- a/tools/trond/docs/trond_node_run-multi_stop.md +++ b/tools/trond/docs/trond_node_run-multi_stop.md @@ -9,6 +9,31 @@ The following configuration files are required: - Configuration file(by default, these exist in the current repository directory) ./conf/private_net_layout.toml +SECURITY CONFIGURATION: + +When stopping remote nodes, SSH connections are used to execute docker-compose commands. By default, host key verification is disabled for ease of testing. + +For Production Environments: + +Enable strict host key checking to prevent man-in-the-middle attacks: + + # Enable strict host key verification + export TROND_STRICT_HOST_KEY_CHECK=true + + # Optional: Specify custom known_hosts file location + export TROND_KNOWN_HOSTS_FILE=/path/to/your/known_hosts + + # Then run the command + ./trond node run-multi stop + +For Testing/Development: + +Host key verification is disabled by default. You'll see a warning message: + ⚠️ WARNING: Host key verification is DISABLED (testing mode) + ⚠️ For production use, set TROND_STRICT_HOST_KEY_CHECK=true + +For more details, see SECURITY.md + ``` @@ -18,7 +43,11 @@ trond node run-multi stop [flags] ### Examples ``` -# Stop multi java-tron node +# Stop multi java-tron node (testing mode) +$ ./trond node run-multi stop + +# Stop with strict host key verification (production mode) +$ export TROND_STRICT_HOST_KEY_CHECK=true $ ./trond node run-multi stop ``` diff --git a/tools/trond/utils/http.go b/tools/trond/utils/http.go index a319cb06..41dbb8ea 100644 --- a/tools/trond/utils/http.go +++ b/tools/trond/utils/http.go @@ -21,6 +21,44 @@ import ( "golang.org/x/net/html" ) +// secureResolveWithinDir resolves a path through the real filesystem (following symlinks) +// and verifies the result stays within the allowed directory. +// This is the single, authoritative boundary check used by the tar extractor. +func secureResolveWithinDir(destDir, targetPath string) error { + // Resolve destDir itself to an absolute, symlink-free path + absDestDir, err := filepath.Abs(destDir) + if err != nil { + return fmt.Errorf("failed to resolve destination directory: %v", err) + } + // EvalSymlinks on destDir (which must already exist) + realDestDir, err := filepath.EvalSymlinks(absDestDir) + if err != nil { + return fmt.Errorf("failed to resolve destination directory symlinks: %v", err) + } + + // Resolve the target through the real filesystem. + // Walk up to the deepest existing ancestor, then validate the rest lexically. + realTarget, err := filepath.EvalSymlinks(targetPath) + if err != nil { + // If the full path doesn't exist yet, resolve the parent (which should exist + // because we create dirs in order) and append the base name. + parentDir := filepath.Dir(targetPath) + realParent, err2 := filepath.EvalSymlinks(parentDir) + if err2 != nil { + return fmt.Errorf("failed to resolve parent path %q: %v", parentDir, err2) + } + realTarget = filepath.Join(realParent, filepath.Base(targetPath)) + } + + realTarget = filepath.Clean(realTarget) + prefix := realDestDir + string(os.PathSeparator) + + if realTarget != realDestDir && !strings.HasPrefix(realTarget, prefix) { + return fmt.Errorf("path %q resolves to %q which is outside destination %q", targetPath, realTarget, realDestDir) + } + return nil +} + // Function to fetch and parse HTML, extracting absolute links func fetchAndExtractLinks(webURL string) ([]string, error) { resp, err := http.Get(webURL) @@ -460,75 +498,121 @@ func ExtractTgzWithStatus(tgzFile, destDir string) error { if err := os.MkdirAll(target, os.FileMode(header.Mode)); err != nil { return fmt.Errorf("failed to create directory: %v", err) } + // Verify the created directory resolves within destDir + if err := secureResolveWithinDir(destDir, target); err != nil { + return fmt.Errorf("directory traversal detected in dir entry %q: %v", header.Name, err) + } + case tar.TypeReg: // Ensure parent directory exists if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { return fmt.Errorf("failed to create parent directory: %v", err) } - // Create file - outFile, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY, os.FileMode(header.Mode)) + // Verify the target path resolves within destDir BEFORE creating the file. + // This catches traversal via previously extracted symlinks in the path. + if err := secureResolveWithinDir(destDir, target); err != nil { + return fmt.Errorf("directory traversal detected in file entry %q: %v", header.Name, err) + } + + // Create file with explicit O_NOFOLLOW-like behavior: + // use Lstat to check the target doesn't already exist as a symlink + if info, err := os.Lstat(target); err == nil && info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("refusing to write through existing symlink: %s", header.Name) + } + + outFile, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(header.Mode)) if err != nil { return fmt.Errorf("failed to create file: %v", err) } - defer outFile.Close() // Read data in chunks and write to the output file buf := make([]byte, 32*1024) // 32KB buffer for reading for { - n, err := tr.Read(buf) + n, readErr := tr.Read(buf) if n > 0 { - // Write the data to the file if _, writeErr := outFile.Write(buf[:n]); writeErr != nil { + outFile.Close() return fmt.Errorf("failed to write file: %v", writeErr) } - - // Update the total extracted size and file count totalExtractedSize += int64(n) } - if err == io.EOF { + if readErr == io.EOF { break } - if err != nil { - return fmt.Errorf("error reading file: %v", err) + if readErr != nil { + outFile.Close() + return fmt.Errorf("error reading file: %v", readErr) } } + outFile.Close() - // Increment the file count totalFilesExtracted++ + case tar.TypeSymlink: - // Resolve the symlink target - resolvedLinkname, err := filepath.EvalSymlinks(filepath.Join(filepath.Dir(target), header.Linkname)) + // Validate: the symlink TARGET must resolve within destDir. + // Compute what the symlink would point to. + linkTarget := filepath.Join(filepath.Dir(target), header.Linkname) + linkTarget = filepath.Clean(linkTarget) + + // Validate the resolved link target stays within destDir. + // Use lexical check first (works even if target doesn't exist yet). + absDestDir, err := filepath.Abs(destDir) + if err != nil { + return fmt.Errorf("failed to resolve destDir: %v", err) + } + absLinkTarget, err := filepath.Abs(linkTarget) if err != nil { return fmt.Errorf("failed to resolve symlink target: %v", err) } - // Sanitize symlink target to prevent directory traversal - if !strings.HasPrefix(filepath.Clean(resolvedLinkname), filepath.Clean(destDir)+string(os.PathSeparator)) { - return fmt.Errorf("invalid symlink target in archive: %s -> %s", header.Name, header.Linkname) + if absLinkTarget != absDestDir && !strings.HasPrefix(absLinkTarget, absDestDir+string(os.PathSeparator)) { + return fmt.Errorf("symlink %q -> %q escapes destination directory (resolves to %q)", header.Name, header.Linkname, absLinkTarget) + } + + // If the parent directory of the symlink target exists on the filesystem, + // also verify through EvalSymlinks (catches multi-hop symlink chains). + if _, statErr := os.Lstat(filepath.Dir(linkTarget)); statErr == nil { + if err := secureResolveWithinDir(destDir, linkTarget); err != nil { + return fmt.Errorf("symlink chain traversal detected: %q -> %q: %v", header.Name, header.Linkname, err) + } + } + + // Also verify the symlink itself will be placed within destDir + if err := secureResolveWithinDir(destDir, filepath.Dir(target)); err != nil { + return fmt.Errorf("symlink location traversal detected: %q: %v", header.Name, err) } - // Create symlink - if err := os.Symlink(resolvedLinkname, target); err != nil { + // Remove any existing entry at target to prevent symlink-following attacks + os.Remove(target) + + // Create symlink using the ORIGINAL relative linkname (not an absolute resolved path) + if err := os.Symlink(header.Linkname, target); err != nil { return fmt.Errorf("failed to create symlink: %v", err) } + case tar.TypeLink: - // Create hard link + // Hard link: both source and target must resolve within destDir linkTarget := filepath.Join(destDir, header.Linkname) - // Sanitize the link target to prevent directory traversal - // Resolve the hard link target + + // Verify the hard link target resolves within destDir + if err := secureResolveWithinDir(destDir, linkTarget); err != nil { + return fmt.Errorf("hard link target traversal: %q -> %q: %v", header.Name, header.Linkname, err) + } + // Verify the hard link location resolves within destDir + if err := secureResolveWithinDir(destDir, filepath.Dir(target)); err != nil { + return fmt.Errorf("hard link location traversal: %q: %v", header.Name, err) + } + + // Use the RESOLVED target for creating the hard link (consistent with validation) resolvedLinkTarget, err := filepath.EvalSymlinks(linkTarget) if err != nil { return fmt.Errorf("failed to resolve hard link target: %v", err) } - // Ensure the resolved hard link target stays within the destination directory - if !strings.HasPrefix(filepath.Clean(resolvedLinkTarget), filepath.Clean(destDir)+string(os.PathSeparator)) { - return fmt.Errorf("attempted directory traversal in hard link: %s -> %s", header.Name, header.Linkname) - } - - if err := os.Link(linkTarget, target); err != nil { + if err := os.Link(resolvedLinkTarget, target); err != nil { return fmt.Errorf("failed to create hard link: %v", err) } + default: fmt.Printf("Skipping unknown file type: %v\n", header.Typeflag) } diff --git a/tools/trond/utils/ssh.go b/tools/trond/utils/ssh.go index b36bfc5c..3a96bf51 100644 --- a/tools/trond/utils/ssh.go +++ b/tools/trond/utils/ssh.go @@ -6,13 +6,86 @@ import ( "net" "os" "path/filepath" + "regexp" + "strings" "time" "github.com/schollz/progressbar/v3" "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/agent" + "golang.org/x/crypto/ssh/knownhosts" ) +// validPathPattern allows only safe characters in remote paths: +// alphanumeric, slash, dot, hyphen, underscore +var validPathPattern = regexp.MustCompile(`^[a-zA-Z0-9/._-]+$`) + +// validateRemotePath checks that a path contains no shell-injectable characters. +// This is the primary defense against command injection via user-controlled paths. +func validateRemotePath(path string) error { + if path == "" { + return fmt.Errorf("path cannot be empty") + } + if !validPathPattern.MatchString(path) { + return fmt.Errorf("invalid characters in remote path %q: only alphanumeric, '/', '.', '-', '_' are allowed", path) + } + // Block path traversal attempts + if strings.Contains(path, "..") { + return fmt.Errorf("path traversal ('..') is not allowed in remote path: %s", path) + } + return nil +} + +// validateSCPFileName validates a filename for SCP transfer. +// SCP protocol is sensitive to special characters that could break the control line format. +// The control line format is: C \n +// Filenames with newlines, carriage returns, or other control characters can break this format. +func validateSCPFileName(fileName string) error { + if fileName == "" { + return fmt.Errorf("filename cannot be empty") + } + + // Check for path traversal attempts + if strings.Contains(fileName, "/") || strings.Contains(fileName, "\\") { + return fmt.Errorf("filename cannot contain path separators: %s", fileName) + } + + // Block path traversal attempts + if strings.Contains(fileName, "..") { + return fmt.Errorf("filename cannot contain path traversal sequences: %s", fileName) + } + + // Check for control characters that could break SCP protocol + // SCP control line ends with \n, so any \n or \r in filename breaks the protocol + for _, ch := range fileName { + // Block control characters (0x00-0x1F) and DEL (0x7F) + if ch < 0x20 || ch == 0x7F { + return fmt.Errorf("filename contains invalid control character (0x%02X): %s", ch, fileName) + } + // Block characters that have special meaning in SCP protocol or shell + // While we use proper escaping, defense-in-depth means rejecting these outright + if strings.ContainsRune("*?[]{}()<>|;&$`\\\"'", ch) { + return fmt.Errorf("filename contains shell-special character %q: %s", ch, fileName) + } + } + + // Ensure filename is not too long (SCP protocol limitation) + if len(fileName) > 255 { + return fmt.Errorf("filename exceeds maximum length (255 bytes): %s", fileName) + } + + return nil +} + +// shellQuote wraps a string in single quotes with proper escaping. +// This is the secondary defense — even if validation is bypassed, +// the shell will treat the entire string as a literal argument. +func shellQuote(s string) string { + // Replace each single quote with: end quote, escaped single quote, start quote + // e.g., "it's" -> "'it'\''s'" + return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'" +} + // Check if a port is open on the given IP func CheckPort(ip string, port int) bool { timeout := 5 * time.Second @@ -24,10 +97,54 @@ func CheckPort(ip string, port int) bool { return true } +// getHostKeyCallback returns the appropriate HostKeyCallback based on environment +// For production use, set TROND_STRICT_HOST_KEY_CHECK=true environment variable +func getHostKeyCallback() (ssh.HostKeyCallback, error) { + // Check if strict host key checking is enabled via environment variable + strictCheck := os.Getenv("TROND_STRICT_HOST_KEY_CHECK") + + if strictCheck == "true" || strictCheck == "1" { + // Production mode: Use known_hosts file for verification + knownHostsPath := os.Getenv("TROND_KNOWN_HOSTS_FILE") + if knownHostsPath == "" { + // Default to user's known_hosts file + homeDir, err := os.UserHomeDir() + if err != nil { + return nil, fmt.Errorf("failed to get home directory: %v", err) + } + knownHostsPath = filepath.Join(homeDir, ".ssh", "known_hosts") + } + + // Check if known_hosts file exists + if _, err := os.Stat(knownHostsPath); os.IsNotExist(err) { + return nil, fmt.Errorf("known_hosts file not found at %s. Please create it or disable strict host key checking for testing", knownHostsPath) + } + + callback, err := knownhosts.New(knownHostsPath) + if err != nil { + return nil, fmt.Errorf("failed to load known_hosts file: %v", err) + } + + fmt.Println(" ✓ Using strict host key verification (production mode)") + return callback, nil + } + + // Development/Testing mode: Accept any host key with warning + fmt.Println(" ⚠️ WARNING: Host key verification is DISABLED (testing mode)") + fmt.Println(" ⚠️ For production use, set TROND_STRICT_HOST_KEY_CHECK=true") + return ssh.InsecureIgnoreHostKey(), nil +} + func SSHConnect(ip string, port int, user, password, keyPath string) (*ssh.Client, error) { + // Get appropriate host key callback + hostKeyCallback, err := getHostKeyCallback() + if err != nil { + return nil, fmt.Errorf("failed to configure host key verification: %v", err) + } + sshConfig := &ssh.ClientConfig{ User: user, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), // Replace with known hosts verification in production + HostKeyCallback: hostKeyCallback, Timeout: 5 * time.Second, } @@ -139,14 +256,26 @@ func SCPFile(ip string, port int, user, password, keyPath, localPath, remotePath } defer stdin.Close() - // Start SCP command on the remote server - scpCmd := fmt.Sprintf("scp -t %s", filepath.Dir(remotePath)) + // Validate remote path before constructing shell command + remotePathDir := filepath.Dir(remotePath) + if err := validateRemotePath(remotePathDir); err != nil { + return fmt.Errorf("unsafe remote path for SCP: %v", err) + } + + // Start SCP command on the remote server (quoted to prevent injection) + scpCmd := fmt.Sprintf("scp -t %s", shellQuote(remotePathDir)) if err := session.Start(scpCmd); err != nil { return fmt.Errorf("failed to start SCP command: %v", err) } - // Send SCP file metadata - _, err = fmt.Fprintf(stdin, "C0644 %d %s\n", fileSize, filepath.Base(remotePath)) + // Validate and sanitize the filename + fileName := filepath.Base(remotePath) + if err := validateSCPFileName(fileName); err != nil { + return fmt.Errorf("unsafe filename for SCP: %v", err) + } + + // Send SCP file metadata with properly escaped filename + _, err = fmt.Fprintf(stdin, "C0644 %d %s\n", fileSize, fileName) if err != nil { return fmt.Errorf("failed to send file metadata: %v", err) } @@ -195,8 +324,13 @@ func SSHMkdirIfNotExist(ip string, port int, user, password, keyPath, remoteDir } defer session.Close() - // Check if the directory exists - checkCmd := fmt.Sprintf("[ -d \"%s\" ] && echo \"exists\" || echo \"not exists\"", remoteDir) + // Validate remote directory path before constructing shell commands + if err := validateRemotePath(remoteDir); err != nil { + return fmt.Errorf("unsafe remote directory path: %v", err) + } + + // Check if the directory exists (quoted to prevent injection) + checkCmd := fmt.Sprintf("[ -d %s ] && echo \"exists\" || echo \"not exists\"", shellQuote(remoteDir)) output, err := session.CombinedOutput(checkCmd) if err != nil { return fmt.Errorf("failed to check directory existence: %v", err) @@ -215,8 +349,8 @@ func SSHMkdirIfNotExist(ip string, port int, user, password, keyPath, remoteDir } defer session.Close() - // Execute mkdir command remotely - cmd := fmt.Sprintf("mkdir -p %s", remoteDir) + // Execute mkdir command remotely (path already validated above) + cmd := fmt.Sprintf("mkdir -p %s", shellQuote(remoteDir)) if err := session.Run(cmd); err != nil { return fmt.Errorf("failed to create directory: %v", err) } @@ -241,10 +375,15 @@ func RunRemoteCompose(ip string, port int, user, password, keyPath, composePath } defer session.Close() - // Construct the docker-compose command - cmd := fmt.Sprintf("docker-compose -f %s up -d", composePath) + // Validate compose file path before constructing shell command + if err := validateRemotePath(composePath); err != nil { + return fmt.Errorf("unsafe docker-compose path: %v", err) + } + + // Construct the docker-compose command (quoted to prevent injection) + cmd := fmt.Sprintf("docker-compose -f %s up -d", shellQuote(composePath)) if down { - cmd = fmt.Sprintf("docker-compose -f %s down", composePath) + cmd = fmt.Sprintf("docker-compose -f %s down", shellQuote(composePath)) } // Run the command remotely