diff --git a/.github/workflows/check-pr.yaml b/.github/workflows/check-pr.yaml index b375b41b22..953ddc8d46 100644 --- a/.github/workflows/check-pr.yaml +++ b/.github/workflows/check-pr.yaml @@ -17,7 +17,7 @@ jobs: lang-changed: ${{ steps.filter.outputs.lang == 'true' }} steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 + - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0 with: distribution: 'temurin' java-version: '17' @@ -66,7 +66,7 @@ jobs: matrix: os: [macos-15-intel, macos-15, ubuntu-24.04, ubuntu-24.04-arm, windows-2022] steps: - - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 + - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0 with: distribution: 'temurin' java-version: '17' @@ -91,7 +91,7 @@ jobs: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: repository: ${{ matrix.repo }} - - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 + - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0 with: distribution: 'temurin' java-version: '17' @@ -106,7 +106,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 + - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0 with: distribution: 'temurin' java-version: '17' @@ -141,7 +141,7 @@ jobs: PR_NUMBER: ${{ github.event.number }} steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 + - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0 with: distribution: 'temurin' java-version: '17' diff --git a/.github/workflows/on-push-default-branch.yml b/.github/workflows/on-push-default-branch.yml index 6d80e8b269..d68b46402a 100644 --- a/.github/workflows/on-push-default-branch.yml +++ b/.github/workflows/on-push-default-branch.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 + - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0 with: distribution: 'temurin' java-version: '17' @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 + - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0 with: distribution: 'temurin' java-version: '17' diff --git a/.github/workflows/on-release-published.yml b/.github/workflows/on-release-published.yml index b72b43ff6f..955d5f3789 100644 --- a/.github/workflows/on-release-published.yml +++ b/.github/workflows/on-release-published.yml @@ -30,7 +30,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 with: username: ${{ secrets.DOCKERHUB_USER }} password: ${{ secrets.DOCKERHUB_PASSWORD }} @@ -42,7 +42,7 @@ jobs: tags: | type=ref,event=tag type=raw,value=latest,enable=${{ github.event.release.prerelease == false }} - - uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0 + - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 with: platforms: linux/amd64,linux/arm64 context: ./docker/private @@ -58,7 +58,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 + - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0 with: distribution: 'temurin' java-version: '17' @@ -66,7 +66,7 @@ jobs: - uses: sbt/setup-sbt@3afe9cf056c5d139bfc46579af1192d77a2f0821 # v1.4.0 - run: sbt --batch 'buildRIDERunnerForDocker' - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 with: username: ${{ secrets.DOCKERHUB_USER }} password: ${{ secrets.DOCKERHUB_PASSWORD }} @@ -78,7 +78,7 @@ jobs: tags: | type=match,pattern=v(.*),group=1 type=raw,value=latest,enable=${{ github.event.release.prerelease == false }} - - uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0 + - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 with: context: ./ride-runner/docker push: ${{ github.event_name == 'release' }} @@ -103,7 +103,7 @@ jobs: run: | echo "::error::Release version $RELEASE_VERSION does not match SBT version $SBT_VERSION" exit 1 - - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 + - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0 with: distribution: 'temurin' java-version: '17' diff --git a/.github/workflows/prepare-release-draft.yml b/.github/workflows/prepare-release-draft.yml index a79ad6a2e9..8f7bea0e05 100644 --- a/.github/workflows/prepare-release-draft.yml +++ b/.github/workflows/prepare-release-draft.yml @@ -21,7 +21,7 @@ jobs: id-token: write steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 + - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0 with: distribution: 'temurin' java-version: '17' diff --git a/.github/workflows/publish-aptly-repo.yml b/.github/workflows/publish-aptly-repo.yml index 6ce398e658..a5b75d0cc7 100644 --- a/.github/workflows/publish-aptly-repo.yml +++ b/.github/workflows/publish-aptly-repo.yml @@ -30,7 +30,7 @@ jobs: echo "aptly_cachekey=aptly-$APTLY_VERSION" >> "$GITHUB_OUTPUT" - name: Cache aptly id: cache-aptly - uses: actions/cache@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v6.0.0 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.cache-aptly-path.outputs.aptly_toolpath }} key: ${{ steps.cache-aptly-path.outputs.aptly_cachekey }} diff --git a/.github/workflows/publish-docker-image.yml b/.github/workflows/publish-docker-image.yml index efd66cadf0..62fe3e6444 100644 --- a/.github/workflows/publish-docker-image.yml +++ b/.github/workflows/publish-docker-image.yml @@ -51,7 +51,7 @@ jobs: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: ref: ${{ inputs.ref }} - - uses: regclient/actions/regctl-installer@4b4db1dcc7dad75ad67a788a380f75a20cc8a040 # v0.1.0 + - uses: regclient/actions/regctl-installer@39bf30ced4be9d147faaabbde2682e9725fb8bd1 # v0.1.0 - uses: proudust/gh-describe@80412be8ce0e77d8afba6b340e34790bc772aa45 # v2.2.0 id: ghd with: @@ -64,7 +64,7 @@ jobs: echo "base-image-digest=$(regctl manifest digest $BASE_IMAGE)" >> "$GITHUB_OUTPUT" echo "node-version=$(echo ${{ steps.ghd.outputs.describe }} | cut -c 2-)" >> "$GITHUB_OUTPUT" id: versions - - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 + - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0 with: distribution: 'temurin' java-version: '17' @@ -107,12 +107,12 @@ jobs: echo EOF } >> "$GITHUB_OUTPUT" id: annotations - - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 if: inputs.environment != '' with: username: ${{ secrets.DOCKERHUB_USER }} @@ -127,7 +127,7 @@ jobs: flavor: latest=${{ contains(inputs.docker-tags, 'latest') }} tags: ${{ env.TAGS }} annotations: ${{ steps.annotations.outputs.annotations }} - - uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 + - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 id: push with: context: ./docker @@ -174,7 +174,7 @@ jobs: echo ${{ inputs.docker-tags }} | tr ',' '\n' | sed -e 's/^/type=raw,value=/' echo EOF } >> "$GITHUB_ENV" - - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -193,7 +193,7 @@ jobs: org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} org.opencontainers.image.version=${{ needs.publish.outputs.node-version }} org.opencontainers.image.base.digest=${{ needs.publish.outputs.digest }} - - uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0 + - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 with: platforms: linux/amd64,linux/arm64 context: ./docker/private diff --git a/.github/workflows/publish-to-npmjs.yml b/.github/workflows/publish-to-npmjs.yml index 7e2c8090b3..af5929edd2 100644 --- a/.github/workflows/publish-to-npmjs.yml +++ b/.github/workflows/publish-to-npmjs.yml @@ -31,7 +31,7 @@ jobs: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: proudust/gh-describe@80412be8ce0e77d8afba6b340e34790bc772aa45 # v2.2.0 id: ghd - - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 + - uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0 with: distribution: 'temurin' java-version: '17' diff --git a/benchmark/build.sbt b/benchmark/build.sbt index 0be555d141..d9fa4e2beb 100644 --- a/benchmark/build.sbt +++ b/benchmark/build.sbt @@ -10,7 +10,7 @@ Test / classLoaderLayeringStrategy := ClassLoaderLayeringStrategy.Flat libraryDependencies ++= Seq( "org.scodec" %% "scodec-core" % "2.3.3", "org.eclipse.collections" % "eclipse-collections" % "13.0.0", - "commons-codec" % "commons-codec" % "1.22.0", + "commons-codec" % "commons-codec" % "1.22.1", "org.apache.commons" % "commons-lang3" % "3.20.0" ) ++ Dependencies.logDeps diff --git a/grpc-server/src/main/scala/com/wavesplatform/api/grpc/package.scala b/grpc-server/src/main/scala/com/wavesplatform/api/grpc/package.scala index b19ce00378..18e28f00a9 100644 --- a/grpc-server/src/main/scala/com/wavesplatform/api/grpc/package.scala +++ b/grpc-server/src/main/scala/com/wavesplatform/api/grpc/package.scala @@ -100,7 +100,7 @@ package object grpc { sendNextItem() p.future - } else Future.failed(new IllegalStateException(s"An element ${nextItem()} is pending")) + } else Future.failed(new IllegalStateException(s"An element ${nextItem.get()} is pending")) }, err => cso.onError(err), { () => diff --git a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/evaluator/ctx/impl/PureContext.scala b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/evaluator/ctx/impl/PureContext.scala index 357097acfb..9b212fb7d1 100644 --- a/lang/shared/src/main/scala/com/wavesplatform/lang/v1/evaluator/ctx/impl/PureContext.scala +++ b/lang/shared/src/main/scala/com/wavesplatform/lang/v1/evaluator/ctx/impl/PureContext.scala @@ -23,7 +23,7 @@ import com.wavesplatform.lang.v1.parser.BinaryOperation.* import com.wavesplatform.lang.v1.{BaseGlobal, CTX, FunctionHeader, compiler} import java.nio.charset.StandardCharsets.UTF_8 -import java.nio.charset.{MalformedInputException, StandardCharsets} +import java.nio.charset.{CharsetDecoder, MalformedInputException, StandardCharsets} import java.nio.{BufferUnderflowException, ByteBuffer} import scala.annotation.tailrec import scala.collection.mutable.ArrayBuffer @@ -947,7 +947,9 @@ object PureContext { notImplemented[Id, EVALUATED]("dropRight(xs: String, number: Int)", xs) } - val UTF8Decoder = UTF_8.newDecoder + val UTF8Decoder = new ThreadLocal[CharsetDecoder] { + override def initialValue(): CharsetDecoder = UTF_8.newDecoder() + } def toUtf8String(reduceLimit: Boolean): BaseFunction[NoContext] = NativeFunction( @@ -959,7 +961,7 @@ object PureContext { ) { case CONST_BYTESTR(u) :: Nil => Try(ByteBuffer.wrap(u.arr)) - .map(UTF8Decoder.decode) + .map(UTF8Decoder.get().decode) .toEither .map(_.toString) .flatMap(CONST_STRING(_, reduceLimit)) diff --git a/node/src/main/scala/com/wavesplatform/Application.scala b/node/src/main/scala/com/wavesplatform/Application.scala index c8cfb8c12d..9a559b3dab 100644 --- a/node/src/main/scala/com/wavesplatform/Application.scala +++ b/node/src/main/scala/com/wavesplatform/Application.scala @@ -85,6 +85,14 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con forceStopApplication(FatalDBError) } + private def reportErrorAndBlacklist(errorMessage: String, source: Channel)(maybeThrowable: Option[Throwable]): Task[Unit] = { + maybeThrowable.foreach { t => + peerDatabase.blacklistAndClose(source, errorMessage) + log.warn(errorMessage, t) + } + Task.now(()) + } + private val appenderScheduler = singleThread("appender", stopOnAppendError) private val extensionLoaderScheduler = singleThread("rx-extension-loader", reporter = log.error("Error in Extension Loader", _)) @@ -344,12 +352,7 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con syncWithChannelClosed, extensionLoaderScheduler, timeoutSubject - ) { case (c, b) => - processFork(c, b).doOnFinish { - case None => Task.now(()) - case Some(e) => Task(stopOnAppendError.reportFailure(e)) - } - } + ) { case (c, b) => processFork(c, b).doOnFinish(reportErrorAndBlacklist("Exception caught while processing extension", c)) } TransactionSynchronizer( settings.synchronizationSettings.utxSynchronizer, @@ -360,11 +363,10 @@ class Application(val actorSystem: ActorSystem, val settings: WavesSettings, con Observable( microblockDataWithSnapshot - .mapEval(processMicroBlock.tupled), + .mapEval { case (c, md, sd) => processMicroBlock(c, md, sd).doOnFinish(reportErrorAndBlacklist("Error appending microblock", c)) }, newBlocksWithSnapshot - .mapEval(processBlock.tupled) + .mapEval { case (c, b, s) => processBlock(c, b, s).doOnFinish(reportErrorAndBlacklist("Error appending blocks", c)) } ).mergeMap(identity) - .onErrorHandle(stopOnAppendError.reportFailure) .subscribe() // API start diff --git a/node/src/main/scala/com/wavesplatform/database/RDB.scala b/node/src/main/scala/com/wavesplatform/database/RDB.scala index 17d2c47f5c..177c4b52e3 100644 --- a/node/src/main/scala/com/wavesplatform/database/RDB.scala +++ b/node/src/main/scala/com/wavesplatform/database/RDB.scala @@ -32,6 +32,14 @@ object RDB extends StrictLogging { final class TxSnapshotHandle private[RDB] (val handle: ColumnFamilyHandle) final class ApiHandle private[RDB] (val handle: ColumnFamilyHandle) + val TxMetaCF = "tx-meta" + val TxCF = "tx" + val TxSnapshotCF = "tx-snapshot" + val ApiCF = "api" + + /** Column families that only contain data derived from the blockchain and can be truncated without losing the state */ + val NonEssentialColumnFamilies: Seq[String] = Seq(TxSnapshotCF, ApiCF) + case class OptionsWithResources[A](options: A, resources: Seq[RocksObject]) def open(settings: DBSettings): RDB = { @@ -63,26 +71,26 @@ object RDB extends StrictLogging { .setCfPaths(Seq(new DbPath(new File(dbDir, "default").toPath, 0L)).asJava) ), new ColumnFamilyDescriptor( - "tx-meta".utf8Bytes, + TxMetaCF.utf8Bytes, txMetaCfOptions.options .optimizeForPointLookup(16 << 20) // Iterators might not work with this option .setDisableAutoCompactions(true) - .setCfPaths(Seq(new DbPath(new File(dbDir, "tx-meta").toPath, 0L)).asJava) + .setCfPaths(Seq(new DbPath(new File(dbDir, TxMetaCF).toPath, 0L)).asJava) ), new ColumnFamilyDescriptor( - "tx".utf8Bytes, + TxCF.utf8Bytes, txCfOptions.options - .setCfPaths(Seq(new DbPath(new File(dbDir, "tx").toPath, 0L)).asJava) + .setCfPaths(Seq(new DbPath(new File(dbDir, TxCF).toPath, 0L)).asJava) ), new ColumnFamilyDescriptor( - "tx-snapshot".utf8Bytes, + TxSnapshotCF.utf8Bytes, txSnapshotCfOptions.options - .setCfPaths(Seq(new DbPath(new File(dbDir, "tx-snapshot").toPath, 0L)).asJava) + .setCfPaths(Seq(new DbPath(new File(dbDir, TxSnapshotCF).toPath, 0L)).asJava) ), new ColumnFamilyDescriptor( - "api".utf8Bytes, + ApiCF.utf8Bytes, apiCfOptions.options - .setCfPaths(Seq(new DbPath(new File(dbDir, "api").toPath, 0L)).asJava) + .setCfPaths(Seq(new DbPath(new File(dbDir, ApiCF).toPath, 0L)).asJava) ) ).asJava, handles @@ -94,10 +102,27 @@ object RDB extends StrictLogging { new TxHandle(handles.get(2)), new TxSnapshotHandle(handles.get(3)), new ApiHandle(handles.get(4)), - dbOptions.resources ++ defaultCfOptions.resources ++ txMetaCfOptions.resources ++ txCfOptions.resources ++ txSnapshotCfOptions.resources + dbOptions.resources ++ defaultCfOptions.resources ++ txMetaCfOptions.resources ++ txCfOptions.resources ++ + txSnapshotCfOptions.resources ++ apiCfOptions.resources ) } + /** Removes all the data from a non-essential column family by dropping it. It is re-created on the next [[open]], because the DB is opened + * with `createMissingColumnFamilies`. + */ + def truncateColumnFamily(settings: DBSettings, cfName: String): Unit = { + val handleOf: RDB => ColumnFamilyHandle = cfName match { + case TxSnapshotCF => _.txSnapshotHandle.handle + case ApiCF => _.apiHandle.handle + case _ => throw new IllegalArgumentException(s"$cfName is not one of ${NonEssentialColumnFamilies.mkString(", ")}") + } + + Using.resource(open(settings)) { rdb => + logger.info(s"Truncating $cfName") + rdb.db.dropColumnFamily(handleOf(rdb)) + } + } + private def newColumnFamilyOptions( bitsPerKey: Double, blockSize: Long, diff --git a/node/src/main/scala/com/wavesplatform/network/BasicMessagesRepo.scala b/node/src/main/scala/com/wavesplatform/network/BasicMessagesRepo.scala index 29d88c3e5d..ee289f553d 100644 --- a/node/src/main/scala/com/wavesplatform/network/BasicMessagesRepo.scala +++ b/node/src/main/scala/com/wavesplatform/network/BasicMessagesRepo.scala @@ -46,7 +46,7 @@ abstract class InetSocketAddressSeqSpec[A <: AnyRef: ClassTag] extends MessageSp override val messageCode: Message.MessageCode = 2: Byte - override val maxLength: Int = DataLength + 1000 * (AddressLength + PortLength) + override val maxLength: Int = DataLength + InetSocketAddressSeqSpec.MaxAddressCount * (AddressLength + PortLength) protected def unwrap(v: A): Seq[InetSocketAddress] protected def wrap(addresses: Seq[InetSocketAddress]): A @@ -82,6 +82,10 @@ abstract class InetSocketAddressSeqSpec[A <: AnyRef: ClassTag] extends MessageSp } } +object InetSocketAddressSeqSpec { + val MaxAddressCount: Int = 1000 +} + object PeersSpec extends InetSocketAddressSeqSpec[KnownPeers] { override protected def unwrap(v: KnownPeers): Seq[InetSocketAddress] = v.peers diff --git a/node/src/main/scala/com/wavesplatform/network/MicroBlockSynchronizer.scala b/node/src/main/scala/com/wavesplatform/network/MicroBlockSynchronizer.scala index f043108140..2d12bb8ceb 100644 --- a/node/src/main/scala/com/wavesplatform/network/MicroBlockSynchronizer.scala +++ b/node/src/main/scala/com/wavesplatform/network/MicroBlockSynchronizer.scala @@ -204,6 +204,7 @@ object MicroBlockSynchronizer extends ScorexLogging { CacheBuilder .newBuilder() .expireAfterWrite(timeout.toJava) + .maximumSize(200) .build[K, V]() case class CacheSizes(microBlockOwners: Long, nextInvs: Long, awaiting: Long, successfullyReceived: Long) diff --git a/node/src/main/scala/com/wavesplatform/network/PeerSynchronizer.scala b/node/src/main/scala/com/wavesplatform/network/PeerSynchronizer.scala index 7d0814e004..1d2ba8b05b 100644 --- a/node/src/main/scala/com/wavesplatform/network/PeerSynchronizer.scala +++ b/node/src/main/scala/com/wavesplatform/network/PeerSynchronizer.scala @@ -25,7 +25,7 @@ class PeerSynchronizer(peerDatabase: PeerDatabase, peerRequestInterval: FiniteDu requestPeers(ctx) super.channelRead(ctx, msg) case GetPeers => - ctx.writeAndFlush(KnownPeers(peerDatabase.knownPeers.keys.toSeq)) + ctx.writeAndFlush(KnownPeers(peerDatabase.knownPeers.keys.take(InetSocketAddressSeqSpec.MaxAddressCount).toSeq)) case KnownPeers(peers) if peersRequested => peersRequested = false val (added, notAdded) = peers.partition(peerDatabase.addCandidate) diff --git a/node/src/main/scala/com/wavesplatform/utils/UtilApp.scala b/node/src/main/scala/com/wavesplatform/utils/UtilApp.scala index 1535f2e660..252577f7c8 100644 --- a/node/src/main/scala/com/wavesplatform/utils/UtilApp.scala +++ b/node/src/main/scala/com/wavesplatform/utils/UtilApp.scala @@ -7,6 +7,7 @@ import com.wavesplatform.common.utils.EitherExt2.explicitGet import com.wavesplatform.common.utils.{Base58, Base64, FastBase58} import com.wavesplatform.crypto.bls.{BlsKeyPair, BlsSignature} import com.wavesplatform.crypto.{P256Curve, Sha256} +import com.wavesplatform.database.RDB import com.wavesplatform.features.EstimatorProvider.* import com.wavesplatform.lang.ValidationError import com.wavesplatform.lang.script.{Script, ScriptReader} @@ -24,11 +25,11 @@ import java.nio.charset.StandardCharsets import java.nio.file.{Files, Paths} import java.util import scala.annotation.nowarn -import scala.util.Random +import scala.util.{Random, Try} object UtilApp { enum Mode { - case CompileScript, DecompileScript, SignBytes, VerifySignature, CreateKeyPair, Hash, SerializeTx, SignTx, SmokeTest + case CompileScript, DecompileScript, SignBytes, VerifySignature, CreateKeyPair, Hash, SerializeTx, SignTx, SmokeTest, TruncateCF } case class CompileOptions(assetScript: Boolean = false) @@ -54,7 +55,8 @@ object UtilApp { signOptions: String | KeyPair = "", verifyOptions: VerifyOptions = VerifyOptions(), hashOptions: HashOptions = HashOptions(), - keyPairOptions: KeyPairOptions = KeyPairOptions() + keyPairOptions: KeyPairOptions = KeyPairOptions(), + columnFamily: String = "" ) private def maybeFindKeyPair(cmd: Command): Either[ValidationError, KeyPair] = { @@ -79,6 +81,7 @@ object UtilApp { case Mode.SerializeTx => Actions.doSerializeTx(inBytes) case Mode.SignTx => maybeFindKeyPair(cmd).flatMap(Actions.doSignTx(_, inBytes)) case Mode.SmokeTest => Actions.doSmokeTest() + case Mode.TruncateCF => Actions.doTruncateCF(Application.loadApplicationConfig(cmd.configFile.map(new File(_))), cmd.columnFamily) } result match { @@ -213,6 +216,19 @@ object UtilApp { ) ), cmd("smoke").action((_, c) => c.copy(mode = Mode.SmokeTest, inputData = Input.Str(""))), + cmd("truncate-cf") + .text(s"Removes all data from a non-essential column family (${RDB.NonEssentialColumnFamilies.mkString("|")})") + .action((_, c) => c.copy(mode = Mode.TruncateCF, inputData = Input.Str(""))) + .children( + arg[String]("") + .text(s"Column family to truncate (${RDB.NonEssentialColumnFamilies.mkString("|")})") + .required() + .action((cf, c) => c.copy(columnFamily = cf)) + .validate { + case cf if RDB.NonEssentialColumnFamilies.contains(cf) => success + case cf => failure(s"Column family $cf can not be truncated, expected one of ${RDB.NonEssentialColumnFamilies.mkString(", ")}") + } + ), help("help").hidden(), checkConfig(_.mode match { case null => failure("Command should be provided") @@ -296,6 +312,11 @@ object UtilApp { .map(_.json().toString().getBytes()) } + def doTruncateCF(settings: WavesSettings, cfName: String): ActionResult = + Try(RDB.truncateColumnFamily(settings.dbSettings, cfName)).toEither.left + .map(_.toString) + .map(_ => Array.emptyByteArray) + def doSmokeTest(): ActionResult = { val message = Base64.decode( "AgIZGwP/AAYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFQAAAAAAAADnAAAAAAAAALeumraedvd5Slaw2xkVKB1DXUiMkdQG7TOnk5yvhzD4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADcnip8b5SPF0dONKf8Q+0DD3wVY/G6vd9jQMguDlSoxQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIABwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADQ0PMxCMjQs6H9Ericuy2oMAj6fPa7h5C7H86EurUIgwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" diff --git a/node/tests/src/test/scala/com/wavesplatform/database/TruncateColumnFamilySpec.scala b/node/tests/src/test/scala/com/wavesplatform/database/TruncateColumnFamilySpec.scala new file mode 100644 index 0000000000..030e366ad6 --- /dev/null +++ b/node/tests/src/test/scala/com/wavesplatform/database/TruncateColumnFamilySpec.scala @@ -0,0 +1,62 @@ +package com.wavesplatform.database + +import com.wavesplatform.TestHelpers +import com.wavesplatform.db.DBCacheSettings +import com.wavesplatform.settings.DBSettings +import com.wavesplatform.test.FreeSpec +import com.wavesplatform.utils.* +import org.rocksdb.ColumnFamilyHandle + +import java.nio.file.Files +import scala.util.Using + +class TruncateColumnFamilySpec extends FreeSpec with DBCacheSettings { + private def withDbSettings(f: DBSettings => Unit): Unit = { + val path = Files.createTempDirectory("rocks-temp-truncate-cf").toAbsolutePath + try f(dbSettings.copy(directory = path.toAbsolutePath.toString)) + finally TestHelpers.deleteRecursively(path) + } + + private def handleOf(rdb: RDB, cfName: String): ColumnFamilyHandle = cfName match { + case RDB.TxSnapshotCF => rdb.txSnapshotHandle.handle + case RDB.ApiCF => rdb.apiHandle.handle + } + + private def put(rdb: RDB, handle: ColumnFamilyHandle, count: Int): Unit = + (1 to count).foreach(i => rdb.db.put(handle, s"key-$i".utf8Bytes, s"value-$i".utf8Bytes)) + + private def count(rdb: RDB, handle: ColumnFamilyHandle): Int = + Using.resource(rdb.db.newIterator(handle)) { iter => + iter.seekToFirst() + var result = 0 + while (iter.isValid) { + result += 1 + iter.next() + } + result + } + + RDB.NonEssentialColumnFamilies.foreach { cfName => + s"$cfName is truncated, other column families are left intact" in withDbSettings { settings => + Using.resource(RDB.open(settings)) { rdb => + put(rdb, rdb.db.getDefaultColumnFamily, 10) + put(rdb, rdb.txHandle.handle, 10) + RDB.NonEssentialColumnFamilies.foreach(name => put(rdb, handleOf(rdb, name), 10)) + } + + RDB.truncateColumnFamily(settings, cfName) + + Using.resource(RDB.open(settings)) { rdb => + count(rdb, handleOf(rdb, cfName)) shouldBe 0 + count(rdb, rdb.db.getDefaultColumnFamily) shouldBe 10 + count(rdb, rdb.txHandle.handle) shouldBe 10 + RDB.NonEssentialColumnFamilies.filterNot(_ == cfName).foreach(name => count(rdb, handleOf(rdb, name)) shouldBe 10) + } + } + } + + "essential column family can not be truncated" in withDbSettings { settings => + Using.resource(RDB.open(settings))(_ => ()) + intercept[IllegalArgumentException](RDB.truncateColumnFamily(settings, RDB.TxCF)) + } +} diff --git a/node/tests/src/test/scala/com/wavesplatform/state/snapshot/TxStateSnapshotHashSpec.scala b/node/tests/src/test/scala/com/wavesplatform/state/snapshot/TxStateSnapshotHashSpec.scala index 7d91df5ceb..868e4f90fa 100644 --- a/node/tests/src/test/scala/com/wavesplatform/state/snapshot/TxStateSnapshotHashSpec.scala +++ b/node/tests/src/test/scala/com/wavesplatform/state/snapshot/TxStateSnapshotHashSpec.scala @@ -335,4 +335,5 @@ class TxStateSnapshotHashSpec extends PropSpec { } } } + } diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 003bb7331d..91e3b2985f 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -4,9 +4,9 @@ import sbt.{Def, *} import scalapb.compiler.Version.scalapbVersion object Dependencies { - private def nettyModule(module: String) = "io.netty" % s"netty-$module" % "4.2.15.Final" + private def nettyModule(module: String) = "io.netty" % s"netty-$module" % "4.2.17.Final" - val gProtoVersion = "4.35.1" + val gProtoVersion = "4.36.0" val gProto = "com.google.protobuf" % "protobuf-java" % Dependencies.gProtoVersion val overrides = Def.setting( Seq( @@ -35,9 +35,9 @@ object Dependencies { lazy val protoSchemasLib = "com.wavesplatform" % "protobuf-schemas" % "1.6.0" classifier "protobuf-src" intransitive () - private def pekkoModule(module: String) = "org.apache.pekko" %% s"pekko-$module" % "1.6.0" + private def pekkoModule(module: String) = "org.apache.pekko" %% s"pekko-$module" % "1.7.0" - private def pekkoHttpModule(module: String, version: String = "1.3.0") = "org.apache.pekko" %% module % version + private def pekkoHttpModule(module: String, version: String = "1.4.0") = "org.apache.pekko" %% module % version private def kamonModule(module: String) = "io.kamon" %% s"kamon-$module" % "2.8.1" @@ -47,14 +47,14 @@ object Dependencies { def monixModule(module: String): Def.Initialize[ModuleID] = Def.setting("io.monix" %%% s"monix-$module" % "3.4.1") - private def grpcModule(module: String) = "io.grpc" % module % "1.82.1" + private def grpcModule(module: String) = "io.grpc" % module % "1.83.1" val pekkoHttp = pekkoHttpModule("pekko-http") - val googleGuava = "com.google.guava" % "guava" % "33.6.0-jre" + val googleGuava = "com.google.guava" % "guava" % "33.7.1-jre" val kamonCore = kamonModule("core") val machinist = "org.typelevel" %% "machinist" % "0.6.8" - val logback = "ch.qos.logback" % "logback-classic" % "1.5.37" - val asyncHttpClient = "org.asynchttpclient" % "async-http-client" % "3.0.11" + val logback = "ch.qos.logback" % "logback-classic" % "1.6.3" + val asyncHttpClient = "org.asynchttpclient" % "async-http-client" % "3.0.13" val curve25519 = "com.wavesplatform" % "curve25519-java" % "0.6.6" val nettyHandler = nettyModule("handler") @@ -74,11 +74,11 @@ object Dependencies { val cryptoProviders = Seq( // Windows x86_64, Windows x86, macOS x86_64, linux x86_64 - "org.conscrypt" % "conscrypt-openjdk-uber" % "2.5.2", + "org.conscrypt" % "conscrypt-openjdk-uber" % "2.6.3", // macOS aarch64 amazonCorretto("osx-aarch_64"), // fallback Java - "org.bouncycastle" % "bcprov-jdk18on" % "1.84" + "org.bouncycastle" % "bcprov-jdk18on" % "1.85.2" ) val lang = Def.setting( @@ -107,9 +107,9 @@ object Dependencies { lazy val it = scalaTest +: Seq( logback, - "com.github.jnr" % "jnr-unixsocket" % "0.39.1", // To support Apple ARM + "com.github.jnr" % "jnr-unixsocket" % "0.39.3", // To support Apple ARM "com.spotify" % "docker-client" % "8.16.0", - jacksonModule("dataformat", "dataformat-properties", "2.22.0"), + jacksonModule("dataformat", "dataformat-properties", "2.22.2"), asyncHttpClient ).map(_ % Test) diff --git a/project/plugins.sbt b/project/plugins.sbt index 4cebb2c2b2..efe60a636e 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -6,24 +6,24 @@ resolvers ++= Seq( // Should go before Scala.js addSbtPlugin("com.thesamet" % "sbt-protoc" % "1.0.8") -libraryDependencies += "com.thesamet.scalapb" %% "compilerplugin" % "1.0.0-alpha.5" +libraryDependencies += "com.thesamet.scalapb" %% "compilerplugin" % "1.0.0-alpha.6" Seq( - "com.eed3si9n" % "sbt-assembly" % "2.3.1", + "com.eed3si9n" % "sbt-assembly" % "2.4.1", "com.github.sbt" % "sbt-git" % "2.1.0", "com.github.sbt" % "sbt-native-packager" % "1.11.7", "com.github.sbt" % "sbt-pgp" % "2.3.1", "com.lightbend.sbt" % "sbt-javaagent" % "0.1.6", - "org.portable-scala" % "sbt-scalajs-crossproject" % "1.3.2", - "org.scala-js" % "sbt-scalajs" % "1.21.0", - "org.scalameta" % "sbt-scalafmt" % "2.6.1", + "org.portable-scala" % "sbt-scalajs-crossproject" % "1.4.0", + "org.scala-js" % "sbt-scalajs" % "1.22.0", + "org.scalameta" % "sbt-scalafmt" % "2.6.2", "pl.project13.scala" % "sbt-jmh" % "0.4.8" ).map(addSbtPlugin) val dockerJavaVersion = "3.7.1" libraryDependencies ++= Seq( - "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.22.0", + "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.22.2", "org.hjson" % "hjson" % "3.1.0", "org.vafer" % "jdeb" % "1.14" artifacts Artifact("jdeb", "jar", "jar"), "org.slf4j" % "jcl-over-slf4j" % "2.0.18",