feat: sqlite indexer

CHANGELOG.md

@@ -31,6 +31,8 @@
 
 ### Added
 
+- [#6720](https://github.com/ChainSafe/forest/pull/6720): Added SQL index and `forest-cli index validate-backfill`(Compatible with `lotus index validate-backfill`) for backfilling the SQL index.
+
 - [#7426](https://github.com/ChainSafe/forest/pull/7426): Added `forest-tool db import-tipset-lookup` for importing a tipset lookup snapshot.
 
 - [#7408](https://github.com/ChainSafe/forest/pull/7408): Impl `Forest.IndexBackfill`, `Forest.IndexBackfillStatus` and `Forest.IndexBackfillCancel` RPC methods and the `forest-cli index backfill` command to back-fill the chain index (Ethereum mappings, events, block blooms) through the running daemon, so the node no longer needs to be stopped. An interrupted or cancelled run can be resumed with `--resume`.

Cargo.toml

@@ -211,6 +211,7 @@ smallvec = "1"
 smart-default = "0.7"
 sonic-rs = "0.5"
 spire_enum = "1"
+sqlx = { version = "0.9", default-features = false, features = ["sqlite", "runtime-tokio", "macros"] }
 stacker = "0.1"
 static_assertions = "1"
 statrs = "0.19"
@@ -241,7 +242,6 @@ zstd = "0.13"
 # optional dependencies
 console-subscriber = { version = "0.5", features = ["parking_lot"], optional = true }
 libtest-mimic = "0.8"
-sqlx = { version = "0.9", default-features = false, features = ["sqlite", "runtime-tokio", "macros"], optional = true }
 tikv-jemallocator = { version = "0.7", optional = true }
 tracing-loki = { version = "0.2", default-features = false, features = ["compat-0-2-1", "rustls"], optional = true }
 
@@ -348,7 +348,7 @@ debug = "line-tables-only"
 
 # These should be refactored (probably removed) in #2984
 [features]
-default = ["jemalloc", "tracing-loki", "sqlite"]
+default = ["jemalloc", "tracing-loki"]
 test = [] # default feature set for unit tests
 slim = ["rustalloc"]
 cargo-test = [] # group of tests that is recommended to run with `cargo test` instead of `nextest`
@@ -360,7 +360,6 @@ jemalloc-profiling = [
   "tikv-jemallocator?/profiling",
   "tikv-jemallocator?/unprefixed_malloc_on_supported_platforms",
 ] # enable jemalloc profiling features, use with `release-symbols` build profile for best results
-sqlite = ["dep:sqlx"]
 
 # Allocator. Use at most one of these.
 rustalloc = []

docs/docs/users/reference/env_variables.md

@@ -79,6 +79,7 @@ process.
 | `FOREST_ETH_RPC_COMPUTE_BLOOM_ON_MISS`                           | 1 or true                        | false                                                      | 1                                                             | Allows `eth` block RPC methods to compute (and store) the block `logsBloom` when it is not already stored, otherwise such blocks report an all-ones bloom                                                                                                                      |
 | `FOREST_RPC_METRICS_DISABLED`                                    | 1 or true                        | false                                                      | 1                                                             | Disable per-method JSON-RPC metrics only, leaving the metrics endpoint and all other metrics (cache, sync, database, ...) intact. To turn off metrics entirely, disable the endpoint instead with `--no-metrics`                                                               |
 | `FOREST_STATE_COMPUTE_MAX_RANGE`                                 | positive integer                 | 2000                                                       | 100                                                           | The maximum `nEpochs` value `Forest.StateCompute` accepts                                                                                                                                                                                                                      |
+| `FOREST_RPC_SQL_RANGE_THRESHOLD`                                 | positive integer                 | 500                                                        | 1000                                                          | The RPC tipset range threshold above which SQL index is prefered over in-memory event collection                                                                                                                                                                               |
 
 ### `FOREST_F3_SIDECAR_FFI_BUILD_OPT_OUT`
 

scripts/tests/api_compare/docker-compose.yml

@@ -91,6 +91,9 @@ services:
 
         # Backfill the NOSQL index db
         forest-cli index backfill --from $$SNAPSHOT_EPOCH --n-tipsets 200 --recompute --allow-near-head
+
+        # backfill the SQL index db
+        forest-cli index validate-backfill --from $$SNAPSHOT_EPOCH --to $(($$SNAPSHOT_EPOCH - 200))
   forest-wallet-import:
     depends_on:
       forest:

scripts/tests/api_compare/filter-list-gateway

@@ -8,6 +8,7 @@
 !Filecoin.ChainSetHead
 !Filecoin.ChainStatObj
 !Filecoin.ChainTipSetWeight
+!Filecoin.ChainValidateIndex
 !Filecoin.EthGetBlockReceiptsLimited
 !Filecoin.EthGetTransactionByHashLimited
 !Filecoin.F3

scripts/tests/api_compare/filter-list-offline

@@ -1,5 +1,7 @@
 # This list contains potentially broken methods (or tests) that are ignored.
 # They should be considered bugged, and not used until the root cause is resolved.
+# Indexer disabled
+!Filecoin.ChainValidateIndex
 !Filecoin.EthSyncing
 !eth_syncing
 !Filecoin.NetAddrsListen

src/beacon/drand.rs

@@ -23,7 +23,6 @@ use crate::utils::net::global_http_client;
 use ambassador::{Delegate, delegatable_trait};
 use backon::{ExponentialBuilder, Retryable};
 use bls_signatures::Serialize as _;
-use nonzero_ext::nonzero;
 use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize};
 use tracing::debug;
 use url::Url;