diff --git a/.github/workflows/cargo.yml b/.github/workflows/cargo.yml index 668f46f44..24cf4acbc 100644 --- a/.github/workflows/cargo.yml +++ b/.github/workflows/cargo.yml @@ -17,7 +17,8 @@ jobs: components: cargo, rustc, rustfmt profile: minimal - - uses: actions-rs/cargo@v1 + - name: cargo fmt + uses: actions-rs/cargo@v1 with: command: fmt args: --all --check @@ -43,19 +44,27 @@ jobs: target/ key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', 'rust-toolchain.toml') }} - - uses: actions-rs/cargo@v1 + - name: cargo clippy --lib + uses: actions-rs/cargo@v1 with: command: clippy - args: --all --lib --bins --all-features --no-deps --profile=debug-lite + args: --workspace --lib --all-features --no-deps --profile=debug-lite + + - name: cargo clippy --bins + uses: actions-rs/cargo@v1 + with: + command: clippy + args: --workspace --bins --all-features --no-deps --profile=debug-lite - name: check lockfile run: | diff Cargo.lock <(git show HEAD:Cargo.lock) - - uses: actions-rs/cargo@v1 + - name: cargo doc --lib + uses: actions-rs/cargo@v1 with: command: doc - args: --all --lib --all-features --no-deps --profile=debug-lite + args: --workspace --lib --all-features --no-deps --profile=debug-lite release-check: name: cargo check/doc/clippy (release profile) @@ -78,12 +87,20 @@ jobs: target/ key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', 'rust-toolchain.toml') }} - - uses: actions-rs/cargo@v1 + - name: cargo clippy --lib --release + uses: actions-rs/cargo@v1 + with: + command: clippy + args: --workspace --lib --all-features --profile=release-lite + + - name: cargo clippy --bins --release + uses: actions-rs/cargo@v1 with: command: clippy - args: --all --lib --bins --all-features --profile=release-lite + args: --workspace --bins --all-features --profile=release-lite - - uses: actions-rs/cargo@v1 + - name: cargo doc --lib --release + uses: actions-rs/cargo@v1 with: command: doc - args: --all --lib --all-features --no-deps --profile=release-lite + args: --workspace --lib --all-features --no-deps --profile=release-lite diff --git a/.github/workflows/cargo_test.yml b/.github/workflows/cargo_test.yml index cc5bb8f6a..8466527d9 100644 --- a/.github/workflows/cargo_test.yml +++ b/.github/workflows/cargo_test.yml @@ -2,14 +2,28 @@ name: Cargo test on: pull_request: - branches: [master] + branches: [dev, master] push: branches: [dev, master] jobs: - debug-build: + debug-test: name: cargo build/test (debug profile) runs-on: indexer-test-runner + + services: + postgres: + image: postgres:14.1 + + env: + POSTGRES_USER: ci + POSTGRES_PASSWORD: ci + POSTGRES_PORT: 5432 + POSTGRES_DB: ci + + ports: + - 5432:5432 + steps: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 @@ -28,14 +42,39 @@ jobs: target/ key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', 'rust-toolchain.toml') }} - - uses: actions-rs/cargo@v1 + - name: cargo test --lib + uses: actions-rs/cargo@v1 + env: + DATABASE_URL: postgres://ci:ci@127.0.0.1:5432/ci + with: + command: test + args: --workspace --lib --all-features --profile=debug-lite + + - name: cargo test --bins + uses: actions-rs/cargo@v1 + env: + DATABASE_URL: postgres://ci:ci@127.0.0.1:5432/ci with: command: test - args: --all --lib --bins --all-features --profile=debug-lite + args: --workspace --bins --all-features --profile=debug-lite - release-build: + release-test: name: cargo build/test (release profile) runs-on: indexer-test-runner + + services: + postgres: + image: postgres:14.1 + + env: + POSTGRES_USER: ci + POSTGRES_PASSWORD: ci + POSTGRES_PORT: 5432 + POSTGRES_DB: ci + + ports: + - 5432:5432 + steps: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 @@ -54,7 +93,18 @@ jobs: target/ key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', 'rust-toolchain.toml') }} - - uses: actions-rs/cargo@v1 + - name: cargo test --lib --release + uses: actions-rs/cargo@v1 + env: + DATABASE_URL: postgres://ci:ci@127.0.0.1:5432/ci + with: + command: test + args: --workspace --lib --all-features --profile=release-lite + + - name: cargo test --bins --release + uses: actions-rs/cargo@v1 + env: + DATABASE_URL: postgres://ci:ci@127.0.0.1:5432/ci with: command: test - args: --all --lib --bins --all-features --profile=release-lite + args: --workspace --bins --all-features --profile=release-lite diff --git a/.github/workflows/deploy_to_heroku_production.yml b/.github/workflows/deploy_to_heroku_production.yml deleted file mode 100644 index 5343585cf..000000000 --- a/.github/workflows/deploy_to_heroku_production.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Heroku (production) - -on: - push: - branches: [master] - -jobs: - build: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - - name: Geyser consumer - uses: gonuit/heroku-docker-deploy@v1.3.3 - with: - email: ${{ secrets.HEROKU_EMAIL }} - heroku_api_key: ${{ secrets.HEROKU_API_KEY }} - heroku_app_name: accounts-db-consumer - dockerfile_name: Dockerfile - docker_options: '--target geyser-consumer' - process_type: worker - - - name: HTTP consumer - uses: gonuit/heroku-docker-deploy@v1.3.3 - with: - email: ${{ secrets.HEROKU_EMAIL }} - heroku_api_key: ${{ secrets.HEROKU_API_KEY }} - heroku_app_name: accountsdbhttpconsumer - dockerfile_name: Dockerfile - docker_options: '--target http-consumer' - process_type: worker - - - name: GraphQL server - uses: gonuit/heroku-docker-deploy@v1.3.3 - with: - email: ${{ secrets.HEROKU_EMAIL }} - heroku_api_key: ${{ secrets.HEROKU_API_KEY }} - heroku_app_name: indexer-graphql - dockerfile_name: Dockerfile - docker_options: '--target graphql' - - - name: Storefronts Indexer - uses: gonuit/heroku-docker-deploy@v1.3.3 - with: - email: ${{ secrets.HEROKU_EMAIL }} - heroku_api_key: ${{ secrets.HEROKU_API_KEY }} - heroku_app_name: legacy-storefronts - dockerfile_name: Dockerfile - docker_options: '--target legacy-storefronts' - process_type: worker diff --git a/.github/workflows/deploy_to_heroku_staging.yml b/.github/workflows/deploy_to_heroku_staging.yml deleted file mode 100644 index 565452a4d..000000000 --- a/.github/workflows/deploy_to_heroku_staging.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Heroku (staging) - -on: - push: - branches: [dev, deploy_config, debug_validator] - -jobs: - build: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - - name: Geyser consumer - uses: gonuit/heroku-docker-deploy@v1.3.3 - with: - email: ${{ secrets.HEROKU_EMAIL }} - heroku_api_key: ${{ secrets.HEROKU_API_KEY }} - heroku_app_name: accounts-db-consumer-staging - dockerfile_name: Dockerfile - docker_options: '--target geyser-consumer' - process_type: worker - - - name: HTTP consumer - uses: gonuit/heroku-docker-deploy@v1.3.3 - with: - email: ${{ secrets.HEROKU_EMAIL }} - heroku_api_key: ${{ secrets.HEROKU_API_KEY }} - heroku_app_name: accountsdbhttpconsumerstaging - dockerfile_name: Dockerfile - docker_options: '--target http-consumer' - process_type: worker - - - name: GraphQL server - uses: gonuit/heroku-docker-deploy@v1.3.3 - with: - email: ${{ secrets.HEROKU_EMAIL }} - heroku_api_key: ${{ secrets.HEROKU_API_KEY }} - heroku_app_name: indexer-graphql-staging - dockerfile_name: Dockerfile - docker_options: '--target graphql' diff --git a/Cargo.lock b/Cargo.lock index c55c5f979..65202e5c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2939,6 +2939,7 @@ dependencies = [ "diesel_migrations", "dotenv", "env_logger", + "lazy_static", "log", "md5", "meilisearch-sdk", diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 232557651..a91b3714d 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -71,3 +71,6 @@ solana-program = { version = "1.9.4", optional = true } cid = { version = "0.7.0", optional = true } url = "2.2.2" md5 = { version = "0.7.0", optional = true } + +[dev-dependencies] +lazy_static = "1.4.0" diff --git a/crates/core/src/db/mod.rs b/crates/core/src/db/mod.rs index b750bb57e..9544965da 100644 --- a/crates/core/src/db/mod.rs +++ b/crates/core/src/db/mod.rs @@ -118,6 +118,31 @@ impl From for ConnectionType { } } +fn migrate( + conn: &C, + desc: impl std::fmt::Display, + run: impl FnOnce(&C, &mut dyn std::io::Write) -> Result<(), diesel_migrations::RunMigrationsError>, +) -> Result<()> { + info!("Running {}...", desc); + + let mut out = vec![]; + + run(conn, &mut out).with_context(|| format!("Failed to run {}", desc))?; + + match std::str::from_utf8(&out) { + Ok(s) => { + let s = s.trim(); + + if !s.is_empty() { + info!("Output from {}:\n{}", desc, s); + } + }, + Err(e) => warn!("Failed to read output from {}: {}", desc, e), + } + + Ok(()) +} + /// Create a pooled connection to the Postgres database, using the given CLI /// arguments and a hint indicating if the database is writable. /// @@ -159,8 +184,6 @@ pub fn connect(args: ConnectArgs, mode: ConnectMode) -> Result { .build(man) .context("Failed to create database connection pool")?; - let mut out = vec![]; - if cfg!(not(debug_assertions)) && matches!(ty, ConnectionType::Default) { warn!("Cannot determine if database is writable; assuming yes"); } @@ -187,25 +210,64 @@ pub fn connect(args: ConnectArgs, mode: ConnectMode) -> Result { }; if migrated { - info!("Running database migrations..."); + migrate( + &*pool + .get() + .context("Failed to acquire database connection")?, + "database migrations", + embedded_migrations::run_with_output, + )?; + } + + Ok(ConnectResult { pool, ty, migrated }) +} + +#[cfg(test)] +pub mod test { + embed_migrations!("test_migrations"); - embedded_migrations::run_with_output( - &pool.get().context("Failed to connect to the database")?, - &mut out, + fn initialize() -> super::ConnectResult { + dotenv::from_filename(".env.dev").expect("Failed to load .env.dev"); + dotenv::from_filename(".env").expect("Failed to load .env"); + + let conn_args = super::ConnectArgs { + database_read_url: None, + database_write_url: Some( + std::env::var("DATABASE_URL") + .expect("Failed to load DATABASE_URL from environment"), + ), + database_url: None, + }; + let res = super::connect(conn_args, super::ConnectMode::Write { migrate: true }) + .expect("Failed to connect to database"); + + super::migrate( + &*res + .pool + .get() + .expect("Failed to acquire database connection"), + "test seed migrations", + embedded_migrations::run_with_output, ) - .context("Failed to run database migrations")?; + .expect("Seed migrations failed"); + + res } - match std::str::from_utf8(&out) { - Ok(s) => { - let s = s.trim(); + lazy_static::lazy_static! { + static ref DATABASE: super::ConnectResult = initialize(); + } - if !s.is_empty() { - info!("Output from migrations:\n{}", s); - } - }, - Err(e) => warn!("Failed to read migration output: {}", e), + #[must_use] + pub fn connect() -> super::PooledConnection { + DATABASE + .pool + .get() + .expect("Failed to acquire database connection") } - Ok(ConnectResult { pool, ty, migrated }) + pub mod prelude { + pub use super::connect; + pub use crate::{db::custom_types::*, prelude::*}; + } } diff --git a/crates/core/src/db/queries/activities.rs b/crates/core/src/db/queries/activities.rs index bb4bc2c8d..6092241a7 100644 --- a/crates/core/src/db/queries/activities.rs +++ b/crates/core/src/db/queries/activities.rs @@ -57,3 +57,11 @@ pub fn list( .load(conn) .context("Failed to load activities") } + +#[cfg(test)] +mod tests { + #[test] + fn test_list() { + todo!("Test list()"); + } +} diff --git a/crates/core/src/db/queries/bonding_changes.rs b/crates/core/src/db/queries/bonding_changes.rs index 9cff4b0f6..2d9ed3e0c 100644 --- a/crates/core/src/db/queries/bonding_changes.rs +++ b/crates/core/src/db/queries/bonding_changes.rs @@ -54,3 +54,11 @@ pub fn list( .load(conn) .context("Failed to load enriched bonding changes") } + +#[cfg(test)] +mod tests { + #[test] + fn test_list() { + todo!("Test list()"); + } +} diff --git a/crates/core/src/db/queries/charts.rs b/crates/core/src/db/queries/charts.rs index fd20667de..93136f254 100644 --- a/crates/core/src/db/queries/charts.rs +++ b/crates/core/src/db/queries/charts.rs @@ -136,3 +136,19 @@ pub fn total_volume_prices( .load(conn) .context("Failed to load average prices") } + +#[cfg(test)] +mod tests { + #[test] + fn test_floor_prices() { + todo!("Test floor_prices()"); + } + #[test] + fn test_average_prices() { + todo!("Test average_prices()"); + } + #[test] + fn test_total_volume_prices() { + todo!("Test total_volume_prices()"); + } +} diff --git a/crates/core/src/db/queries/collections.rs b/crates/core/src/db/queries/collections.rs index 73680818b..feb403a2d 100644 --- a/crates/core/src/db/queries/collections.rs +++ b/crates/core/src/db/queries/collections.rs @@ -235,3 +235,49 @@ pub fn collection_activities( .load(conn) .context("Failed to load collection activities") } + +#[cfg(test)] +mod tests { + use crate::db::test::prelude::*; + + #[test] + fn test_collections_featured_by_marketcap_returns_non_empty() { + let conn = connect(); + + let result = super::by_market_cap( + &conn, + None::>, + OrderDirection::Desc, + Utc.ymd(1901, 1, 1).and_hms(0, 0, 0), + Utc.ymd(3000, 1, 1).and_hms(0, 0, 0), + 50, + 0, + ) + .unwrap(); + + assert!(!result.is_empty(), "Expected at least one row"); + } + + #[test] + fn test_collections_featured_by_volume_returns_non_empty() { + let conn = connect(); + + let result = super::by_volume( + &conn, + None::>, + OrderDirection::Desc, + Utc.ymd(1901, 1, 1).and_hms(0, 0, 0), + Utc.ymd(3000, 1, 1).and_hms(0, 0, 0), + 50, + 0, + ) + .unwrap(); + + assert!(!result.is_empty(), "Expected at least one row"); + } + + #[test] + fn test_collection_activities() { + todo!("Test collection_activities()"); + } +} diff --git a/crates/core/src/db/queries/featured_listings.rs b/crates/core/src/db/queries/featured_listings.rs index f44974dea..56de54122 100644 --- a/crates/core/src/db/queries/featured_listings.rs +++ b/crates/core/src/db/queries/featured_listings.rs @@ -88,3 +88,11 @@ pub fn list( .load(conn) .context("Failed to load featured listings") } + +#[cfg(test)] +mod tests { + #[test] + fn test_list() { + todo!("Test list()"); + } +} diff --git a/crates/core/src/db/queries/feed_event.rs b/crates/core/src/db/queries/feed_event.rs index 517747269..e4b152856 100644 --- a/crates/core/src/db/queries/feed_event.rs +++ b/crates/core/src/db/queries/feed_event.rs @@ -370,3 +370,15 @@ pub fn list_relay( .load(conn) .context("Failed to load feed events") } + +#[cfg(test)] +mod tests { + #[test] + fn test_list() { + todo!("Test list()"); + } + #[test] + fn test_list_relay() { + todo!("Test list_relay()"); + } +} diff --git a/crates/core/src/db/queries/graph_connection.rs b/crates/core/src/db/queries/graph_connection.rs index ab7dba65d..db2d80f38 100644 --- a/crates/core/src/db/queries/graph_connection.rs +++ b/crates/core/src/db/queries/graph_connection.rs @@ -69,3 +69,16 @@ pub fn list( .load(conn) .context("failed to load twitter enriched graph connections by addresses") } + +#[cfg(test)] +mod tests { + #[test] + fn test_connections() { + todo!("Test connections()"); + } + + #[test] + fn test_list() { + todo!("Test list()"); + } +} diff --git a/crates/core/src/db/queries/metadatas.rs b/crates/core/src/db/queries/metadatas.rs index 7df5a5c24..739bbe2b0 100644 --- a/crates/core/src/db/queries/metadatas.rs +++ b/crates/core/src/db/queries/metadatas.rs @@ -468,3 +468,15 @@ pub fn activities( .load(conn) .context("Failed to load nft(s) activities") } + +#[cfg(test)] +mod tests { + #[test] + fn test_list() { + todo!("Test list()"); + } + #[test] + fn test_activities() { + todo!("Test activities()"); + } +} diff --git a/crates/core/src/db/queries/nft_count.rs b/crates/core/src/db/queries/nft_count.rs index e471adad6..b82497084 100644 --- a/crates/core/src/db/queries/nft_count.rs +++ b/crates/core/src/db/queries/nft_count.rs @@ -303,3 +303,16 @@ pub fn store_creators( .load(conn) .context("Failed to load store creators counts") } + +#[cfg(test)] +mod tests { + use crate::db::test::prelude::*; + + #[test] + fn test_store_creators_minimal_passes() { + let conn = connect(); + + std::mem::drop(super::store_creators(&conn, Vec::::new()).unwrap()); + todo!("Check that the result is non-empty"); + } +} diff --git a/crates/core/src/db/queries/stats.rs b/crates/core/src/db/queries/stats.rs index da8bf2b5c..5903fa7d5 100644 --- a/crates/core/src/db/queries/stats.rs +++ b/crates/core/src/db/queries/stats.rs @@ -141,3 +141,19 @@ pub fn collection( .load(conn) .context("Failed to load collection mint stats") } + +#[cfg(test)] +mod tests { + #[test] + fn test_mint() { + todo!("Test mint()"); + } + #[test] + fn test_market() { + todo!("Test market()"); + } + #[test] + fn test_collection() { + todo!("Test collection()"); + } +} diff --git a/crates/core/src/db/queries/wallet.rs b/crates/core/src/db/queries/wallet.rs index ea8a83d89..3983c94e6 100644 --- a/crates/core/src/db/queries/wallet.rs +++ b/crates/core/src/db/queries/wallet.rs @@ -64,3 +64,11 @@ pub fn activities(conn: &Connection, address: impl ToSql) -> Result, Vec> for Batcher { async fn load( diff --git a/diesel.sh b/diesel.sh index 95057c1e4..67299313d 100755 --- a/diesel.sh +++ b/diesel.sh @@ -1,6 +1,7 @@ #!/bin/bash set -e +cd "$(dirname "$0")" for f in '' .dev .local; do f="./.env$f" @@ -13,6 +14,6 @@ done export DATABASE_URL -cd "$(dirname "$0")"/crates/core +cd crates/core -diesel "$@" +exec diesel "$@" diff --git a/scripts/pre-push.sh b/scripts/pre-push.sh index 4f8d3218e..4a4c3d170 100755 --- a/scripts/pre-push.sh +++ b/scripts/pre-push.sh @@ -5,14 +5,17 @@ set -e cd "$(dirname "$0")/.." lib_flags=(--workspace --lib --all-features) -build_flags=("${lib_flags[@]}") -build_flags+=(--bins) +bin_flags=(--workspace --bins --all-features) [[ -z "$CARGO" ]] && CARGO=cargo diff --unified <(./diesel.sh print-schema) crates/core/src/db/schema.rs "$CARGO" fmt --all --check -"$CARGO" clippy "${build_flags[@]}" --no-deps +"$CARGO" clippy "${lib_flags[@]}" --no-deps "$CARGO" doc "${lib_flags[@]}" --no-deps -"$CARGO" build "${build_flags[@]}" -"$CARGO" test "${build_flags[@]}" +"$CARGO" build "${lib_flags[@]}" +"$CARGO" test "${lib_flags[@]}" + +"$CARGO" clippy "${bin_flags[@]}" --no-deps +"$CARGO" build "${bin_flags[@]}" +"$CARGO" test "${bin_flags[@]}" diff --git a/test-migration.sh b/test-migration.sh new file mode 100755 index 000000000..3b4f3888d --- /dev/null +++ b/test-migration.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e + +cd "$(dirname "$0")" + +exec ./diesel.sh migration --migration-dir test_migrations "$@"