Repository navigation
Python and TypeScript SDKs: upload Flow Definition Graphs to the Dex Server #912
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Server ZD CI | |
| on: | |
| push: | |
| branches: [main] | |
| paths: | |
| - "server-zd/**" | |
| - "s3local/**" | |
| - "protos/**" | |
| - ".github/workflows/server-zd-ci.yml" | |
| pull_request: | |
| # labeled runs only the db9 object-store suite when the db9-tests label is added. | |
| types: [opened, synchronize, reopened, labeled] | |
| paths: | |
| - "server-zd/**" | |
| - "s3local/**" | |
| - "protos/**" | |
| - ".github/workflows/server-zd-ci.yml" | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: server-zd-${{ github.workflow }}-${{ github.ref }}-${{ github.event.action == 'labeled' && 'labeled' || 'changes' }} | |
| cancel-in-progress: true | |
| env: | |
| CARGO_TERM_COLOR: always | |
| CARGO_INCREMENTAL: "0" | |
| RUST_BACKTRACE: "1" | |
| RUSTUP_TOOLCHAIN: "1.97.1" | |
| # Formatting, Clippy, and tests run as parallel jobs. Each compiling job restores the cache that the | |
| # latest main run saved and never saves its own, so PRs start warm without evicting main's cache. | |
| jobs: | |
| fmt: | |
| name: Format | |
| if: github.event.action != 'labeled' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| defaults: | |
| run: | |
| working-directory: server-zd | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Install Rust | |
| run: rustup toolchain install 1.97.1 --profile minimal --component rustfmt | |
| - name: Check formatting | |
| run: make fmt-check | |
| clippy: | |
| name: Clippy | |
| if: github.event.action != 'labeled' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| defaults: | |
| run: | |
| working-directory: server-zd | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Install Rust | |
| run: rustup toolchain install 1.97.1 --profile minimal --component clippy | |
| - name: Install libclang for the RocksDB bindings | |
| run: sudo apt-get update && sudo apt-get install -y libclang-dev | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: server-zd | |
| shared-key: server-zd-clippy | |
| save-if: ${{ github.ref == 'refs/heads/main' }} | |
| - name: Run Clippy | |
| run: make clippy | |
| # The db9 suite runs the db9 object-store tests against PostgreSQL, which speaks the same wire | |
| # protocol. Pull requests include it only while they carry the db9-tests label. | |
| test: | |
| name: Tests (${{ matrix.suite.name }}) | |
| if: github.event.action != 'labeled' || github.event.label.name == 'db9-tests' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| suite: >- | |
| ${{ fromJSON( | |
| github.event.action == 'labeled' | |
| && '[{"name":"db9 against PostgreSQL","target":"db9ObjectStoreTests","postgres_image":"postgres:16"}]' | |
| || (contains(github.event.pull_request.labels.*.name, 'db9-tests') | |
| && '[{"name":"workspace","target":"test","postgres_image":""},{"name":"db9 against PostgreSQL","target":"db9ObjectStoreTests","postgres_image":"postgres:16"}]' | |
| || '[{"name":"workspace","target":"test","postgres_image":""}]') | |
| ) }} | |
| services: | |
| # An empty image skips the service, so only the db9 suite starts PostgreSQL. | |
| postgres: | |
| image: ${{ matrix.suite.postgres_image }} | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: dexzd | |
| ports: | |
| - 5432:5432 | |
| options: >- | |
| --health-cmd "pg_isready -U postgres -d dexzd" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 10 | |
| env: | |
| DEX_ZD_DB9_CONNECTION_STRING: postgres://postgres:postgres@localhost:5432/dexzd | |
| defaults: | |
| run: | |
| working-directory: server-zd | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Install Rust | |
| run: rustup toolchain install 1.97.1 --profile minimal | |
| - name: Install libclang for the RocksDB bindings | |
| run: sudo apt-get update && sudo apt-get install -y libclang-dev | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: server-zd | |
| shared-key: server-zd-test | |
| save-if: ${{ github.ref == 'refs/heads/main' }} | |
| - name: Run tests | |
| run: make ${{ matrix.suite.target }} | |
| paradedb: | |
| name: ParadeDB visibility parity | |
| if: github.event.action != 'labeled' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 45 | |
| defaults: | |
| run: | |
| working-directory: server-zd | |
| services: | |
| paradedb: | |
| image: paradedb/paradedb:0.26.1 | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: dexzd | |
| ports: | |
| - 5432:5432 | |
| options: >- | |
| --health-cmd "pg_isready -U postgres -d dexzd" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 10 | |
| env: | |
| DEX_ZD_PARADEDB_URL: postgres://postgres:postgres@localhost:5432/dexzd | |
| CARGO_BUILD_JOBS: "4" | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Install Rust | |
| run: rustup toolchain install 1.97.1 --profile minimal | |
| - name: Install libclang for the RocksDB bindings | |
| run: sudo apt-get update && sudo apt-get install -y libclang-dev | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: server-zd | |
| shared-key: server-zd-paradedb | |
| save-if: ${{ github.ref == 'refs/heads/main' }} | |
| - name: Compare ParadeDB and builtin search | |
| run: cargo test -p dexzd-search sql_visibility::integration::paradedb_matches_builtin_evaluator -- --exact --nocapture | |
| - name: Preserve legacy ParadeDB mixed-type errors | |
| run: cargo test -p dexzd-search paradedb_mixed_numeric_and_scalar_array_errors_are_preexisting -- --nocapture | |
| - name: Write the Postgres Attribute Store | |
| run: cargo test -p dexzd-attribute-store postgres_integration_tests -- --skip postgres_verify_full | |
| - name: Start PostgreSQL with a self-signed TLS certificate | |
| run: | | |
| set -euo pipefail | |
| tls_directory="$RUNNER_TEMP/dexzd-postgres-tls" | |
| mkdir -p "$tls_directory" | |
| openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj "/CN=Dex test CA" -addext "basicConstraints=critical,CA:TRUE" -addext "keyUsage=critical,keyCertSign,cRLSign" -keyout "$tls_directory/ca.key" -out "$tls_directory/ca.crt" | |
| openssl req -newkey rsa:2048 -nodes -subj "/CN=localhost" -keyout "$tls_directory/server.key" -out "$tls_directory/server.csr" | |
| cat > "$tls_directory/server.ext" <<'EOF' | |
| basicConstraints=critical,CA:FALSE | |
| keyUsage=critical,digitalSignature,keyEncipherment | |
| subjectAltName=DNS:localhost,IP:127.0.0.1 | |
| extendedKeyUsage=serverAuth | |
| EOF | |
| openssl x509 -req -days 1 -in "$tls_directory/server.csr" -CA "$tls_directory/ca.crt" -CAkey "$tls_directory/ca.key" -CAcreateserial -extfile "$tls_directory/server.ext" -out "$tls_directory/server.crt" | |
| openssl verify -CAfile "$tls_directory/ca.crt" "$tls_directory/server.crt" | |
| postgres_uid="$(docker run --rm --entrypoint id postgres:16 -u postgres)" | |
| chmod 600 "$tls_directory/server.key" | |
| sudo chown "$postgres_uid" "$tls_directory/server.key" | |
| chmod 644 "$tls_directory/ca.crt" "$tls_directory/server.crt" | |
| docker run -d --rm --name dexzd-postgres-tls -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=dexzd -p 5433:5432 -v "$tls_directory:/certs:ro" postgres:16 -c ssl=on -c ssl_cert_file=/certs/server.crt -c ssl_key_file=/certs/server.key -c ssl_ca_file=/certs/ca.crt | |
| for attempt in $(seq 1 30); do | |
| if docker exec dexzd-postgres-tls pg_isready -h 127.0.0.1 -p 5432 -U postgres -d dexzd; then | |
| exit 0 | |
| fi | |
| echo "Waiting for TLS PostgreSQL startup (attempt $attempt)" | |
| sleep 1 | |
| done | |
| docker logs dexzd-postgres-tls | |
| exit 1 | |
| - name: Verify Attribute Store PostgreSQL TLS connections | |
| env: | |
| DEX_ZD_TLS_POSTGRES_URL: postgres://postgres:postgres@localhost:5433/dexzd | |
| DEX_ZD_TLS_POSTGRES_CA_CERT: ${{ runner.temp }}/dexzd-postgres-tls/ca.crt | |
| run: | | |
| set -euo pipefail | |
| : "${DEX_ZD_TLS_POSTGRES_URL:?the TLS Postgres URL is required}" | |
| cargo test -p dexzd-attribute-store postgres_verify_full | |
| - name: Start TLS-only ParadeDB 17 for direct negotiation | |
| run: | | |
| set -euo pipefail | |
| image=paradedb/paradedb:0.26.1-pg17 | |
| docker run --rm --entrypoint postgres "$image" --version | grep -E 'PostgreSQL\) 17\.' | |
| tls_directory="$RUNNER_TEMP/dexzd-paradedb-direct-tls" | |
| mkdir -p "$tls_directory" | |
| openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj "/CN=Dex search test CA" -addext "basicConstraints=critical,CA:TRUE" -addext "keyUsage=critical,keyCertSign,cRLSign" -keyout "$tls_directory/ca.key" -out "$tls_directory/ca.crt" | |
| openssl req -newkey rsa:2048 -nodes -subj "/CN=localhost" -keyout "$tls_directory/server.key" -out "$tls_directory/server.csr" | |
| cat > "$tls_directory/server.ext" <<'EOF' | |
| basicConstraints=critical,CA:FALSE | |
| keyUsage=critical,digitalSignature,keyEncipherment | |
| subjectAltName=DNS:localhost,IP:127.0.0.1 | |
| extendedKeyUsage=serverAuth | |
| EOF | |
| openssl x509 -req -days 1 -in "$tls_directory/server.csr" -CA "$tls_directory/ca.crt" -CAkey "$tls_directory/ca.key" -CAcreateserial -extfile "$tls_directory/server.ext" -out "$tls_directory/server.crt" | |
| openssl verify -CAfile "$tls_directory/ca.crt" "$tls_directory/server.crt" | |
| cat > "$tls_directory/pg_hba.conf" <<'EOF' | |
| local all all trust | |
| hostssl all all 0.0.0.0/0 scram-sha-256 | |
| hostssl all all ::/0 scram-sha-256 | |
| hostnossl all all 0.0.0.0/0 reject | |
| hostnossl all all ::/0 reject | |
| EOF | |
| postgres_uid="$(docker run --rm --entrypoint id "$image" -u postgres)" | |
| chmod 600 "$tls_directory/server.key" | |
| sudo chown "$postgres_uid" "$tls_directory/server.key" | |
| chmod 644 "$tls_directory/ca.crt" "$tls_directory/server.crt" "$tls_directory/pg_hba.conf" | |
| docker run -d --rm --name dexzd-paradedb-direct-tls -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=dexzd -p 5434:5432 -v "$tls_directory:/certs:ro" "$image" -c ssl=on -c ssl_cert_file=/certs/server.crt -c ssl_key_file=/certs/server.key -c ssl_ca_file=/certs/ca.crt -c hba_file=/certs/pg_hba.conf | |
| for attempt in $(seq 1 30); do | |
| if docker exec dexzd-paradedb-direct-tls pg_isready -h 127.0.0.1 -p 5432 -U postgres -d dexzd; then | |
| exit 0 | |
| fi | |
| echo "Waiting for TLS ParadeDB startup (attempt $attempt)" | |
| sleep 1 | |
| done | |
| docker logs dexzd-paradedb-direct-tls | |
| exit 1 | |
| - name: Project and search over direct TLS | |
| env: | |
| DEX_ZD_PARADEDB_TLS_URL: postgres://postgres:postgres@localhost:5434/dexzd | |
| DEX_ZD_PARADEDB_TLS_CA_CERT: ${{ runner.temp }}/dexzd-paradedb-direct-tls/ca.crt | |
| DEX_ZD_REQUIRE_PARADEDB_TLS: '1' | |
| run: | | |
| set -euo pipefail | |
| : "${DEX_ZD_PARADEDB_TLS_URL:?the TLS ParadeDB URL is required}" | |
| : "${DEX_ZD_PARADEDB_TLS_CA_CERT:?the TLS ParadeDB CA certificate is required}" | |
| cargo test -p dexzd-search sql_visibility::integration::paradedb_projection_and_search_use_direct_tls -- --exact 2>&1 | tee "$RUNNER_TEMP/paradedb-direct-tls-test.log" | |
| grep -Eq 'test result: ok\. 1 passed; 0 failed;' "$RUNNER_TEMP/paradedb-direct-tls-test.log" | |
| - name: Serve with ParadeDB search | |
| run: cargo test -p dexzd-server --test serve_process paradedb_search_projects_commits_and_loads_a_cleared_schema -- --exact | |
| - name: Project the ParadeDB search outbox through an outage, a crash, a takeover, a backend switch, and a purge | |
| run: cargo test -p dexzd-server --test serve_process search_outbox::paradedb_ -- --nocapture | |
| - name: Hide purging runs from ParadeDB search | |
| run: cargo test -p dexzd-server --test serve_process purging_reads::frozen_purge_reads_are_not_found_except_load_blobs_with_paradedb_search -- --exact | |
| - name: Search Flows with work queue permissions in ParadeDB | |
| run: cargo test -p dexzd-server --test serve_process paradedb_search_finds_flows_with_work_queue_permissions -- --exact | |
| - name: Sync Attributes to PostgreSQL through the dev server | |
| run: 'cargo test -p dexzd-e2e-tests --test dev_server attribute_sync:: -- --skip attribute_sync::commit_to_table_p99_latency' | |
| # Runs alone so other tests do not share its CPU; --nocapture prints the percentiles. | |
| - name: Measure Attribute Sync commit-to-table latency | |
| run: | | |
| set -o pipefail | |
| cargo test -p dexzd-e2e-tests --test dev_server attribute_sync::commit_to_table_p99_latency_stays_within_two_seconds_under_steady_load -- --exact --nocapture 2>&1 | tee "$RUNNER_TEMP/attribute-sync-latency.log" | |
| grep -q '^attribute_sync_commit_to_table_latency:' "$RUNNER_TEMP/attribute-sync-latency.log" | |
| postgres: | |
| name: PostgreSQL visibility parity | |
| if: github.event.action != 'labeled' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 45 | |
| defaults: | |
| run: | |
| working-directory: server-zd | |
| services: | |
| postgres: | |
| image: postgres:17 | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: dexzd | |
| ports: | |
| - 5432:5432 | |
| options: >- | |
| --health-cmd "pg_isready -U postgres -d dexzd" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 10 | |
| env: | |
| DEX_ZD_POSTGRES_URL: postgres://postgres:postgres@localhost:5432/dexzd | |
| CARGO_BUILD_JOBS: "4" | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Install Rust | |
| run: rustup toolchain install 1.97.1 --profile minimal | |
| - name: Install libclang for the RocksDB bindings | |
| run: sudo apt-get update && sudo apt-get install -y libclang-dev | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: server-zd | |
| # Both dialects compile the same tests; reuse the existing main cache. | |
| shared-key: server-zd-paradedb | |
| save-if: false | |
| - name: Compare PostgreSQL and builtin search, including projection | |
| run: cargo test -p dexzd-search sql_visibility::integration::postgres_matches_builtin_evaluator -- --exact --nocapture | |
| - name: Backfill slots, verify plans, and check dialect isolation | |
| run: cargo test -p dexzd-search sql_visibility::slots::integration -- --nocapture | |
| - name: Serve, recover, and search with PostgreSQL | |
| run: cargo test -p dexzd-server --test serve_process postgres_ -- --nocapture |