ci.yml

name: CI

on:
  push:
    branches: [master]
    paths-ignore:
      - '*.md'
      - 'docs/**'
      - 'examples/**/*.md'
      - '.gitignore'
      - 'LICENSE*'
      - '.devcontainer/**'
  pull_request:
    branches: [master]
    paths-ignore:
      - '*.md'
      - 'docs/**'
      - 'examples/**/*.md'
      - '.gitignore'
      - 'LICENSE*'
      - '.devcontainer/**'

# Automatically cancel in-progress runs on the same branch
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/master' }}

env:
  CARGO_TERM_COLOR: always
  # Full debug info is most of what a debug target directory holds, and CI
  # caches those directories. Line tables keep file and line numbers in
  # panics and backtraces, which is all a failing CI run needs.
  CARGO_PROFILE_DEV_DEBUG: line-tables-only
  CARGO_PROFILE_TEST_DEBUG: line-tables-only

permissions:
  contents: read

jobs:
  python-interpreters:
    name: Python Interpreter Contract
    runs-on: ubuntu-latest
    outputs:
      versions: ${{ steps.contract.outputs.versions }}
    steps:
      - uses: actions/checkout@v7.0.1
        with:
          ref: ${{ github.event.pull_request.head.sha || github.sha }}
          token: ${{ github.token }}
          persist-credentials: false
      - uses: actions/setup-python@v7
        with:
          python-version: '3.14'
      - name: Validate metadata and generate interpreter matrix
        id: contract
        run: python .github/scripts/wheel_interpreters.py --github-output "$GITHUB_OUTPUT"

  test-suite:
    name: Test Suite
    runs-on: ubuntu-latest
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        test-scope: [core, features]
        include:
          - test-scope: core
            test-type: core
            features: ""
            all-features: false
            timeout: 12
            skip-on-draft: false
          - test-scope: features
            test-type: all
            features: ""
            all-features: true
            timeout: 30
            skip-on-draft: true
    steps:
      - name: Checkout code
        uses: actions/checkout@v7.0.1
        with:
          ref: ${{ github.event.pull_request.head.sha || github.sha }}
          token: ${{ github.token }}

      - name: Setup Rust
        uses: ./.github/actions/setup-rust
        with:
          cache-prefix: rust-ci-${{ matrix.test-scope }}

      - name: Run ${{ matrix.test-scope }} tests
        if: github.event_name == 'push' || matrix.skip-on-draft == false || github.event.pull_request.draft == false
        uses: ./.github/actions/run-tests
        with:
          test-type: ${{ matrix.test-type }}
          features: ${{ matrix.features }}
          all-features: ${{ matrix.all-features }}
          timeout: ${{ matrix.timeout }}

  quality-checks:
    name: Code Quality
    runs-on: ubuntu-latest
    # Usually about a minute, but it hit the 10-minute ceiling and was
    # cancelled on a cold cache. A timeout here is worse than a slow run: the
    # cache is only written on success, so the next run starts cold too.
    timeout-minutes: 20
    steps:
      - name: Checkout code
        uses: actions/checkout@v7.0.1
        with:
          ref: ${{ github.event.pull_request.head.sha || github.sha }}
          token: ${{ github.token }}

      - name: Setup Rust
        uses: ./.github/actions/setup-rust
        with:
          cache-prefix: rust-ci

      - name: Check formatting
        run: |
          echo "Checking code formatting..."
          cargo fmt --all -- --check
          echo "Formatting check passed"

      - name: Run clippy
        run: |
          echo "Running clippy analysis..."
          cargo clippy --lib --tests --all-features -- -D warnings
          echo "Clippy analysis passed"

      - name: Check profile report schema drift
        run: |
          cargo run --example generate_profile_schema
          if [ -n "$(git status --porcelain -- docs/schema/)" ]; then
            git status --short -- docs/schema/
            git diff -- docs/schema/
            exit 1
          fi

      - name: Check documentation
        run: |
          echo "Checking documentation..."
          cargo doc --no-deps --document-private-items
          echo "Documentation check passed"

      # Guard for issue #364: swallowed errors on decode paths must carry a
      # decode-audit classification comment.
      - name: Check decode-path error handling
        run: python3 .github/scripts/check_decode_audit.py

  feature-checks:
    name: Feature Checks (${{ matrix.scope }})
    runs-on: ubuntu-latest
    # Same cold-cache trap as python-wheel-smoke below. A dependency bump
    # changes Cargo.lock and so the rust-cache key, and the all-features-crates
    # scope builds four feature graphs back to back: warm it is ~3m, cold it
    # needed ~21m on the #566 parquet bump. At 20 the tests all passed and the
    # job was killed 87s later during the post-job cache save, so nothing was
    # written — leaving the next run on that same lockfile cold too. The
    # timeout has to clear the build *and* the save, or the job cannot recover.
    #
    # Raised 35 -> 45 when the scope went from four member crates to eight: the
    # cold figure above was measured against the smaller list.
    timeout-minutes: 45
    strategy:
      fail-fast: false
      matrix:
        include:
          - scope: minimal-facade
            commands: |
              cargo check --no-default-features
              cargo test --test public_api_facade --no-default-features
          - scope: async-streaming
            commands: |
              cargo check --no-default-features --features async-streaming
              cargo test --test public_api_facade --no-default-features --features async-streaming
              cargo test --no-default-features --features async-streaming --test async_streaming -- profile_url
          - scope: all-features-crates
            commands: |
              cargo test --test public_api_facade --all-features
              # `cargo test --lib`/`--doc` at the root only covers the root
              # `dataprof` package, so a member crate runs only if it is named
              # here. Every library member is listed: leaving one out silently
              # ungates its whole suite, which is how 517 tests in core,
              # metrics, csv and runtime went unrun. dataprof-db and
              # dataprof-python are deliberately absent, covered by the
              # database and Python jobs respectively.
              cargo test -p dataprof-core --all-features
              cargo test -p dataprof-csv --all-features
              cargo test -p dataprof-engines --all-features
              cargo test -p dataprof-json --all-features
              cargo test -p dataprof-metrics --all-features
              cargo test -p dataprof-parquet --all-features
              cargo test -p dataprof-partial --all-features
              cargo test -p dataprof-runtime --all-features
    steps:
      - name: Checkout code
        uses: actions/checkout@v7.0.1
        with:
          ref: ${{ github.event.pull_request.head.sha || github.sha }}
          token: ${{ github.token }}

      - name: Setup Rust
        uses: ./.github/actions/setup-rust
        with:
          cache-prefix: rust-features-${{ matrix.scope }}

      - name: Run feature checks
        run: ${{ matrix.commands }}

  # The workspace advertises Rust 1.96 in Cargo.toml, README.md and AGENTS.md,
  # and until this job nothing compiled against it: every other Rust job here
  # runs the pinned 1.98, and the 1.96 job in release.yml only ran
  # `cargo metadata`, which parses manifests without touching source. #674.
  msrv:
    name: MSRV Compile Gate
    runs-on: ubuntu-latest
    # Six feature graphs back to back, so the cold-cache reasoning under
    # feature-checks applies here too.
    timeout-minutes: 45
    steps:
      - name: Checkout code
        uses: actions/checkout@v7.0.1
        with:
          ref: ${{ github.event.pull_request.head.sha || github.sha }}
          token: ${{ github.token }}

      # Deliberately not ./.github/actions/setup-rust: that pins 1.98, the lint
      # and test toolchain. This job's whole point is the *minimum* one, and a
      # `uses:` ref cannot be templated, so the version is written out here.
      # msrv_check.py fails if this pin and `rust-version` ever disagree.
      - name: Install Rust 1.96 (workspace MSRV)
        uses: dtolnay/rust-toolchain@1.96

      - name: Cache Rust dependencies
        uses: Swatinem/rust-cache@v2
        with:
          prefix-key: rust-msrv
          shared-key: "v4"
          # As in setup-rust: a PR-scoped cache only evicts master's.
          save-if: ${{ github.event_name != 'pull_request' }}

      # pyo3's build script needs an interpreter to configure the extension
      # crate arm of the matrix.
      - name: Setup Python
        uses: actions/setup-python@v7
        with:
          python-version: '3.10'

      - name: Compile MSRV feature graphs
        run: python3 .github/scripts/msrv_check.py

  python-quality:
    name: Python Lint & Format
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
      - name: Checkout code
        uses: actions/checkout@v7.0.1
        with:
          ref: ${{ github.event.pull_request.head.sha || github.sha }}
          token: ${{ github.token }}

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          cache-dependency-glob: |
            pyproject.toml
            uv.lock

      # Plain `uv run` syncs the project first, which compiles the Rust
      # extension through the PEP 517 backend. Nothing here needs it: ruff
      # reads source, and ty resolves the native module from the checked-in
      # .pyi stubs. It is not free either — when #595 added async-streaming
      # and parquet-async to [tool.maturin] features, rustls/tokio/reqwest
      # entered the graph and the build alone took 5m06s, overrunning this
      # job's 5-minute timeout (which GitHub reports as "cancelled").
      - name: Install dev tools
        run: uv sync --no-install-project

      - name: Check formatting
        run: uv run --no-sync ruff format --check python/ .github/scripts/ .claude/skills/dataprof/scripts/

      - name: Lint
        run: uv run --no-sync ruff check python/ .github/scripts/ .claude/skills/dataprof/scripts/

      # ty is pre-1.0 (Astral's type checker) but the codebase passes it, so
      # keep it blocking to prevent regressions.
      - name: Type check (ty)
        run: uv run --no-sync ty check python/

  python-tests:
    name: Python Tests (${{ matrix.python }})
    runs-on: ubuntu-latest
    needs: python-interpreters
    timeout-minutes: 15
    strategy:
      fail-fast: false
      matrix:
        python: ${{ fromJSON(needs.python-interpreters.outputs.versions) }}
    env:
      UV_PYTHON: ${{ matrix.python }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v7.0.1
        with:
          ref: ${{ github.event.pull_request.head.sha || github.sha }}
          token: ${{ github.token }}

      # One cache for every interpreter, written by the oldest. Only pyo3 and
      # the extension crate differ between interpreters; the ~400 crates under
      # them are identical, and five copies of them filled the cache quota.
      - name: Setup Rust
        uses: ./.github/actions/setup-rust
        with:
          cache-prefix: rust-python
          save-cache: ${{ matrix.python == fromJSON(needs.python-interpreters.outputs.versions)[0] }}

      - name: Setup Python
        uses: actions/setup-python@v7
        with:
          python-version: ${{ matrix.python }}

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          cache-dependency-glob: |
            pyproject.toml
            uv.lock

      # Plain `uv run maturin develop` syncs the project first, which builds
      # the extension through the PEP 517 backend, and then maturin compiles
      # the whole tree again: ~6 minutes, then 7m13s, on run 36228290605.
      # Every later `uv run` needs --no-sync, or it would build it once more.
      - name: Build Python bindings
        run: |
          uv sync --no-install-project
          uv run --no-sync maturin develop

      - name: Run Python tests
        run: uv run --no-sync pytest python/tests/ -v -ra

  python-wheel-smoke:
    # The pytest job runs against an editable build inside the dev environment,
    # so it cannot see packaging mistakes or an accidental dependency on pandas.
    # This one installs the built wheel into a bare venv, the way a user would.
    name: Wheel Smoke (${{ matrix.os }}, Python ${{ matrix.python }})
    runs-on: ${{ matrix.os }}
    needs: python-interpreters
    # A dependency bump changes Cargo.lock, which changes the rust-cache key, so
    # every such PR builds ~400 crates cold. Warm, Windows takes 6 minutes; cold,
    # it needs more than the 20 this used to allow, and a timeout is worse than a
    # slow run because the cache is only written on success — so the next run is
    # cold too, and it never recovers on its own.
    timeout-minutes: 35
    strategy:
      fail-fast: false
      matrix:
        python: ${{ fromJSON(needs.python-interpreters.outputs.versions) }}
        os: [ubuntu-latest, macos-latest, windows-latest]
        include:
          - os: ubuntu-latest
            venv_python: .smoke/bin/python
          - os: macos-latest
            venv_python: .smoke/bin/python
          - os: windows-latest
            venv_python: .smoke/Scripts/python
    env:
      UV_PYTHON: ${{ matrix.python }}
    defaults:
      run:
        shell: bash
    steps:
      - name: Checkout code
        uses: actions/checkout@v7.0.1
        with:
          ref: ${{ github.event.pull_request.head.sha || github.sha }}
          token: ${{ github.token }}

      # One cache per OS, written by the oldest interpreter. Each pair builds
      # a different extension, but only pyo3 and the extension crate differ.
      - name: Setup Rust
        uses: ./.github/actions/setup-rust
        with:
          cache-prefix: rust-wheel-smoke-${{ matrix.os }}
          save-cache: ${{ matrix.python == fromJSON(needs.python-interpreters.outputs.versions)[0] }}

      - name: Setup Python
        uses: actions/setup-python@v7
        with:
          python-version: ${{ matrix.python }}

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          cache-dependency-glob: |
            pyproject.toml
            uv.lock

      - name: Build wheel
        # Deliberately not `uv run maturin build`: that syncs the project first,
        # which builds the extension through the PEP 517 backend into its own
        # temporary target dir, and then maturin compiles the whole tree again
        # into ./target. With a warm cache the duplication is invisible; on a
        # cold one it cost nine minutes here and blew the Windows timeout.
        # Nothing needs the project inside .venv — the wheel goes into a bare
        # venv in the next step, which is the whole point of this job.
        run: |
          uv sync --no-install-project
          uv run --no-sync maturin build --out dist

      - name: Install the wheel into a bare venv
        # --no-index proves the wheel needs nothing from PyPI: the base package
        # declares no dependencies, and the acceptance snippets must not need one.
        run: |
          python -m venv .smoke
          ${{ matrix.venv_python }} -m pip install --upgrade pip
          ${{ matrix.venv_python }} -m pip install --no-index --find-links dist dataprof

      - name: Run acceptance snippets
        # Run from outside the repo so `import dataprof` cannot resolve to python/.
        run: |
          cp .github/scripts/wheel_smoke.py "${{ runner.temp }}/wheel_smoke.py"
          SMOKE_PYTHON="$PWD/${{ matrix.venv_python }}"
          cd "${{ runner.temp }}"
          "$SMOKE_PYTHON" wheel_smoke.py

  examples-smoke:
    # The examples print real profiler output, so they double as executable docs:
    # a change to a metric's semantics breaks them here rather than in a user's
    # first five minutes. Both languages must agree on the numbers they print.
    name: Examples Smoke
    runs-on: ubuntu-latest
    # Cold, the Rust examples take ~8m and the Python bindings build ~11.5m
    # (run 35999584206), so 20 was hit during the post-job cache save and left
    # the next run on that lockfile cold as well.
    timeout-minutes: 30
    steps:
      - name: Checkout code
        uses: actions/checkout@v7.0.1
        with:
          ref: ${{ github.event.pull_request.head.sha || github.sha }}
          token: ${{ github.token }}

      - name: Setup Rust
        uses: ./.github/actions/setup-rust
        with:
          cache-prefix: rust-examples

      - name: Setup Python
        uses: actions/setup-python@v7
        with:
          python-version: '3.10'

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          cache-dependency-glob: |
            pyproject.toml
            uv.lock

      - name: Run Rust examples (default features)
        run: |
          cargo run --example messy_csv_inspection
          cargo run --example etl_quality_gate
          cargo run --example before_after_cleaning

      # Sync without the project, then build it once; see python-tests.
      - name: Build Python bindings
        run: |
          uv sync --no-install-project
          uv run --no-sync maturin develop

      - name: Run Python examples
        run: |
          uv run --no-sync python python/examples/messy_csv_inspection.py
          uv run --no-sync python python/examples/etl_quality_gate.py
          uv run --no-sync python python/examples/before_after_cleaning.py

  python-async-url-tests:
    name: Python Async URL Tests (${{ matrix.variant }})
    runs-on: ubuntu-latest
    timeout-minutes: 15
    strategy:
      fail-fast: false
      matrix:
        include:
          - variant: async-streaming
            features: "python,python-async,async-streaming"
          - variant: parquet-async
            features: "python,python-async,parquet-async"
    steps:
      - name: Checkout code
        uses: actions/checkout@v7.0.1
        with:
          ref: ${{ github.event.pull_request.head.sha || github.sha }}
          token: ${{ github.token }}

      - name: Setup Rust
        uses: ./.github/actions/setup-rust
        with:
          cache-prefix: rust-python-url-${{ matrix.variant }}

      - name: Setup Python
        uses: actions/setup-python@v7
        with:
          python-version: '3.10'

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          cache-dependency-glob: |
            pyproject.toml
            uv.lock

      # Sync without the project, then build it once; see python-tests.
      - name: Build Python bindings
        run: |
          uv sync --no-install-project
          uv run --no-sync maturin develop --features "${{ matrix.features }}"

      - name: Run async URL tests
        run: uv run --no-sync pytest python/tests/test_async_url_api.py -v -ra

  db-integration-tests:
    name: Database Integration Tests
    runs-on: ubuntu-latest
    timeout-minutes: 20
    services:
      postgres:
        image: postgres:16-alpine
        env:
          POSTGRES_DB: dataprof_test
          POSTGRES_USER: dataprof
          POSTGRES_PASSWORD: dev_password_123
        ports:
          - 5432:5432
        options: >-
          --health-cmd "pg_isready -U dataprof -d dataprof_test"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
      mysql:
        image: mysql:8.0
        env:
          MYSQL_ROOT_PASSWORD: root_password_123
          MYSQL_DATABASE: dataprof_test
          MYSQL_USER: dataprof
          MYSQL_PASSWORD: dev_password_123
        ports:
          - 3306:3306
        options: >-
          --health-cmd "mysqladmin ping -h localhost"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 10
          --health-start-period 30s
    env:
      POSTGRES_TEST_URL: "postgresql://dataprof:dev_password_123@localhost:5432/dataprof_test"
      MYSQL_TEST_URL: "mysql://dataprof:dev_password_123@127.0.0.1:3306/dataprof_test"
    steps:
      - name: Checkout code
        uses: actions/checkout@v7.0.1
        with:
          ref: ${{ github.event.pull_request.head.sha || github.sha }}
          token: ${{ github.token }}

      - name: Setup Rust
        uses: ./.github/actions/setup-rust
        with:
          cache-prefix: rust-db
          components: ''

      - name: Initialize test databases
        run: |
          sudo apt-get update && sudo apt-get install -y postgresql-client default-mysql-client
          PGPASSWORD=dev_password_123 psql -h localhost -U dataprof -d dataprof_test \
            -f .devcontainer/init-scripts/postgres/01-init-schema.sql || true
          mysql -h 127.0.0.1 -u dataprof -pdev_password_123 dataprof_test \
            < .devcontainer/init-scripts/mysql/01-init-schema.sql || true

      # all-db already includes the SQLite feature and its tests. Running a
      # SQLite-only command first forces Cargo to build two feature graphs and
      # can prevent a cold job from reaching the post-job cache save.
      - name: Run Rust database tests (all DBs)
        run: cargo test --test database_integration --features "all-db"

      # The facade command above does not select member-crate tests. Run the
      # whole database package, including unit, decoding and column-order tests.
      - name: Run all dataprof-db tests
        run: cargo test -p dataprof-db --all-features

  # The Python extension uses a different Cargo profile and feature graph.
  # Build it in parallel so neither cold build blocks the other's cache save.
  python-db-tests:
    name: Python Database Tests
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
      - name: Checkout code
        uses: actions/checkout@v7.0.1
        with:
          ref: ${{ github.event.pull_request.head.sha || github.sha }}
          token: ${{ github.token }}

      - name: Setup Rust
        uses: ./.github/actions/setup-rust
        with:
          cache-prefix: rust-python-db
          components: ''

      - name: Setup Python
        uses: actions/setup-python@v7
        with:
          python-version: '3.10'

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          cache-dependency-glob: |
            pyproject.toml
            uv.lock

      # Sync without the project, then build it once; see python-tests.
      - name: Build Python bindings with database support
        run: |
          uv sync --no-install-project
          uv run --no-sync maturin develop --features "python,python-async,sqlite"

      - name: Require SQLite database support
        run: |
          uv run --no-sync python - <<'PY'
          import dataprof
          from dataprof._dataprof import (
              analyze_database_async,
              count_table_rows_async,
              get_table_schema_async,
              test_connection_async,
          )

          caps = dataprof.capabilities()
          assert caps.database and "sqlite" in caps.database_connectors, caps
          print(caps)
          PY

      - name: Run Python database tests
        run: >-
          uv run --no-sync pytest
          python/tests/test_database_api.py
          python/tests/test_database_option_parity.py
          python/tests/test_column_order.py
          python/tests/test_engine_parity.py::test_sqlite_parity
          -v -ra