spark_test.yaml

name: "Delta Spark"
on:
  push:
    branches: [master, branch-*]
    paths-ignore:
      - '**.md'
      - '**.txt'
  pull_request:
    branches: [master, branch-*]
    paths-ignore:
      - '**.md'
      - '**.txt'
  workflow_dispatch:
    inputs:
      source-cache-key-suffix:
        description: Optional suffix to force a distinct Spark Maven cache key
        required: false
        type: string
# Every PR runs:
# 1. generate-matrix: split Spark versions (from CrossSparkVersions.scala) into published
#    vs source-build lists.
# 2. prepare_spark_source_cache: for each source-build Spark version, build Spark once and
#    save the Maven cache (via the prepare-spark-env action with build-on-miss). This job is
#    non-blocking; any source Spark build/cache failure is reported through source_spark_test.
# 3. test: published Spark shards (4.0/4.1) without waiting on step 2.
# 4. source_spark_test: source-built Spark shards (4.2 today) after step 2. It still runs if
#    prepare_spark_source_cache fails, so the source-built Spark signal stays non-blocking.
#
# test and source_spark_test share two composite actions:
#   - .github/actions/prepare-spark-env: installs the JDK and, for source-build rows,
#     restores the Spark Maven cache (the prepare-cache job builds/saves it; shards restore).
#   - .github/actions/run-delta-spark-tests: restores the SBT cache, publishes the pinned
#     Unity Catalog build, and runs the sharded tests.
# The Spark SHA, artifact version, and exact Maven cache key all come from
# project/scripts/get_spark_version_info.py, so they are not hand-assembled in this workflow.
jobs:
  # Generate Spark versions matrix from CrossSparkVersions.scala
  # This ensures the workflow always uses the versions defined in the build
  generate-matrix:
    name: "Generate Spark Versions Matrix"
    runs-on: ubuntu-24.04
    outputs:
      spark_versions: ${{ steps.generate.outputs.spark_versions }}
      source_build_spark_versions: ${{ steps.generate.outputs.source_build_spark_versions }}
      non_source_build_spark_versions: ${{ steps.generate.outputs.non_source_build_spark_versions }}
    steps:
      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
      - name: install java
        uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1
        with:
          distribution: "zulu"
          java-version: "17"
      - name: Generate Spark versions matrix
        id: generate
        run: |
          # The script automatically generates spark-versions.json from CrossSparkVersions.scala
          SPARK_VERSIONS=$(python3 project/scripts/get_spark_version_info.py --all-spark-versions)
          SOURCE_BUILD_SPARK_VERSIONS=$(python3 project/scripts/get_spark_version_info.py --source-build-spark-versions)
          NON_SOURCE_BUILD_SPARK_VERSIONS=$(python3 project/scripts/get_spark_version_info.py --non-source-build-spark-versions)
          echo "spark_versions=$SPARK_VERSIONS" >> $GITHUB_OUTPUT
          echo "source_build_spark_versions=$SOURCE_BUILD_SPARK_VERSIONS" >> $GITHUB_OUTPUT
          echo "non_source_build_spark_versions=$NON_SOURCE_BUILD_SPARK_VERSIONS" >> $GITHUB_OUTPUT
          echo "Generated Spark versions: $SPARK_VERSIONS"
          echo "Generated source-build Spark versions: $SOURCE_BUILD_SPARK_VERSIONS"
          echo "Generated non-source-build Spark versions: $NON_SOURCE_BUILD_SPARK_VERSIONS"

  # Builds ~/.m2/repository/org/apache/spark once per source-build Spark version so the
  # sharded source_spark_test jobs all restore the same cache instead of rebuilding 8x.
  prepare_spark_source_cache:
    name: "Prepare Spark ${{ matrix.spark_version }} source cache"
    runs-on: ubuntu-24.04
    needs: generate-matrix
    # Source-built Spark tracks unreleased APIs. Keep the producer non-blocking too,
    # otherwise a checkout/build/cache-save failure blocks the whole workflow before
    # source_spark_test can surface the same signal.
    continue-on-error: true
    strategy:
      fail-fast: false
      matrix:
        spark_version: ${{ fromJson(needs.generate-matrix.outputs.source_build_spark_versions) }}
    steps:
      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
      - name: Prepare Spark environment (build + cache)
        uses: ./.github/actions/prepare-spark-env
        with:
          spark-version: ${{ matrix.spark_version }}
          use-spark-source-cache: 'true'
          build-on-miss: 'true'
          cache-key-suffix: ${{ github.event.inputs['source-cache-key-suffix'] || '' }}

  # Published Spark artifacts only. This job does not depend on prepare_spark_source_cache.
  test:
    name: "DS: Spark ${{ matrix.spark_version }}, Scala ${{ matrix.scala }}, Shard ${{ matrix.shard }}"
    runs-on: ubuntu-24.04
    needs: generate-matrix
    strategy:
      fail-fast: false
      matrix:
        # Spark versions are dynamically generated from CrossSparkVersions.scala
        # DO NOT hardcode versions here - they are automatically loaded from the build configuration
        spark_version: ${{ fromJson(needs.generate-matrix.outputs.non_source_build_spark_versions) }}
        # These Scala versions must match those in the build.sbt
        scala: [2.13.16]
        # Important: This list of shards must be [0..NUM_SHARDS - 1]
        shard: [0, 1, 2, 3, 4, 5, 6, 7]
    steps:
      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
      - name: Prepare Spark environment
        uses: ./.github/actions/prepare-spark-env
        with:
          spark-version: ${{ matrix.spark_version }}
      - name: Run Delta Spark tests
        uses: ./.github/actions/run-delta-spark-tests
        with:
          spark-version: ${{ matrix.spark_version }}
          scala: ${{ matrix.scala }}
          shard: ${{ matrix.shard }}
          # Important: This must be the same as the length of shards in matrix
          num-shards: '8'

  # Validates source-built Spark for every PR using the SHA in this branch's metadata.
  source_spark_test:
    name: "DS Source Spark ${{ matrix.spark_version }}, Scala ${{ matrix.scala }}, Shard ${{ matrix.shard }}"
    runs-on: ubuntu-24.04
    needs: [generate-matrix, prepare_spark_source_cache]
    # Run even if the non-blocking prepare job failed. If no cache was produced, each
    # shard fails during restore/verification under this already non-blocking job.
    if: ${{ always() && needs.generate-matrix.result == 'success' }}
    # Source-built Spark lanes track unreleased Spark APIs and remain non-blocking
    # while still producing a full pass/fail signal for every PR.
    continue-on-error: true
    strategy:
      fail-fast: false
      matrix:
        spark_version: ${{ fromJson(needs.generate-matrix.outputs.source_build_spark_versions) }}
        scala: [2.13.16]
        shard: [0, 1, 2, 3, 4, 5, 6, 7]
    steps:
      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
      - name: Prepare Spark environment (restore source cache)
        uses: ./.github/actions/prepare-spark-env
        with:
          spark-version: ${{ matrix.spark_version }}
          use-spark-source-cache: 'true'
          build-on-miss: 'false'
          cache-key-suffix: ${{ github.event.inputs['source-cache-key-suffix'] || '' }}
      - name: Run Delta Spark tests
        uses: ./.github/actions/run-delta-spark-tests
        with:
          spark-version: ${{ matrix.spark_version }}
          scala: ${{ matrix.scala }}
          shard: ${{ matrix.shard }}
          num-shards: '8'