smartcontractkit · prashantkumar1982 · Aug 5, 2026 · Jul 31, 2026 · Aug 3, 2026 · Aug 4, 2026
@@ -0,0 +1,338 @@
+name: CRE Mixed-Env Tests
+
+# Dedicated workflow for the "mixed-env" topology (2 nodes on the PR image, 2 on a
+# develop image) that fails a job if PR and develop nodes disagree — see
+# core/scripts/cre/environment/docs/mixed-env.md. Kept separate from
+# cre-system-tests.yaml so that workflow stays simple; this one is 100% mixed-env
+# so it needs no per-topology conditionals.
+
+on:
+  workflow_dispatch:
+    inputs:
+      chainlink_image_repository_path:
+        description:
+          "ECR repository name used to compose the PR image with chainlink_image_tag."
+        required: true
+        type: string
+      chainlink_image_tag:
+        required: true
+        type: string
+        description: "Chainlink (PR) image tag to use."
+      ecr:
+        required: false
+        type: choice
+        options:
+          - "sdlc"
+          - "public"
+        default: "sdlc"
+        description: "SDLC or public ECR registry for the Chainlink image."
+      chainlink_version:
+        required: false
+        type: string
+        default: ""
+        description: "Chainlink repo ref to check out. Defaults to github.sha."
+      mixed_env_baseline_image:
+        required: false
+        type: string
+        default: ""
+        description:
+          "Full baseline (develop) chainlink image ref. If empty, defaults to the
+          develop nightly image."
+  workflow_call:
+    inputs:
+      chainlink_image_repository_path:
+        description:
+          "ECR repository name used to compose the PR image with chainlink_image_tag."
+        required: true
+        type: string
+      chainlink_image_tag:
+        required: true
+        type: string
+        description: "Chainlink (PR) image tag to use."
+      ecr:
+        type: string
+        required: true
+        description: "SDLC (sdlc) or public (public) ECR registry for the Chainlink image."
+      chainlink_version:
+        required: false
+        type: string
+        default: ""
+        description: "Chainlink repo ref to check out. Defaults to github.sha."
+      mixed_env_baseline_image:
+        required: false
+        type: string
+        default: ""
+        description:
+          "Full baseline (develop) chainlink image ref. If empty, defaults to the
+          develop nightly image."
+
+jobs:
+  define-test-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.define-matrix.outputs.matrix }}
+    permissions:
+      contents: read
+    steps:
+      - name: Define mixed-env test matrix
+        id: define-matrix
+        shell: bash
+        run: |
+          # Tests worth running under mixed-env: the OCR3/DON2DON-heavy ones.
+          TESTS='[
+            "Test_CRE_V2_Suite_Bucket_A",
+            "Test_CRE_V2_Suite_Bucket_B",
+            "Test_CRE_V2_EVM_Read_HeavyCalls",
+            "Test_CRE_V2_EVM_Read_StateQueries",
+            "Test_CRE_V2_EVM_Read_TxArtifacts"
+          ]'
+          matrix=$(jq -c -n \
+            --argjson tests "$TESTS" \
+            --argjson run_id "${{ github.run_id }}" \
+            --arg run_attempt "${{ github.run_attempt }}" '
+            $tests | to_entries | map({
+              test_name: .value,
+              test_id: .key,
+              runs_on: "runs-on=\($run_id)-\(.key)-\($run_attempt)/cpu=16/ram=64/family=m7i+m8i/spot=co/image=ubuntu24-full-x64/extras=s3-cache+tmpfs"
+            })')
+          echo "matrix=$matrix" | tee -a "${GITHUB_OUTPUT}"
+
+  run-mixed-env-tests:
+    name: ${{ matrix.tests.test_name }} (mixed-env)
+    permissions:
+      contents: read
+      id-token: write
+    strategy:
+      fail-fast: false
+      matrix:
+        tests: ${{ fromJson(needs.define-test-matrix.outputs.matrix) }}
+    needs: [define-test-matrix]
+    runs-on: ${{ matrix.tests.runs_on }}
+    environment:
+      name: integration
+      deployment: false
+    timeout-minutes: 10
+    env:
+      ENABLE_AUTO_QUARANTINE: "true"
+      # PR image; the develop/baseline image is resolved in the render step below.
+      CHAINLINK_IMAGE_FULL: ${{ inputs.ecr == 'public' && format('public.ecr.aws/{0}:{1}', inputs.chainlink_image_repository_path, inputs.chainlink_image_tag) || format('{0}.dkr.ecr.{1}.amazonaws.com/{2}:{3}', secrets.QA_AWS_ACCOUNT_NUMBER, secrets.QA_AWS_REGION, inputs.chainlink_image_repository_path, inputs.chainlink_image_tag) }}
+      CTF_CHIP_INGRESS_IMAGE: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/atlas-chip-ingress:da84cb72d3a160e02896247d46ab4b9806ebee2f
+      CTF_CHIP_CONFIG_IMAGE: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/atlas-chip-config:7b4e9ee68fd1c737dd3480b5a3ced0188f29b969
+
+    steps:
+      - name: Enable S3 Cache for Self-Hosted Runners
+        uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc # v2.2.0
+
+      - name: Checkout
+        uses: actions/checkout@v7
+        with:
+          ref: ${{ inputs.chainlink_version || github.sha }}
+          persist-credentials: false
+
+      - name: Set up Go
+        id: setup-go
+        uses: smartcontractkit/.github/actions/ctf-setup-go@fa1d48a33e24f9b3b9f8c52e99a578a4597cb2a5 # v0.4.0
+        with:
+          go_mod_path: system-tests/tests/go.mod
+          cache_key_id: integration-tests-v1
+          cache_builds: true
+          cache_restore_only: "true"
+
+      - name: Restore Pre-Compiled Test Binaries from S3 Cache
+        uses: actions/cache/restore@v6
+        with:
+          path: system-tests/tests/bin/
+          key: test-binaries-cre-${{ inputs.chainlink_version || github.sha }}
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3
+        with:
+          aws-region: ${{ secrets.QA_AWS_REGION }}
+          role-to-assume: ${{ secrets.AWS_CTF_READ_ACCESS_ROLE_ARN }}
+          role-duration-seconds: 1800
+          mask-aws-account-id: true
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64 # v2.1.6
+        with:
+          registries: ${{ format('{0},{1}', secrets.QA_AWS_ACCOUNT_NUMBER,
+            secrets.AWS_ACCOUNT_ID_PROD) }}
+        env:
+          AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+
+      - name: Authenticate to ECR (public)
+        id: login-ecr-public
+        uses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64 # v2.1.6
+        with:
+          registry-type: public
+        env:
+          AWS_REGION: us-east-1
+
+      - name: Cache gotestsum
+        id: cache-gotestsum
+        uses: actions/cache@v6
+        with:
+          path: ~/go/bin/gotestsum
+          key: ${{ runner.os }}-${{ runner.arch }}-gotestsum-1.13.0
+
+      - name: Set up gotestsum
+        shell: bash
+        run: |
+          if [ -f ~/go/bin/gotestsum ]; then
+            echo "Using cached gotestsum"
+          else
+            echo "::group::Install gotestsum"
+            go install gotest.tools/gotestsum@v1.13.0
+            echo "::endgroup::"
+          fi
+
+      - name: Render mixed-env topology
+        working-directory: core/scripts/cre/environment
+        shell: bash
+        env:
+          CRE_PR_IMAGE: ${{ env.CHAINLINK_IMAGE_FULL }}
+          BASELINE_IMAGE_OVERRIDE: ${{ inputs.mixed_env_baseline_image }}
+          ECR_ACCOUNT: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
+          ECR_REGION: ${{ secrets.QA_AWS_REGION }}
+        run: |
+          if [[ -n "${BASELINE_IMAGE_OVERRIDE}" ]]; then
+            CRE_BASELINE_IMAGE="${BASELINE_IMAGE_OVERRIDE}"
+          else
+            # Default to the develop nightly image (built daily + on develop pushes).
+            # Core variant (no -plugins) to match the PR image built by integration-tests.
+            DATE="$(date +'%Y%m%d')"
+            CRE_BASELINE_IMAGE="${ECR_ACCOUNT}.dkr.ecr.${ECR_REGION}.amazonaws.com/chainlink:nightly-${DATE}"
+          fi
+          export CRE_PR_IMAGE CRE_BASELINE_IMAGE
+          echo "PR image:       ${CRE_PR_IMAGE}"
+          echo "Baseline image: ${CRE_BASELINE_IMAGE}"
+          bash configs/render-mixed-env.sh
+
+      - name: Start local CRE
+        id: start-local-cre
+        uses: ./.github/actions/start-local-cre-environment
+        with:
+          jd-image:
+            "${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION
+            }}.amazonaws.com/job-distributor:0.28.0"
+          # Empty on purpose: images are baked per-node into the rendered TOML, and a
+          # non-empty CTF_CHAINLINK_IMAGE would force every node onto one image.
+          chainlink-image: ""
+          chip-router-image: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{
+            secrets.QA_AWS_REGION }}.amazonaws.com/local-cre-chip-router:v1.0.1"
+          ctf-configs: configs/mixed-env-don.toml
+          retry-count: "3"
+          retry-delay-seconds: "15"
+          cleanup-on-error: "false"
+          working-directory: core/scripts/cre/environment
+
+      - name: Setup GitHub token using GATI
+        id: github-token
+        uses: smartcontractkit/.github/actions/setup-github-token@setup-github-token/v1
+        with:
+          aws-role-arn: ${{ secrets.AWS_OIDC_CHAINLINK_READ_ONLY_TOKEN_ISSUER_ROLE_ARN }}
+          aws-lambda-url: ${{ secrets.AWS_INFRA_RELENG_TOKEN_ISSUER_LAMBDA_URL }}
+          aws-region: us-west-2
+          aws-role-duration-seconds: "1800"
+          set-git-config: "true"
+
+      - name: Run CRE Smoke system tests (mixed-env)
+        id: run-tests
+        shell: bash
+        working-directory: system-tests/tests
+        continue-on-error: ${{ env.ENABLE_AUTO_QUARANTINE == 'true' }}
+        env:
+          TEST_NAME: ${{ matrix.tests.test_name }}
+          TEST_TIMEOUT: 7m
+          RUN_QUARANTINED_TESTS: "true"
+          TOPOLOGY_NAME: mixed-env
+          GITHUB_TOKEN: ${{ steps.github-token.outputs.access-token || '' }}
+          PARALLEL_COUNT: "10"
+          CRE_TEST_PARALLEL_ENABLED: "true"
+        run: |
+          echo "Starting test: '${TEST_NAME}' (mixed-env)"
+          if [ -f "./bin/cre-smoke.test" ]; then
+            echo "Using precompiled binary"
+            (
+              cd smoke/cre && \
+              gotestsum \
+                --jsonfile=/tmp/gotest.log \
+                --junitfile=/tmp/junit-report.xml \
+                --format=github-actions \
+                --raw-command -- \
+                go tool test2json -t -p github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre ../../bin/cre-smoke.test -test.v -test.run "^(${TEST_NAME})$" -test.timeout "${TEST_TIMEOUT}" -test.count=1 -test.parallel="${PARALLEL_COUNT}"
+            )
+          else
+            echo "No precompiled binary found. Building from scratch..."
+            gotestsum \
+              --jsonfile=/tmp/gotest.log \
+              --junitfile=/tmp/junit-report.xml \
+              --format=github-actions \
+              -- \
+              -v -run "^(${TEST_NAME})$" -timeout "${TEST_TIMEOUT}" -count=1 -parallel="${PARALLEL_COUNT}" \
+              github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre
+          fi
+
+          exit_code="$?"
+          if [ "$exit_code" -eq 0 ]; then
+            echo "tests_result=✅ Tests passed" >> "${GITHUB_OUTPUT}"
+          fi
+
+      - name: Check for non-determinism
+        # Hard gate. The smoke step above is continue-on-error (auto-quarantine),
+        # which would swallow the in-test TestMain scan, so re-scan the live node
+        # containers here and fail the job on any marker.
+        if: ${{ always() }}
+        shell: bash
+        run: |
+          set -uo pipefail
+          markers=(
+            "This is commonly caused by non-determinism"
+            "received messages with the same id and different payloads"
+            "received multiple unique responses for the same request"
+            "response quorum unreachable"
+          )
+          hit=0
+          for c in $(docker ps -a --format '{{.Names}}'); do
+            logs="$(docker logs "$c" 2>&1 || true)"
+            for m in "${markers[@]}"; do
+              if grep -qF -- "$m" <<<"$logs"; then
+                echo "::error::Non-Determinism introduced — container=${c} marker=\"${m}\""
+                hit=1
+              fi
+            done
+          done
+          if [[ "${hit}" -ne 0 ]]; then
+            echo "Non-Determinism introduced: PR and develop nodes disagreed."
+            exit 1
+          fi
+          echo "mixed-env non-determinism check: no markers found"
+
+      - name:
+          Analyze and upload test results (${{ steps.run-tests.outputs.tests_result
+          }})
+        if: ${{ !cancelled() }}
+        uses: smartcontractkit/.github/actions/branch-out-upload@branch-out-upload/v1
+        with:
+          junit-file-path: "/tmp/junit-report.xml"
+          trunk-org-slug: chainlink
+          trunk-token: ${{ secrets.TRUNK_API_KEY }}
+          trunk-previous-step-outcome: ${{ steps.run-tests.outcome }}
+          trunk-upload-only: ${{ env.ENABLE_AUTO_QUARANTINE != 'true' }}
+          artifact-name: mixed-env-${{ matrix.tests.test_id }}_test_logs
+
+      - name: Show Docker containers status
+        if: failure() || cancelled()
+        shell: bash
+        run: docker ps -a
+
+      - name: Save Smoke tests Docker logs
+        if: failure() || cancelled()
+        shell: bash
+        working-directory: system-tests/tests/smoke/cre
+        run: |
+          mkdir -p logs
+          for c in $(docker ps -a --format '{{.Names}}'); do
+            docker logs "$c" > "logs/${c}.log" 2>&1
+          done
@@ -112,7 +112,7 @@ jobs:
              ]
           }'
 
-          test_names=$(grep -rh -oP '^func \K(Test|Example)[^(]+' system-tests/tests/smoke/cre/*_test.go | grep -v Test_Upgrade)
+          test_names=$(grep -rh -oP '^func \K(Test|Example)[^(]+' system-tests/tests/smoke/cre/*_test.go | grep -v Test_Upgrade | grep -v '^TestMain$')
 
           tests=$(echo "$test_names" | jq -c -R -s \
             --argjson run_id "${{ github.run_id }}" \

@@ -470,6 +470,24 @@ jobs:
       chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }}
     secrets: inherit
 
+  run-core-cre-mixed-env-tests:
+    name: Run Core CRE Mixed-Env Tests
+    needs: [build-chainlink, run-core-cre-e2e-tests-setup, compile-tests]
+    permissions:
+      actions: read
+      checks: write
+      pull-requests: write
+      id-token: write
+      contents: read
+    if: needs.run-core-cre-e2e-tests-setup.outputs.should-run == 'true'
+    uses: ./.github/workflows/cre-mixed-env-tests.yaml
+    with:
+      ecr: "sdlc"
+      chainlink_image_repository_path: ${{ inputs.ecr_name || 'chainlink-integration-tests' }}
+      chainlink_version: ${{ inputs.evm-ref || inputs.cl_ref || github.sha }}
+      chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }}
+    secrets: inherit
+
   run-ccip-v1-6-e2e-tests-setup:
     name: Run CCIP v1.6 E2E Tests Setup
     runs-on: ubuntu-latest

@@ -15,6 +15,9 @@ cre_v*
 configs/*-cache.toml
 cache/
 
+# rendered mixed-env topology (generated from configs/mixed-env-don.toml.tmpl)
+configs/mixed-env-don.toml
+
 logs/
 
 # CRE CLI settings