[CRE] Mixed-env nightly full-matrix sweep + chain-specific variants #23312
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,105 @@ | ||
| name: CRE Mixed-Env Nightly Sweep | ||
|
|
||
| # Runs the full CRE system-test matrix under the mixed-env topology (2 nodes on | ||
| # today's develop nightly image, 2 on yesterday's) to catch non-determinism / | ||
| # cross-version incompatibilities introduced into develop between nightlies — | ||
| # across the whole suite (EVM + Solana/Aptos/Stellar), not just the per-PR subset. | ||
| # | ||
| # It calls cre-system-tests.yaml directly with mixed_env_all: true, so no image | ||
| # build is needed: both images are the cached nightly develop builds. | ||
|
|
||
| on: | ||
| schedule: | ||
| # After the nightly image build (docker-build.yml runs at 03:00 UTC). | ||
| - cron: "0 5 * * *" | ||
| workflow_dispatch: | ||
| inputs: | ||
| baseline_date: | ||
| description: "Baseline nightly date (YYYYMMDD). Defaults to yesterday." | ||
| required: false | ||
| type: string | ||
| default: "" | ||
|
|
||
| permissions: | ||
| contents: read | ||
| id-token: write | ||
|
|
||
| jobs: | ||
| set-dates: | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| today: ${{ steps.d.outputs.today }} | ||
| baseline: ${{ steps.d.outputs.baseline }} | ||
| steps: | ||
| - id: d | ||
| shell: bash | ||
| run: | | ||
| today="$(date -u +'%Y%m%d')" | ||
| baseline="${{ inputs.baseline_date }}" | ||
| if [[ -z "${baseline}" ]]; then | ||
| baseline="$(date -u -d 'yesterday' +'%Y%m%d')" | ||
| fi | ||
| echo "today=${today}" | tee -a "$GITHUB_OUTPUT" | ||
| echo "baseline=${baseline}" | tee -a "$GITHUB_OUTPUT" | ||
|
|
||
| mixed-env-sweep: | ||
| needs: set-dates | ||
| permissions: | ||
| contents: read | ||
| id-token: write | ||
| uses: ./.github/workflows/cre-mixed-env-tests.yaml | ||
| secrets: inherit | ||
| with: | ||
| ecr: "sdlc" | ||
| chainlink_image_repository_path: "chainlink" | ||
| # "PR" side = today's develop nightly. | ||
| chainlink_image_tag: "nightly-${{ needs.set-dates.outputs.today }}" | ||
| # Baseline side = a previous develop nightly, so the 2-2 split compares two | ||
| # develop builds and surfaces non-determinism introduced since then. | ||
| mixed_env_baseline_image: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink:nightly-${{ needs.set-dates.outputs.baseline }}" | ||
|
Check failure on line 59 in .github/workflows/cre-mixed-env-nightly.yaml
|
||
| full_matrix: true | ||
|
|
||
| # There is no PR to block on a scheduled run, so alert the CRE team on Slack when | ||
| # the sweep fails (a failure usually means non-determinism was introduced into | ||
| # develop between the two nightlies, or a job/infra error). | ||
| notify-on-failure: | ||
| name: Notify Slack on failure | ||
| if: failure() | ||
| needs: [set-dates, mixed-env-sweep] | ||
| runs-on: ubuntu-latest | ||
| environment: | ||
| name: integration | ||
| deployment: false | ||
| steps: | ||
| - name: Send Slack notification (#cre-platform-operations) | ||
| uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0 | ||
|
Check warning on line 75 in .github/workflows/cre-mixed-env-nightly.yaml
|
||
| with: | ||
| errors: "true" | ||
| method: chat.postMessage | ||
| token: ${{ secrets.QA_SLACK_API_KEY }} | ||
| # If delivery fails, invite the bot to the channel, or replace the name | ||
| # with the channel ID (Cxxxxxxxx) which chat.postMessage resolves more reliably. | ||
| payload: | | ||
| { | ||
| "channel": "#cre-platform-operations", | ||
| "text": "CRE Mixed-Env Nightly Sweep failed — possible non-determinism introduced into develop.", | ||
| "blocks": [ | ||
| { | ||
| "type": "section", | ||
| "text": { | ||
| "type": "mrkdwn", | ||
| "text": "*:rotating_light: CRE Mixed-Env Nightly Sweep failed :rotating_light:*" | ||
| } | ||
| }, | ||
| { | ||
| "type": "section", | ||
| "text": { | ||
| "type": "mrkdwn", | ||
| "text": "Today's develop nightly `nightly-${{ needs.set-dates.outputs.today }}` vs baseline `nightly-${{ needs.set-dates.outputs.baseline }}`. A failure usually means *non-determinism was introduced into develop* between these nightlies (or a job/infra error). Run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.run_id }}>. Please investigate." | ||
| } | ||
| }, | ||
| { | ||
| "type": "divider" | ||
| } | ||
| ] | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,95 @@ | ||
| # Mixed-env Aptos topology (template) — derived from workflow-gateway-don-aptos.toml. | ||
| # | ||
| # Same as workflow-gateway-don-aptos, except the workflow DON runs 2 nodes on the PR | ||
| # image and 2 on the develop image, to surface non-determinism between the two code | ||
| # versions (see docs/mixed-env.md). Render with configs/render-mixed-env.sh: | ||
| # | ||
| # CRE_PR_IMAGE=<pr> CRE_BASELINE_IMAGE=<develop> \ | ||
| # ./configs/render-mixed-env.sh configs/mixed-env-aptos-don.toml | ||
| # | ||
| # Keep in sync with workflow-gateway-don-aptos.toml if that base topology changes. | ||
| # Do NOT set CTF_CHAINLINK_IMAGE with this topology (it would force one image on all nodes). | ||
|
|
||
| [chip_router] | ||
| image = "local-cre-chip-router:v1.0.1" | ||
|
|
||
| [[blockchains]] | ||
| type = "anvil" | ||
| chain_id = "1337" | ||
| container_name = "anvil-1337" | ||
| docker_cmd_params = ["-b", "0.5", "--mixed-mining"] | ||
|
|
||
| [[blockchains]] | ||
| type = "aptos" | ||
| chain_id = "4" | ||
|
|
||
| [jd] | ||
| csa_encryption_key = "d1093c0060d50a3c89c189b2e485da5a3ce57f3dcb38ab7e2c0d5f0bb2314a44" | ||
| image = "job-distributor:0.28.0" | ||
|
|
||
| [infra] | ||
| type = "docker" | ||
|
|
||
| [[nodesets]] | ||
| nodes = 4 | ||
| name = "workflow" | ||
| don_family = "test-don-family" | ||
| don_types = ["workflow"] | ||
| override_mode = "each" | ||
| http_port_range_start = 10100 | ||
|
|
||
| supported_evm_chains = [1337] | ||
| env_vars = { CL_CRE_SETTINGS_DEFAULT = '{"PerWorkflow":{"CapabilityCallTimeout":"5m0s","ChainAllowed":{"Default":"false","Values":{"1337":"true","4457093679053095497":"true"}},"ExecutionTimestampsEnabled":"true","FeatureAptosWriteReportBlockTimestampActivePeriod":"[2020-01-01 00:00:00 +0000 UTC,2030-01-01 00:00:00 +0000 UTC]"}}' } | ||
| capabilities = ["cron", "consensus", "aptos-4", "don-time"] | ||
| registry_based_launch_allowlist = ["cron-trigger@1.0.0"] | ||
|
|
||
| [nodesets.db] | ||
| image = "postgres:12.0" | ||
| port = 13000 | ||
|
|
||
| [[nodesets.node_specs]] # node 0 - PR | ||
| roles = ["plugin"] | ||
| [nodesets.node_specs.node] | ||
| image = "${CRE_PR_IMAGE}" | ||
| pull_image = true | ||
| user_config_overrides = "" | ||
| [[nodesets.node_specs]] # node 1 - PR | ||
| roles = ["plugin"] | ||
| [nodesets.node_specs.node] | ||
| image = "${CRE_PR_IMAGE}" | ||
| pull_image = true | ||
| user_config_overrides = "" | ||
| [[nodesets.node_specs]] # node 2 - develop | ||
| roles = ["plugin"] | ||
| [nodesets.node_specs.node] | ||
| image = "${CRE_BASELINE_IMAGE}" | ||
| pull_image = true | ||
| user_config_overrides = "" | ||
| [[nodesets.node_specs]] # node 3 - develop | ||
| roles = ["plugin"] | ||
| [nodesets.node_specs.node] | ||
| image = "${CRE_BASELINE_IMAGE}" | ||
| pull_image = true | ||
| user_config_overrides = "" | ||
|
|
||
| [[nodesets]] | ||
| nodes = 1 | ||
| name = "bootstrap-gateway" | ||
| don_family = "test-don-family" | ||
| don_types = ["bootstrap", "gateway"] | ||
| override_mode = "each" | ||
| http_port_range_start = 10300 | ||
|
|
||
| supported_evm_chains = [1337] | ||
|
|
||
| [nodesets.db] | ||
| image = "postgres:12.0" | ||
| port = 13200 | ||
|
|
||
| [[nodesets.node_specs]] | ||
| roles = ["bootstrap", "gateway"] | ||
| [nodesets.node_specs.node] | ||
| image = "${CRE_BASELINE_IMAGE}" | ||
| pull_image = true | ||
| custom_ports = ["5002:5002", "15002:15002"] | ||
| user_config_overrides = "" |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
what do you think about triggering it with
workflow_callfrom this workflow? that's what we do for all other nightly tests and it is triggered byDocker Build, which build the nightly image