diff --git a/.github/.agents/skills/optimize-workflow/SKILL.md b/.github/.agents/skills/optimize-workflow/SKILL.md
new file mode 100644
index 00000000000..fd902e0ee2a
--- /dev/null
+++ b/.github/.agents/skills/optimize-workflow/SKILL.md
@@ -0,0 +1,77 @@
+---
+name: optimize-workflow
+description: Optimize CI runners and workflow structure for speed, simplicity, stability, and cost.
+disable-model-invocation: true
+---
+
+
+1. Confirm missing inputs: target workflow, scope, priorities (default: stability > speed > cost).
+2. Setup baseline:
+ - Analyze target workflow (runners, caching, bottlenecks, critical path).
+ - Setup test workflow (add `workflow_dispatch`, mock inputs if needed).
+ - Init/resume trial log: `.github/.agents/skills/optimize-workflow/trials/[workflow]/summary.md`.
+ - Record baseline run metrics.
+
+
+
+To prevent index/branch lock conflicts:
+- Do NOT checkout local trial branches.
+- Push trial commits directly from working branch to remote refs:
+ `git push -f origin HEAD:refs/heads/trial/[trial-name]`
+- Delete remote trial refs on cleanup: `git push origin --delete trial/[trial-name]`
+
+
+
+- Git: Direct ref pushes only. No local branch switching during trials.
+- Runners: Prefer `ubuntu-latest`. Use `runs-on` for larger runners (verify API first).
+- Scope: One change per trial. Keep cache status identical across comparison runs.
+- Validation: Lint YAML (`actionlint` or dry-run) before pushing.
+- Tools: Use `gh` CLI and workflow scripts for monitoring/comparisons.
+
+
+
+- Monitor: `python3 .github/.agents/skills/optimize-workflow/scripts/workflow_monitor.py [run_id] [trial-name]`
+ - Outputs `report.json`, `report.md`, and runner logs to `.github/.agents/skills/optimize-workflow/trials/[workflow]/[trial-name]/`
+ - Quick analysis: Inspect `## Longest Jobs (Bottlenecks)` in `report.md` or query `jq '.slowest_jobs[] | select(.is_outlier)' report.json` / `jq '([.jobs[].duration_seconds] | add / length) as $avg | .jobs[] | select(.duration_seconds > $avg * 1.5)' report.json`.
+- Compare: `python3 .github/.agents/skills/optimize-workflow/scripts/workflow_compare.py [trial-1] [trial-2]`
+ - Outputs `.github/.agents/skills/optimize-workflow/trials/[workflow]/[trial-1]-[trial-2]-comparison.md`
+
+
+
+- [runs-on docs](https://runs-on.com/docs/) | [spot pricing](https://runs-on.com/docs/costs/spot-pricing/)
+- [available runners](https://go.runs-on.com/api)
+- [GitHub Actions docs](https://docs.github.com/en/actions)
+
+
+
+1. Define single-variable trial and prompt for user approval.
+2. Push commit to remote ref: `git push -f origin HEAD:refs/heads/trial/[trial-name]`.
+3. Trigger run: `gh workflow run --ref trial/[trial-name]`.
+4. Monitor run with `workflow_monitor.py`.
+5. Compare with `workflow_compare.py` against baseline/previous trial.
+6. Update trial log in `summary.md`. Include before/after diagram if structure changed.
+7. Present condensed result to user; prompt for next trial or completion.
+
+
+| Runner | Structure | Experiment | Expectation | Branch | Run ID | Commit | Stability | Runtime | Cost | Notes |
+|---|---|---|---|---|---|---|---|---|---|---|
+| `config` | `cached/parallel` | Tested change | Expected impact | `trial/...` | `id` | `sha` | Pass/Fail | mm:ss | $ | Notes |
+
+
+
+
+1. Recommend final configuration.
+2. Output PR summary table and before/after Mermaid diagrams:
+
+```md
+### [Workflow/Job Name] Runner & Structure Changes
+
+| Approach | Runner | Structure | Stability | Runtime | Runtime Delta (Abs/%) | Cost | Cost Delta (Abs/%) |
+|---|---|---|---|---|---|---|---|
+| [Old](link) | [runner] | [structure] | Pass/Fail | mm:ss | +0:00 (+0%) | $ | +$ (+0%) |
+| [New](link) | [runner] | [structure] | Pass/Fail | mm:ss | +0:00 (+0%) | $ | +$ (+0%) |
+```
+
+3. Clean up debug code. Make final edits on working branch. Ask user to commit.
+4. Delete remote trial branches (`git push origin --delete trial/[trial-name]`) and temporary files.
+
\ No newline at end of file
diff --git a/.github/.agents/skills/optimize-workflow/scripts/test_workflow_compare.py b/.github/.agents/skills/optimize-workflow/scripts/test_workflow_compare.py
new file mode 100644
index 00000000000..49de31c0f27
--- /dev/null
+++ b/.github/.agents/skills/optimize-workflow/scripts/test_workflow_compare.py
@@ -0,0 +1,272 @@
+import unittest
+import json
+import os
+import tempfile
+from unittest.mock import patch, MagicMock
+import workflow_compare
+
+class TestWorkflowCompare(unittest.TestCase):
+
+ def test_compare_durations_decrease(self):
+ diff, pct_str = workflow_compare.compare_durations("0:03:20", "0:02:40")
+ self.assertEqual(diff, "-0:00:40")
+ self.assertEqual(pct_str, "-20.0%")
+
+ def test_compare_durations_increase(self):
+ diff, pct_str = workflow_compare.compare_durations("0:02:00", "0:02:30")
+ self.assertEqual(diff, "+0:00:30")
+ self.assertEqual(pct_str, "+25.0%")
+
+ def test_compare_durations_equal(self):
+ diff, pct_str = workflow_compare.compare_durations("0:02:00", "0:02:00")
+ self.assertEqual(diff, "0:00:00")
+ self.assertEqual(pct_str, "0.0%")
+
+ def test_compare_costs_decrease(self):
+ diff, pct_str = workflow_compare.compare_costs("$0.1000", "$0.0800")
+ self.assertEqual(diff, "-$0.0200")
+ self.assertEqual(pct_str, "-20.0%")
+
+ def test_compare_costs_increase(self):
+ diff, pct_str = workflow_compare.compare_costs("$0.1000", "$0.1150")
+ self.assertEqual(diff, "+$0.0150")
+ self.assertEqual(pct_str, "+15.0%")
+
+ def test_compare_costs_equal(self):
+ diff, pct_str = workflow_compare.compare_costs("$0.1000", "$0.1000")
+ self.assertEqual(diff, "$0.0000")
+ self.assertEqual(pct_str, "0.0%")
+
+ def test_compare_metrics(self):
+ m1 = {"min": "10.0", "max": "50.0", "avg": "30.0 MB"}
+ m2 = {"min": "12.0", "max": "45.0", "avg": "28.5 MB"}
+
+ diff = workflow_compare.compare_metrics(m1, m2)
+ # We expect it to format average delta: e.g. "avg: 30.0 -> 28.5 MB (-1.5 MB)" or similar
+ self.assertIn("avg: 30.0 MB -> 28.5 MB", diff)
+ self.assertIn("-1.5 MB", diff)
+
+ def test_generate_comparison(self):
+ data1 = {
+ "run": {"id": 123, "runtime": "0:10:00", "status": "completed", "conclusion": "success", "total_cost": "$0.1000"},
+ "logs_dir": "/tmp/logs1",
+ "jobs": [
+ {
+ "name": "build",
+ "status": "completed",
+ "conclusion": "success",
+ "runner": {"labels": ["runs-on"], "name": "runner-1"},
+ "duration": "0:03:20",
+ "metrics": {
+ "Instance Type": "c6in.4xlarge",
+ "Cost": "$0.1000",
+ "system.cpu.load_average.1m": {"min": "0.1", "max": "10.0", "avg": "5.0"}
+ }
+ }
+ ]
+ }
+ data2 = {
+ "run": {"id": 124, "runtime": "0:09:00", "status": "completed", "conclusion": "success", "total_cost": "$0.0800"},
+ "logs_dir": "/tmp/logs2",
+ "jobs": [
+ {
+ "name": "build",
+ "status": "completed",
+ "conclusion": "success",
+ "runner": {"labels": ["runs-on"], "name": "runner-2"},
+ "duration": "0:02:40",
+ "metrics": {
+ "Instance Type": "c7i-flex.8xlarge",
+ "Cost": "$0.0800",
+ "system.cpu.load_average.1m": {"min": "0.2", "max": "12.0", "avg": "6.0"}
+ }
+ }
+ ]
+ }
+
+ report = workflow_compare.generate_comparison(data1, data2)
+ self.assertIn("# Workflow Trial Comparison", report)
+ self.assertIn("c6in.4xlarge", report)
+ self.assertIn("c7i-flex.8xlarge", report)
+ self.assertIn("- **Runtime Delta**: `-0:01:00` (-10.0%)", report)
+ self.assertIn("- **Cost Delta**: `-$0.0200` (-20.0%)", report)
+ self.assertIn("Cost: `$0.1000`", report)
+ self.assertIn("Cost: `$0.0800`", report)
+
+ def test_generate_comparison_overall_cost_fallback_and_na(self):
+ # Fallback when total_cost is not in run dict but in job metrics
+ data1 = {
+ "run": {"id": 1, "runtime": "0:10:00", "status": "completed", "conclusion": "success"},
+ "jobs": [{"name": "j1", "duration": "0:05:00", "metrics": {"Cost": "$0.0500"}}]
+ }
+ data2 = {
+ "run": {"id": 2, "runtime": "0:10:00", "status": "completed", "conclusion": "success"},
+ "jobs": [{"name": "j1", "duration": "0:05:00", "metrics": {"Cost": "$0.0700"}}]
+ }
+ report = workflow_compare.generate_comparison(data1, data2)
+ self.assertIn("- **Cost Delta**: `+$0.0200` (+40.0%)", report)
+
+ # N/A when no cost data
+ data_no_cost = {
+ "run": {"id": 3, "runtime": "0:10:00", "status": "completed", "conclusion": "success"},
+ "jobs": [{"name": "j1", "duration": "0:05:00"}]
+ }
+ report_na = workflow_compare.generate_comparison(data1, data_no_cost)
+ self.assertIn("- **Cost Delta**: `N/A` (N/A)", report_na)
+
+ def test_generate_comparison_markdown_links(self):
+ data1 = {
+ "run": {
+ "id": 123,
+ "runtime": "0:10:00",
+ "status": "completed",
+ "conclusion": "success",
+ "html_url": "https://github.com/owner/repo/actions/runs/123"
+ },
+ "jobs": [
+ {
+ "name": "build",
+ "status": "completed",
+ "conclusion": "success",
+ "html_url": "https://github.com/owner/repo/actions/runs/123/job/10",
+ "duration": "0:05:00"
+ }
+ ]
+ }
+ data2 = {
+ "run": {
+ "id": 124,
+ "runtime": "0:09:00",
+ "status": "completed",
+ "conclusion": "success",
+ "html_url": "https://github.com/owner/repo/actions/runs/124"
+ },
+ "jobs": [
+ {
+ "name": "build",
+ "status": "completed",
+ "conclusion": "success",
+ "html_url": "https://github.com/owner/repo/actions/runs/124/job/20",
+ "duration": "0:04:00"
+ }
+ ]
+ }
+ report = workflow_compare.generate_comparison(data1, data2)
+ self.assertIn("- **Base Run ID (Trial 1)**: [123](https://github.com/owner/repo/actions/runs/123)", report)
+ self.assertIn("Status: [success](https://github.com/owner/repo/actions/runs/123)", report)
+ self.assertIn("- **New Run ID (Trial 2)**: [124](https://github.com/owner/repo/actions/runs/124)", report)
+ self.assertIn("Status: [success](https://github.com/owner/repo/actions/runs/124)", report)
+ self.assertIn("- **Status**: [success](https://github.com/owner/repo/actions/runs/123/job/10) -> [success](https://github.com/owner/repo/actions/runs/124/job/20)", report)
+
+ def test_normalize_name(self):
+ self.assertEqual(workflow_compare.normalize_name("build"), "build")
+ self.assertEqual(workflow_compare.normalize_name("build-job"), "buildjob")
+
+ name = "Run CCIP v1.6 E2E Tests For Workflow Dispatch / smoke/ccip/ccip_reorg_test.go:GreaterThanFinalityTests"
+ expected = "runccipv16e2etestsforworkflowdispatchsmokeccipccipreorgtestgogreaterthanfinalitytests"
+ self.assertEqual(workflow_compare.normalize_name(name), expected)
+
+ def test_matches_job_name(self):
+ api_name = "Run CCIP v1.6 E2E Tests For Workflow Dispatch / smoke/ccip/ccip_reorg_test.go:GreaterThanFinalityTests"
+ log_job_name = "Run CCIP v1.6 E2E Tests For Workflow Dispatch _ smoke_ccip_ccip_reorg_test.goGreaterThanFinalityTests"
+ self.assertTrue(workflow_compare.matches_job_name(api_name, log_job_name))
+
+ # Test suffix or path basename matching
+ api_name_path = "Run CCIP v1.6 E2E Tests / smoke/ccip/ccip_reorg_test.go:GreaterThanFinalityTests"
+ log_job_name_suffix = "smoke_ccip_ccip_reorg_test.goGreaterThanFinalityTests"
+ self.assertTrue(workflow_compare.matches_job_name(api_name_path, log_job_name_suffix))
+
+ def _make_trial_dir(self, trials_dir, workflow, trial, report_data):
+ workflow_dir = os.path.join(trials_dir, workflow)
+ trial_dir = os.path.join(workflow_dir, trial)
+ os.makedirs(trial_dir, exist_ok=True)
+ report_path = os.path.join(trial_dir, "report.json")
+ with open(report_path, 'w', encoding='utf-8') as f:
+ json.dump(report_data, f)
+ return report_path
+
+ def test_resolve_trial_by_path(self):
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ trials_dir = tmp_dir
+ path = self._make_trial_dir(trials_dir, "wf", "t1", {"run": {"id": 1}})
+ workflow, trial, resolved = workflow_compare.resolve_trial(trials_dir, "wf/t1")
+ self.assertEqual(workflow, "wf")
+ self.assertEqual(trial, "t1")
+ self.assertEqual(resolved, path)
+
+ def test_resolve_trial_by_search(self):
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ trials_dir = tmp_dir
+ path = self._make_trial_dir(trials_dir, "wf", "t1", {"run": {"id": 1}})
+ workflow, trial, resolved = workflow_compare.resolve_trial(trials_dir, "t1")
+ self.assertEqual(workflow, "wf")
+ self.assertEqual(trial, "t1")
+ self.assertEqual(resolved, path)
+
+ def test_resolve_trial_legacy(self):
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ trials_dir = tmp_dir
+ workflow_dir = os.path.join(trials_dir, "wf")
+ os.makedirs(workflow_dir, exist_ok=True)
+ legacy_path = os.path.join(workflow_dir, "t1.json")
+ with open(legacy_path, 'w', encoding='utf-8') as f:
+ json.dump({"run": {"id": 1}}, f)
+ workflow, trial, resolved = workflow_compare.resolve_trial(trials_dir, "t1")
+ self.assertEqual(workflow, "wf")
+ self.assertEqual(trial, "t1")
+ self.assertEqual(resolved, legacy_path)
+
+ def test_resolve_trial_not_found(self):
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ trials_dir = tmp_dir
+ os.makedirs(trials_dir, exist_ok=True)
+ with self.assertRaises(FileNotFoundError):
+ workflow_compare.resolve_trial(trials_dir, "missing")
+
+ @patch('workflow_compare.parse_args')
+ @patch('workflow_compare.get_trials_dir')
+ def test_main_creates_comparison(self, mock_get_trials_dir, mock_parse_args):
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ trials_dir = tmp_dir
+ mock_get_trials_dir.return_value = trials_dir
+
+ data1 = {"run": {"id": 100, "runtime": "0:10:00", "status": "completed", "conclusion": "success"}, "jobs": []}
+ data2 = {"run": {"id": 101, "runtime": "0:09:00", "status": "completed", "conclusion": "success"}, "jobs": []}
+ self._make_trial_dir(trials_dir, "wf", "before", data1)
+ self._make_trial_dir(trials_dir, "wf", "after", data2)
+
+ args = MagicMock()
+ args.trial_before = "before"
+ args.trial_after = "after"
+ mock_parse_args.return_value = args
+
+ workflow_compare.main()
+
+ out_file = os.path.join(trials_dir, "wf", "before-after-comparison.md")
+ self.assertTrue(os.path.exists(out_file))
+ with open(out_file, 'r', encoding='utf-8') as f:
+ content = f.read()
+ self.assertIn("# Workflow Trial Comparison", content)
+ self.assertIn("100", content)
+ self.assertIn("101", content)
+
+ @patch('workflow_compare.parse_args')
+ @patch('workflow_compare.get_trials_dir')
+ def test_main_different_workflows_error(self, mock_get_trials_dir, mock_parse_args):
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ trials_dir = tmp_dir
+ mock_get_trials_dir.return_value = trials_dir
+ self._make_trial_dir(trials_dir, "wf1", "t1", {"run": {"id": 1}, "jobs": []})
+ self._make_trial_dir(trials_dir, "wf2", "t2", {"run": {"id": 2}, "jobs": []})
+
+ args = MagicMock()
+ args.trial_before = "t1"
+ args.trial_after = "t2"
+ mock_parse_args.return_value = args
+
+ with self.assertRaises(SystemExit) as cm:
+ workflow_compare.main()
+ self.assertEqual(cm.exception.code, 1)
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/.github/.agents/skills/right-size-runners/scripts/test_workflow_monitor.py b/.github/.agents/skills/optimize-workflow/scripts/test_workflow_monitor.py
similarity index 61%
rename from .github/.agents/skills/right-size-runners/scripts/test_workflow_monitor.py
rename to .github/.agents/skills/optimize-workflow/scripts/test_workflow_monitor.py
index bd27f5e441a..3a2d6c55c77 100644
--- a/.github/.agents/skills/right-size-runners/scripts/test_workflow_monitor.py
+++ b/.github/.agents/skills/optimize-workflow/scripts/test_workflow_monitor.py
@@ -6,6 +6,7 @@
import zipfile
import json
import urllib.error
+import urllib.request
import sys
# Import the stub module
@@ -205,7 +206,70 @@ def test_generate_report_json(self):
self.assertEqual(report["logs_dir"], "/tmp/logs-dir")
self.assertEqual(report["jobs"][0]["metrics"]["Instance Type"], "c6in.4xlarge")
+ def test_parse_duration_seconds(self):
+ start = "2026-07-16T17:16:00Z"
+ end = "2026-07-16T17:18:30Z"
+ self.assertEqual(workflow_monitor.parse_duration_seconds(start, end), 150)
+ self.assertEqual(workflow_monitor.parse_duration_seconds(None, end), 0)
+
+ def test_generate_report_json_includes_seconds_and_slowest_jobs(self):
+ run_data = {"id": 12345, "status": "completed", "conclusion": "success", "run_started_at": "2026-07-16T17:00:00Z", "updated_at": "2026-07-16T17:10:00Z"}
+ jobs = [
+ {"name": "fast_job", "status": "completed", "conclusion": "success", "started_at": "2026-07-16T17:00:00Z", "completed_at": "2026-07-16T17:01:00Z"},
+ {"name": "slow_job", "status": "completed", "conclusion": "success", "started_at": "2026-07-16T17:00:00Z", "completed_at": "2026-07-16T17:05:00Z"}
+ ]
+ report_str = workflow_monitor.generate_report(run_data, jobs, {}, "/tmp/logs", "json")
+ report = json.loads(report_str)
+
+ self.assertEqual(report["run"]["runtime_seconds"], 600)
+ self.assertEqual(report["run"]["avg_job_duration_seconds"], 180)
+ self.assertEqual(report["jobs"][0]["duration_seconds"], 60)
+ self.assertEqual(report["jobs"][1]["duration_seconds"], 300)
+
+ # Check slowest_jobs array is pre-sorted descending
+ self.assertEqual(len(report["slowest_jobs"]), 2)
+ self.assertEqual(report["slowest_jobs"][0]["name"], "slow_job")
+ self.assertEqual(report["slowest_jobs"][0]["duration_seconds"], 300)
+ self.assertTrue(report["slowest_jobs"][0]["is_outlier"])
+
+
+ def test_generate_report_markdown_includes_longest_jobs_summary(self):
+ run_data = {"id": 12345, "status": "completed", "conclusion": "success", "run_started_at": "2026-07-16T17:00:00Z", "updated_at": "2026-07-16T17:10:00Z"}
+ jobs = [
+ {"name": "slow_job", "status": "completed", "conclusion": "success", "started_at": "2026-07-16T17:00:00Z", "completed_at": "2026-07-16T17:05:00Z"}
+ ]
+ report = workflow_monitor.generate_report(run_data, jobs, {}, "/tmp/logs", "markdown")
+ self.assertIn("## Longest Jobs (Bottlenecks)", report)
+ self.assertIn("slow_job", report)
+
+ def test_generate_report_markdown_links(self):
+ run_data = {
+ "id": 12345,
+ "status": "completed",
+ "conclusion": "success",
+ "html_url": "https://github.com/owner/repo/actions/runs/12345",
+ "run_started_at": "2026-07-16T17:00:00Z",
+ "updated_at": "2026-07-16T17:10:00Z"
+ }
+ jobs = [
+ {
+ "name": "build",
+ "status": "completed",
+ "conclusion": "success",
+ "html_url": "https://github.com/owner/repo/actions/runs/12345/job/67890",
+ "started_at": "2026-07-16T17:00:00Z",
+ "completed_at": "2026-07-16T17:05:00Z"
+ }
+ ]
+ report = workflow_monitor.generate_report(run_data, jobs, {}, "/tmp/logs", "markdown")
+ self.assertIn("- **ID**: [12345](https://github.com/owner/repo/actions/runs/12345)", report)
+ self.assertIn("- **Status**: [completed](https://github.com/owner/repo/actions/runs/12345)", report)
+ self.assertIn("| [build](https://github.com/owner/repo/actions/runs/12345/job/67890) |", report)
+ self.assertIn("### Job: [build](https://github.com/owner/repo/actions/runs/12345/job/67890)", report)
+ self.assertIn("- **Status**: [completed](https://github.com/owner/repo/actions/runs/12345/job/67890)", report)
+
@patch('workflow_monitor.parse_args')
+ @patch('workflow_monitor.get_trials_dir')
@patch('workflow_monitor.wait_for_run')
@patch('workflow_monitor.wait_for_completion')
@patch('workflow_monitor.download_logs')
@@ -214,54 +278,71 @@ def test_generate_report_json(self):
@patch('workflow_monitor.generate_report')
@patch('workflow_monitor.log_stderr')
@patch('builtins.print')
- def test_main_with_outfile(self, mock_print, mock_log_stderr, mock_gen_report, mock_fetch_jobs, mock_parse_logs, mock_download, mock_wait_complete, mock_wait_run, mock_parse_args):
- args = MagicMock()
- args.run_id = 12345
- args.repo = "owner/repo"
- args.token = "fake_token"
- args.poll_interval = 10
- args.format = "json"
-
+ def test_main_writes_reports(self, mock_print, mock_log_stderr, mock_gen_report, mock_fetch_jobs, mock_parse_logs, mock_download, mock_wait_complete, mock_wait_run, mock_get_trials_dir, mock_parse_args):
with tempfile.TemporaryDirectory() as tmp_dir:
- out_file = os.path.join(tmp_dir, "report.json")
- args.out_file = out_file
+ mock_get_trials_dir.return_value = tmp_dir
+ args = MagicMock()
+ args.run_id = 12345
+ args.trial_name = "baseline-12345"
+ args.repo = "owner/repo"
+ args.token = "fake_token"
+ args.poll_interval = 10
mock_parse_args.return_value = args
+
+ mock_wait_run.return_value = {"id": 12345, "path": ".github/workflows/integration-tests.yml"}
+ mock_wait_complete.return_value = {"id": 12345}
mock_gen_report.return_value = '{"fake": "report"}'
-
+ mock_download.return_value = True
+ mock_parse_logs.return_value = {}
+ mock_fetch_jobs.return_value = []
+
workflow_monitor.main()
-
- mock_print.assert_not_called()
- self.assertTrue(os.path.exists(out_file))
- with open(out_file, 'r') as f:
+
+ expected_output_dir = os.path.join(tmp_dir, "integration-tests.yml", "baseline-12345")
+ expected_logs_dir = os.path.join(expected_output_dir, "logs")
+ expected_json = os.path.join(expected_output_dir, "report.json")
+ expected_md = os.path.join(expected_output_dir, "report.md")
+
+ self.assertTrue(os.path.isdir(expected_output_dir))
+ self.assertTrue(os.path.isdir(expected_logs_dir))
+ mock_download.assert_called_once_with("owner/repo", 12345, "fake_token", expected_logs_dir)
+ mock_gen_report.assert_any_call({"id": 12345}, [], {}, expected_logs_dir, "json")
+ mock_gen_report.assert_any_call({"id": 12345}, [], {}, expected_logs_dir, "markdown")
+ self.assertTrue(os.path.exists(expected_json))
+ self.assertTrue(os.path.exists(expected_md))
+ with open(expected_json, 'r') as f:
+ self.assertEqual(f.read(), '{"fake": "report"}')
+ with open(expected_md, 'r') as f:
self.assertEqual(f.read(), '{"fake": "report"}')
-
- mock_log_stderr.assert_any_call(f"Report successfully saved to: {out_file}")
- mock_log_stderr.assert_any_call(f"Try exploring with: jq . {out_file}")
- @patch('workflow_monitor.parse_args')
- @patch('workflow_monitor.wait_for_run')
- @patch('workflow_monitor.wait_for_completion')
- @patch('workflow_monitor.download_logs')
- @patch('workflow_monitor.parse_job_logs')
- @patch('workflow_monitor.fetch_jobs')
- @patch('workflow_monitor.generate_report')
- @patch('workflow_monitor.log_stderr')
- @patch('builtins.print')
- def test_main_without_outfile(self, mock_print, mock_log_stderr, mock_gen_report, mock_fetch_jobs, mock_parse_logs, mock_download, mock_wait_complete, mock_wait_run, mock_parse_args):
- args = MagicMock()
- args.run_id = 12345
- args.repo = "owner/repo"
- args.token = "fake_token"
- args.poll_interval = 10
- args.format = "json"
- args.out_file = None
- mock_parse_args.return_value = args
-
- mock_gen_report.return_value = '{"fake": "report"}'
+ def test_sanitize_dir_name(self):
+ self.assertEqual(workflow_monitor.sanitize_dir_name("foo/bar"), "foo_bar")
+ self.assertEqual(workflow_monitor.sanitize_dir_name("a\\b"), "a_b")
+ self.assertEqual(workflow_monitor.sanitize_dir_name(".."), "_")
+ self.assertEqual(workflow_monitor.sanitize_dir_name(" my trial "), "my trial")
+
+ def test_derive_workflow_name(self):
+ self.assertEqual(workflow_monitor.derive_workflow_name({"path": ".github/workflows/integration-tests.yml"}), "integration-tests.yml")
+ self.assertEqual(workflow_monitor.derive_workflow_name({"name": "My Workflow"}), "My Workflow")
+ self.assertEqual(workflow_monitor.derive_workflow_name({}), "unknown")
+
+ def test_normalize_name(self):
+ self.assertEqual(workflow_monitor.normalize_name("build"), "build")
+ self.assertEqual(workflow_monitor.normalize_name("build-job"), "buildjob")
- workflow_monitor.main()
+ name = "Run CCIP v1.6 E2E Tests For Workflow Dispatch / smoke/ccip/ccip_reorg_test.go:GreaterThanFinalityTests"
+ expected = "runccipv16e2etestsforworkflowdispatchsmokeccipccipreorgtestgogreaterthanfinalitytests"
+ self.assertEqual(workflow_monitor.normalize_name(name), expected)
+
+ def test_matches_job_name(self):
+ api_name = "Run CCIP v1.6 E2E Tests For Workflow Dispatch / smoke/ccip/ccip_reorg_test.go:GreaterThanFinalityTests"
+ log_job_name = "Run CCIP v1.6 E2E Tests For Workflow Dispatch _ smoke_ccip_ccip_reorg_test.goGreaterThanFinalityTests"
+ self.assertTrue(workflow_monitor.matches_job_name(api_name, log_job_name))
- mock_print.assert_called_once_with('{"fake": "report"}')
+ # Test suffix or path basename matching
+ api_name_path = "Run CCIP v1.6 E2E Tests / smoke/ccip/ccip_reorg_test.go:GreaterThanFinalityTests"
+ log_job_name_suffix = "smoke_ccip_ccip_reorg_test.goGreaterThanFinalityTests"
+ self.assertTrue(workflow_monitor.matches_job_name(api_name_path, log_job_name_suffix))
if __name__ == '__main__':
unittest.main()
diff --git a/.github/.agents/skills/right-size-runners/scripts/workflow_compare.py b/.github/.agents/skills/optimize-workflow/scripts/workflow_compare.py
similarity index 59%
rename from .github/.agents/skills/right-size-runners/scripts/workflow_compare.py
rename to .github/.agents/skills/optimize-workflow/scripts/workflow_compare.py
index e9cc4d42cc1..2b62b51a212 100644
--- a/.github/.agents/skills/right-size-runners/scripts/workflow_compare.py
+++ b/.github/.agents/skills/optimize-workflow/scripts/workflow_compare.py
@@ -1,16 +1,77 @@
#!/usr/bin/env python3
import argparse
import json
+import os
import re
import sys
def parse_args():
parser = argparse.ArgumentParser(description="Compare GitHub Actions Workflow Run Trials")
- parser.add_argument("file1", help="Path to base JSON report file (Trial 1)")
- parser.add_argument("file2", help="Path to new JSON report file (Trial 2)")
- parser.add_argument("--out-file", help="Path to write the output markdown report.")
+ parser.add_argument("trial_before", help="Name of the base trial (or workflow/trial path)")
+ parser.add_argument("trial_after", help="Name of the new trial (or workflow/trial path)")
return parser.parse_args()
+
+def get_trials_dir():
+ # Trial directories live directly inside the optimize-workflow skill directory.
+ return os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'trials'))
+
+
+def sanitize_dir_name(name):
+ if not name:
+ return "unknown"
+ name = name.replace('/', '_').replace('\\', '_')
+ while '..' in name:
+ name = name.replace('..', '_')
+ return name.strip()
+
+
+def split_workflow_trial(trial_input):
+ parts = trial_input.replace('\\', '/').split('/')
+ if len(parts) == 2:
+ return sanitize_dir_name(parts[0]), sanitize_dir_name(parts[1])
+ return None, None
+
+
+def find_report(trials_dir, workflow_name, trial_name):
+ trial_dir = os.path.join(trials_dir, workflow_name, trial_name)
+ report_path = os.path.join(trial_dir, "report.json")
+ if os.path.isfile(report_path):
+ return report_path
+ legacy_path = os.path.join(trials_dir, workflow_name, f"{trial_name}.json")
+ if os.path.isfile(legacy_path):
+ return legacy_path
+ return None
+
+
+def search_trial(trials_dir, trial_name):
+ safe_name = sanitize_dir_name(trial_name)
+ candidates = []
+ for entry in os.listdir(trials_dir):
+ workflow_dir = os.path.join(trials_dir, entry)
+ if not os.path.isdir(workflow_dir):
+ continue
+ report_path = find_report(trials_dir, entry, safe_name)
+ if report_path:
+ candidates.append((entry, report_path))
+ if len(candidates) == 1:
+ return candidates[0]
+ if len(candidates) > 1:
+ workflows = [c[0] for c in candidates]
+ raise ValueError(f"Trial {trial_name!r} found in multiple workflows: {workflows}")
+ raise FileNotFoundError(f"Could not find report for trial {trial_name!r} under {trials_dir}")
+
+
+def resolve_trial(trials_dir, trial_input):
+ workflow_name, trial_name = split_workflow_trial(trial_input)
+ if workflow_name is not None:
+ report_path = find_report(trials_dir, workflow_name, trial_name)
+ if not report_path:
+ raise FileNotFoundError(f"Could not find report for {trial_input!r}")
+ return workflow_name, trial_name, report_path
+ workflow_name, report_path = search_trial(trials_dir, trial_input)
+ return workflow_name, sanitize_dir_name(trial_input), report_path
+
def load_report(filepath):
with open(filepath, 'r', encoding='utf-8') as f:
return json.load(f)
@@ -135,28 +196,90 @@ def compare_metrics(metric1, metric2):
return f"avg: {avg1} -> {avg2}"
def normalize_name(name):
+ if not name:
+ return ""
name = name.lower()
- name = name.replace('/', '_')
- name = re.sub(r'\s+', '', name)
- name = name.replace('...', '')
- return name.rstrip('.')
+ return re.sub(r'[^a-z0-9]', '', name)
def matches_job_name(name1, name2):
a = normalize_name(name1)
b = normalize_name(name2)
- return a.startswith(b) or b.startswith(a)
+ if not a or not b:
+ return False
+ return (
+ a.startswith(b)
+ or b.startswith(a)
+ or a.endswith(b)
+ or b.endswith(a)
+ or ((len(a) > 10 and len(b) > 10) and (a in b or b in a))
+ )
+
+
+def get_run_url(run_data):
+ if not run_data:
+ return None
+ if run_data.get("html_url"):
+ return run_data.get("html_url")
+ run_id = run_data.get("id")
+ if run_id:
+ return f"https://github.com/smartcontractkit/chainlink/actions/runs/{run_id}"
+ return None
+
+def get_job_url(job, run_url=None):
+ if not job:
+ return None
+ if job.get("html_url"):
+ return job.get("html_url")
+ job_id = job.get("id")
+ if job_id and run_url:
+ return f"{run_url}/job/{job_id}"
+ return None
+
+def get_overall_cost(data):
+ run = data.get("run", {})
+ if "total_cost" in run:
+ return run.get("total_cost") or "N/A"
+ jobs = data.get("jobs", [])
+ total_val = 0.0
+ has_cost = False
+ for j in jobs:
+ c_str = None
+ if "cost" in j:
+ c_str = j.get("cost")
+ elif "metrics" in j and isinstance(j["metrics"], dict) and "Cost" in j["metrics"]:
+ c_str = j["metrics"]["Cost"]
+ if c_str and c_str != "N/A":
+ try:
+ total_val += float(c_str.replace('$', '').strip())
+ has_cost = True
+ except ValueError:
+ pass
+ return f"${total_val:.4f}" if has_cost else "N/A"
+
def generate_comparison(data1, data2):
run1 = data1.get("run", {})
run2 = data2.get("run", {})
+ run1_url = get_run_url(run1)
+ run2_url = get_run_url(run2)
+
run_dur_diff, run_dur_pct = compare_durations(run1.get("runtime"), run2.get("runtime"))
+ cost1 = get_overall_cost(data1)
+ cost2 = get_overall_cost(data2)
+ run_cost_diff, run_cost_pct = compare_costs(cost1, cost2)
+
+ id1_str = f"[{run1.get('id')}]({run1_url})" if run1_url else f"`{run1.get('id')}`"
+ st1_str = f"[{run1.get('conclusion')}]({run1_url})" if run1_url else f"`{run1.get('conclusion')}`"
+ id2_str = f"[{run2.get('id')}]({run2_url})" if run2_url else f"`{run2.get('id')}`"
+ st2_str = f"[{run2.get('conclusion')}]({run2_url})" if run2_url else f"`{run2.get('conclusion')}`"
lines = [
"# Workflow Trial Comparison",
- f"- **Base Run ID (Trial 1)**: `{run1.get('id')}` (Status: `{run1.get('conclusion')}`, Runtime: `{run1.get('runtime')}`)",
- f"- **New Run ID (Trial 2)**: `{run2.get('id')}` (Status: `{run2.get('conclusion')}`, Runtime: `{run2.get('runtime')}`)",
+ f"- **Base Run ID (Trial 1)**: {id1_str} (Status: {st1_str}, Runtime: `{run1.get('runtime')}`, Cost: `{cost1}`)",
+ f"- **New Run ID (Trial 2)**: {id2_str} (Status: {st2_str}, Runtime: `{run2.get('runtime')}`, Cost: `{cost2}`)",
f"- **Runtime Delta**: `{run_dur_diff}` ({run_dur_pct})",
+ f"- **Cost Delta**: `{run_cost_diff}` ({run_cost_pct})",
"",
"## Jobs Comparison"
]
@@ -203,9 +326,14 @@ def generate_comparison(data1, data2):
m2 = j2.get("metrics", {})
cost_diff, cost_pct = compare_costs(m1.get("Cost"), m2.get("Cost"))
+
+ j1_url = get_job_url(j1, run1_url)
+ j2_url = get_job_url(j2, run2_url)
+ j1_st = f"[{j1.get('conclusion')}]({j1_url})" if j1_url else f"`{j1.get('conclusion')}`"
+ j2_st = f"[{j2.get('conclusion')}]({j2_url})" if j2_url else f"`{j2.get('conclusion')}`"
lines.append(f"\n### Job: {name}")
- lines.append(f"- **Status**: `{j1.get('conclusion')}` -> `{j2.get('conclusion')}`")
+ lines.append(f"- **Status**: {j1_st} -> {j2_st}")
lines.append(f"- **Runner**: `{j1.get('runner', {}).get('labels', 'None')}` -> `{j2.get('runner', {}).get('labels', 'None')}`")
lines.append(f"- **Runner Name**: `{j1.get('runner', {}).get('name', 'Unknown')}` -> `{j2.get('runner', {}).get('name', 'Unknown')}`")
lines.append("")
@@ -249,23 +377,38 @@ def generate_comparison(data1, data2):
def main():
args = parse_args()
+ trials_dir = get_trials_dir()
+
+ try:
+ workflow1, trial1, path1 = resolve_trial(trials_dir, args.trial_before)
+ workflow2, trial2, path2 = resolve_trial(trials_dir, args.trial_after)
+ except Exception as e:
+ print(f"Error locating trial reports: {e}", file=sys.stderr)
+ sys.exit(1)
+
+ if workflow1 != workflow2:
+ print(f"Error: trials must belong to the same workflow (found {workflow1!r} and {workflow2!r})", file=sys.stderr)
+ sys.exit(1)
+
try:
- data1 = load_report(args.file1)
- data2 = load_report(args.file2)
+ data1 = load_report(path1)
+ data2 = load_report(path2)
except Exception as e:
print(f"Error loading JSON reports: {e}", file=sys.stderr)
sys.exit(1)
-
+
report = generate_comparison(data1, data2)
print(report)
-
- if args.out_file:
- try:
- with open(args.out_file, 'w', encoding='utf-8') as f:
- f.write(report)
- print(f"Comparison report saved to: {args.out_file}", file=sys.stderr)
- except Exception as e:
- print(f"Error saving report to {args.out_file}: {e}", file=sys.stderr)
+
+ out_dir = os.path.join(trials_dir, workflow1)
+ os.makedirs(out_dir, exist_ok=True)
+ out_file = os.path.join(out_dir, f"{trial1}-{trial2}-comparison.md")
+ try:
+ with open(out_file, 'w', encoding='utf-8') as f:
+ f.write(report)
+ print(f"Comparison report saved to: {out_file}", file=sys.stderr)
+ except Exception as e:
+ print(f"Error saving report to {out_file}: {e}", file=sys.stderr)
if __name__ == "__main__":
main()
diff --git a/.github/.agents/skills/right-size-runners/scripts/workflow_monitor.py b/.github/.agents/skills/optimize-workflow/scripts/workflow_monitor.py
similarity index 63%
rename from .github/.agents/skills/right-size-runners/scripts/workflow_monitor.py
rename to .github/.agents/skills/optimize-workflow/scripts/workflow_monitor.py
index de182533459..57f5c791442 100644
--- a/.github/.agents/skills/right-size-runners/scripts/workflow_monitor.py
+++ b/.github/.agents/skills/optimize-workflow/scripts/workflow_monitor.py
@@ -14,11 +14,10 @@
def parse_args():
parser = argparse.ArgumentParser(description="Monitor GitHub Actions Workflow Run")
parser.add_argument("run_id", type=int, help="GitHub Actions Workflow Run ID")
+ parser.add_argument("trial_name", help="Trial name used to create the output directory.")
parser.add_argument("--repo", help="GitHub repository (owner/repo). Auto-detected if not specified.")
parser.add_argument("--token", help="GitHub token. Defaults to GITHUB_TOKEN env var.")
parser.add_argument("--poll-interval", type=int, default=10, help="Interval (seconds) to poll workflow run progress.")
- parser.add_argument("--format", choices=["markdown", "json"], default="markdown", help="Output format. Defaults to markdown.")
- parser.add_argument("--out-file", help="Path to write the output report.")
return parser.parse_args()
def log_stderr(msg):
@@ -37,6 +36,30 @@ def detect_repo():
pass
return "smartcontractkit/chainlink"
+
+def get_trials_dir():
+ # Trial directories live directly inside the optimize-workflow skill directory.
+ return os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'trials'))
+
+
+def sanitize_dir_name(name):
+ if not name:
+ return "unknown"
+ name = name.replace('/', '_').replace('\\', '_')
+ while '..' in name:
+ name = name.replace('..', '_')
+ return name.strip()
+
+
+def derive_workflow_name(run_data):
+ path = run_data.get('path')
+ if path:
+ name = os.path.basename(path)
+ else:
+ name = run_data.get('name') or 'unknown'
+ return sanitize_dir_name(name)
+
+
def get_headers(token):
headers = {
'Accept': 'application/vnd.github+json',
@@ -192,6 +215,17 @@ def parse_job_logs(log_dir):
results = {k: v for k, v in results.items() if v}
return results
+def parse_duration_seconds(start_str, end_str):
+ if not start_str or not end_str:
+ return 0
+ try:
+ start = datetime.datetime.strptime(start_str.replace("Z", ""), "%Y-%m-%dT%H:%M:%S")
+ end = datetime.datetime.strptime(end_str.replace("Z", ""), "%Y-%m-%dT%H:%M:%S")
+ delta = end - start
+ return int(delta.total_seconds())
+ except Exception:
+ return 0
+
def format_duration(start_str, end_str):
if not start_str or not end_str:
return "N/A"
@@ -207,46 +241,107 @@ def format_duration(start_str, end_str):
return "N/A"
def normalize_name(name):
+ if not name:
+ return ""
name = name.lower()
- name = name.replace('/', '_')
- name = re.sub(r'\s+', '', name)
- name = name.replace('...', '')
- return name.rstrip('.')
+ return re.sub(r'[^a-z0-9]', '', name)
def matches_job_name(api_name, log_job_name):
a = normalize_name(api_name)
b = normalize_name(log_job_name)
- return a.startswith(b) or b.startswith(a)
+ if not a or not b:
+ return False
+ return (
+ a.startswith(b)
+ or b.startswith(a)
+ or a.endswith(b)
+ or b.endswith(a)
+ or ((len(a) > 10 and len(b) > 10) and (a in b or b in a))
+ )
+
+def find_job_metrics(job_name, metrics):
+ for k, v in metrics.items():
+ if normalize_name(job_name) == normalize_name(k):
+ return v
+ for k, v in metrics.items():
+ if matches_job_name(job_name, k):
+ return v
+ return None
+
+def get_job_cost(metrics):
+ return metrics.get("Cost", "N/A") if metrics else "N/A"
+
+def get_run_url(run_data):
+ if not run_data:
+ return None
+ if run_data.get("html_url"):
+ return run_data.get("html_url")
+ run_id = run_data.get("id")
+ if run_id:
+ return f"https://github.com/smartcontractkit/chainlink/actions/runs/{run_id}"
+ return None
+
+def get_job_url(job, run_url=None):
+ if not job:
+ return None
+ if job.get("html_url"):
+ return job.get("html_url")
+ job_id = job.get("id")
+ if job_id and run_url:
+ return f"{run_url}/job/{job_id}"
+ return None
def generate_report(run_data, jobs, metrics, log_dir, format_type):
start_time = run_data.get("run_started_at") or run_data.get("created_at")
end_time = run_data.get("updated_at")
total_runtime = format_duration(start_time, end_time)
+ total_runtime_sec = parse_duration_seconds(start_time, end_time)
+ run_url = get_run_url(run_data)
+
+ # Compute workflow total cost
+ total_cost_val = 0.0
+ has_cost_data = False
+ for job in jobs:
+ jm = find_job_metrics(job.get('name', ''), metrics)
+ c_str = get_job_cost(jm)
+ if c_str != "N/A":
+ try:
+ val = float(c_str.replace('$', '').strip())
+ total_cost_val += val
+ has_cost_data = True
+ except ValueError:
+ pass
+
+ total_workflow_cost = f"${total_cost_val:.4f}" if has_cost_data else "N/A"
if format_type == "json":
jobs_json = []
+ slowest = []
+ durations = []
+ for job in jobs:
+ job_dur_sec = parse_duration_seconds(job.get("started_at"), job.get("completed_at"))
+ if job_dur_sec > 0:
+ durations.append(job_dur_sec)
+
+ avg_dur_sec = int(sum(durations) / len(durations)) if durations else 0
+
for job in jobs:
job_dur = format_duration(job.get("started_at"), job.get("completed_at"))
+ job_dur_sec = parse_duration_seconds(job.get("started_at"), job.get("completed_at"))
labels_list = job.get("labels", [])
runner_name = job.get("runner_name") or "Unknown"
+ is_outlier = job_dur_sec > (1.5 * avg_dur_sec) if avg_dur_sec > 0 else False
- # Find metrics
job_name = job.get('name', '')
- job_metrics = None
- for k, v in metrics.items():
- if normalize_name(job_name) == normalize_name(k):
- job_metrics = v
- break
- if not job_metrics:
- for k, v in metrics.items():
- if matches_job_name(job_name, k):
- job_metrics = v
- break
+ job_metrics = find_job_metrics(job_name, metrics)
+ job_cost = get_job_cost(job_metrics)
+ job_url = get_job_url(job, run_url)
job_entry = {
"name": job.get("name"),
"status": job.get("status"),
"conclusion": job.get("conclusion"),
+ "html_url": job_url,
"runner": {
"labels": labels_list,
"name": runner_name
@@ -254,60 +349,101 @@ def generate_report(run_data, jobs, metrics, log_dir, format_type):
"started_at": job.get("started_at"),
"completed_at": job.get("completed_at"),
"duration": job_dur,
+ "duration_seconds": job_dur_sec,
+ "cost": job_cost,
+ "is_outlier": is_outlier,
"metrics": job_metrics or {}
}
jobs_json.append(job_entry)
-
+ slowest.append({
+ "name": job.get("name"),
+ "duration": job_dur,
+ "duration_seconds": job_dur_sec,
+ "cost": job_cost,
+ "html_url": job_url,
+ "is_outlier": is_outlier,
+ "status": job.get("status"),
+ "conclusion": job.get("conclusion")
+ })
+
+ slowest.sort(key=lambda x: x["duration_seconds"], reverse=True)
+
report_data = {
"run": {
"id": run_data.get("id"),
"status": run_data.get("status"),
"conclusion": run_data.get("conclusion"),
- "runtime": total_runtime
+ "html_url": run_url,
+ "runtime": total_runtime,
+ "runtime_seconds": total_runtime_sec,
+ "total_cost": total_workflow_cost,
+ "avg_job_duration_seconds": avg_dur_sec
},
"logs_dir": log_dir,
+ "slowest_jobs": slowest[:10],
"jobs": jobs_json
}
return json.dumps(report_data, indent=2)
else: # markdown
+ run_id_str = f"[{run_data.get('id')}]({run_url})" if run_url else f"`{run_data.get('id')}`"
+ run_status_str = f"[{run_data.get('status')}]({run_url})" if run_url else f"`{run_data.get('status')}`"
+
lines = [
"# Workflow Run Summary",
- f"- **ID**: `{run_data.get('id')}`",
- f"- **Status**: `{run_data.get('status')}`",
+ f"- **ID**: {run_id_str}",
+ f"- **Status**: {run_status_str}",
f"- **Conclusion**: `{run_data.get('conclusion')}`",
f"- **Runtime**: `{total_runtime}`",
+ f"- **Total Cost**: `{total_workflow_cost}`",
f"- **Logs Directory**: `{log_dir}`",
"",
- "## Jobs Summary"
+ "## Longest Jobs (Bottlenecks)"
]
+
+ # Calculate durations and sort jobs descending
+ sorted_jobs = []
+ for j in jobs:
+ dur_sec = parse_duration_seconds(j.get("started_at"), j.get("completed_at"))
+ sorted_jobs.append((dur_sec, j))
+ sorted_jobs.sort(key=lambda x: x[0], reverse=True)
+
+ lines.append("| Job Name | Duration | Cost | Status | Conclusion | Runner |")
+ lines.append("| --- | --- | --- | --- | --- | --- |")
+ for dur_sec, j in sorted_jobs[:10]:
+ dur_str = format_duration(j.get("started_at"), j.get("completed_at"))
+ jm = find_job_metrics(j.get('name', ''), metrics)
+ c_str = get_job_cost(jm)
+ labels_list = j.get("labels", [])
+ labels_str = ", ".join(labels_list) if labels_list else "None"
+ job_url = get_job_url(j, run_url)
+ job_name_str = f"[{j.get('name')}]({job_url})" if job_url else j.get('name')
+ job_status_str = f"[{j.get('status')}]({job_url})" if job_url else f"`{j.get('status')}`"
+ lines.append(f"| {job_name_str} | `{dur_str}` | `{c_str}` | {job_status_str} | `{j.get('conclusion')}` | `{labels_str}` |")
+
+ lines.append("\n## Jobs Summary")
+
for job in jobs:
job_dur = format_duration(job.get("started_at"), job.get("completed_at"))
+ job_name = job.get('name', '')
+ job_metrics = find_job_metrics(job_name, metrics)
+ job_cost = get_job_cost(job_metrics)
+ job_url = get_job_url(job, run_url)
+ job_name_str = f"[{job.get('name')}]({job_url})" if job_url else job.get('name')
+ job_status_str = f"[{job.get('status')}]({job_url})" if job_url else f"`{job.get('status')}`"
labels_list = job.get("labels", [])
labels_str = ", ".join(labels_list) if labels_list else "None"
runner_name = job.get("runner_name") or "Unknown"
- lines.append(f"\n### Job: {job.get('name')}")
- lines.append(f"- **Status**: `{job.get('status')}`")
+ lines.append(f"\n### Job: {job_name_str}")
+ lines.append(f"- **Status**: {job_status_str}")
lines.append(f"- **Conclusion**: `{job.get('conclusion')}`")
lines.append(f"- **Runner**: `{labels_str}` ({runner_name})")
lines.append(f"- **Start**: `{job.get('started_at')}`")
lines.append(f"- **End**: `{job.get('completed_at')}`")
lines.append(f"- **Duration**: `{job_dur}`")
+ lines.append(f"- **Cost**: `{job_cost}`")
- # Look up job metrics
- job_name = job.get('name', '')
- job_metrics = None
- for k, v in metrics.items():
- if normalize_name(job_name) == normalize_name(k):
- job_metrics = v
- break
- if not job_metrics:
- for k, v in metrics.items():
- if matches_job_name(job_name, k):
- job_metrics = v
- break
-
if job_metrics:
lines.append("- **Runner Details**:")
lines.append(" | metric | value |")
@@ -352,45 +488,63 @@ def main():
args = parse_args()
token = args.token or os.environ.get("GITHUB_TOKEN")
repo = args.repo or detect_repo()
-
+ trial_name = sanitize_dir_name(args.trial_name)
+
log_stderr(f"Monitoring workflow run {args.run_id} in {repo}...")
-
+
try:
run_data = wait_for_run(repo, args.run_id, token)
except RuntimeError as e:
log_stderr(f"Error: {e}")
sys.exit(1)
-
+
+ workflow_name = derive_workflow_name(run_data)
+ trials_dir = get_trials_dir()
+ output_dir = os.path.join(trials_dir, workflow_name, trial_name)
+ logs_dir = os.path.join(output_dir, "logs")
+
+ try:
+ os.makedirs(output_dir, exist_ok=True)
+ os.makedirs(logs_dir, exist_ok=True)
+ except Exception as e:
+ log_stderr(f"Error creating output directories: {e}")
+ sys.exit(1)
+
log_stderr("Workflow run found. Waiting for completion...")
run_data = wait_for_completion(repo, args.run_id, token, args.poll_interval)
-
- # Download logs to persistent temp dir
- import tempfile
- log_dir = tempfile.mkdtemp(prefix=f"workflow-logs-{args.run_id}-")
- log_stderr(f"Downloading logs to: {log_dir}...")
-
+
+ log_stderr(f"Downloading logs to: {logs_dir}...")
metrics = {}
- downloaded = download_logs(repo, args.run_id, token, log_dir)
+ downloaded = download_logs(repo, args.run_id, token, logs_dir)
if downloaded:
- metrics = parse_job_logs(log_dir)
-
+ metrics = parse_job_logs(logs_dir)
+
jobs = fetch_jobs(repo, args.run_id, token)
-
- # Generate and print report
- report = generate_report(run_data, jobs, metrics, log_dir, args.format)
-
- # Write to file if specified
- if args.out_file:
- try:
- with open(args.out_file, 'w', encoding='utf-8') as f:
- f.write(report)
- log_stderr(f"Report successfully saved to: {args.out_file}")
- if args.format == 'json':
- log_stderr(f"Try exploring with: jq . {args.out_file}")
- except Exception as e:
- log_stderr(f"Error saving report to {args.out_file}: {e}")
- else:
- print(report)
+
+ json_path = os.path.join(output_dir, "report.json")
+ md_path = os.path.join(output_dir, "report.md")
+
+ try:
+ json_report = generate_report(run_data, jobs, metrics, logs_dir, "json")
+ with open(json_path, 'w', encoding='utf-8') as f:
+ f.write(json_report)
+ log_stderr(f"JSON report saved to: {json_path}")
+ except Exception as e:
+ log_stderr(f"Error saving JSON report: {e}")
+
+ try:
+ md_report = generate_report(run_data, jobs, metrics, logs_dir, "markdown")
+ with open(md_path, 'w', encoding='utf-8') as f:
+ f.write(md_report)
+ log_stderr(f"Markdown report saved to: {md_path}")
+ except Exception as e:
+ log_stderr(f"Error saving Markdown report: {e}")
+
+ print(f"workflow: {workflow_name}")
+ print(f"trial: {trial_name}")
+ print(f"json: {json_path}")
+ print(f"markdown: {md_path}")
+ print(f"logs: {logs_dir}")
if __name__ == "__main__":
main()
diff --git a/.github/.agents/skills/right-size-runners/SKILL.md b/.github/.agents/skills/right-size-runners/SKILL.md
deleted file mode 100644
index b5365360a83..00000000000
--- a/.github/.agents/skills/right-size-runners/SKILL.md
+++ /dev/null
@@ -1,74 +0,0 @@
----
-name: right-size-runners
-description: Properly size CI runners for price, performance, and stability.
-disable-model-invocation: true
----
-
-
-Missing info? Ask user:
-1. Workflow to optimize.
-2. Priorities (default: stability > speed > cost).
-3. "Speed" definition (e.g., cache hits vs raw execution).
-4. "Stability" definition (e.g., are flaky tests acceptable?).
-5. Determine "spot" setting. Don't ask user directly about what spot setting to use, ask questions to lead you to correct setting. [runs-on spot reference](https://runs-on.com/docs/costs/spot-pricing/)
-
-Setup:
-1. Read workflow. Identify jobs, runners, critical path, and bottlenecks.
-2. Ask to optimize specific job or whole workflow.
-3. Modify workflow for testing (bypass gates, use mock inputs, add `workflow_dispatch`).
-4. Init or resume trial log at `.github/.agents/skills/right-size-runners/trials/.md`.
-5. Run a baseline trial with the current runner configuration to establish a performance and stability benchmark.
-
-
-
-- OOM/Out of Disk Space failures are NEVER acceptable.
-- Prefer default `ubuntu-latest` (4core, 16GB RAM). Use `runs-on` for anything more powerful.
-- Validate runner config via available runners API before use.
-- Use `gh` CLI for workflow execution and PRs.
-- Only change one variable per trial to accurately assess its impact.
-- Always compare Apples to Apples
- - If looking to optimize speed when caching isn't a factor, ensure that cache hits on one trial do not unfairly advantage it over another trial.
- - Always document the exact runner configuration used for each trial to maintain reproducibility.
-
-
-
-- [runs-on docs](https://runs-on.com/docs/)
-- [available runners](https://go.runs-on.com/api)
-- [GitHub Action docs](https://docs.github.com/en/actions)
-
-
-
-1. Define trials. Update `.md` log.
-2. Ask user to approve trials. Check if they want to run them in parallel or sequentially.
-3. New (disposable) branch + commit + push. Message: `cpu=X/ram=Y`. PR title: `[DO NOT MERGE] Trial: ` description: details of trial
-4. Trigger workflow.
-5. Collect the `workflow_run_id` and run `python3 .github/.agents/skills/right-size-runners/scripts/workflow_monitor.py [run_id] --format json --out-file .github/.agents/skills/right-size-runners/trials/[trial-name].json` to monitor the run and collect details.
-6. Analyze results, run `python3 .github/.agents/skills/right-size-runners/scripts/workflow_compare.py .github/.agents/skills/right-size-runners/trials/[trial-1].json .github/.agents/skills/right-size-runners/trials/[trial-2].json --out-file .github/.agents/skills/right-size-runners/trials/[trial-1]-[trial-2]-comparison.md` to compare trial results.
-7. Update the trial log with the results and findings.
-8. Present user with condensed results and ask if they want to run more trials or stop.
-
-
-| Runner | Experiment | Expectation | Branch | Run ID | Commit | Stability | Runtime | Cost | Notes |
-|---|---|---|---|---|---|---|---|---|---|
-| `config` | What's tested | Expected result | `branch` | `run-id` | `sha` | Pass/Fail/Flaky | mm:ss | $ | Findings |
-
-
-
-
-When the user says stop, or all possible experiments have been exhausted:
-
-* Suggest final recommendations.
-* Summarize all findings (cost + speed + stability changes) per workflow/job as table(s) for PR description as below format in raw markdown.
-
-```md
-### [Workflow/Job Name] Runner Changes
-
-| Approach | Runner | Stability | Runtime | Runtime Delta (Abs/%) | Cost | Cost Delta (Abs/%) |
-|---|---|---|---|---|---|---|
-| [Old](https://github.com/link/to/baseline/workflow_run) | [original runner before trials] | Pass/Fail/Flaky | mm:ss | +0:00 (+0%) | $ | +$ (+0%) |
-| [New](https://github.com/link/to/final/workflow_run) | [new runner] | Pass/Fail/Flaky | mm:ss | +0:00 (+0%) | $ | +$ (+0%) |
-```
-
-* Remove all debugs, and make final edits on a new branch after approval, and ask user to commit.
-* Cleanup all trial branches, logs, and PRs.
-
\ No newline at end of file
diff --git a/.github/.agents/skills/right-size-runners/scripts/test_workflow_compare.py b/.github/.agents/skills/right-size-runners/scripts/test_workflow_compare.py
deleted file mode 100644
index 1c54ea1bcbc..00000000000
--- a/.github/.agents/skills/right-size-runners/scripts/test_workflow_compare.py
+++ /dev/null
@@ -1,92 +0,0 @@
-import unittest
-import json
-import workflow_compare
-
-class TestWorkflowCompare(unittest.TestCase):
-
- def test_compare_durations_decrease(self):
- diff, pct_str = workflow_compare.compare_durations("0:03:20", "0:02:40")
- self.assertEqual(diff, "-0:00:40")
- self.assertEqual(pct_str, "-20.0%")
-
- def test_compare_durations_increase(self):
- diff, pct_str = workflow_compare.compare_durations("0:02:00", "0:02:30")
- self.assertEqual(diff, "+0:00:30")
- self.assertEqual(pct_str, "+25.0%")
-
- def test_compare_durations_equal(self):
- diff, pct_str = workflow_compare.compare_durations("0:02:00", "0:02:00")
- self.assertEqual(diff, "0:00:00")
- self.assertEqual(pct_str, "0.0%")
-
- def test_compare_costs_decrease(self):
- diff, pct_str = workflow_compare.compare_costs("$0.1000", "$0.0800")
- self.assertEqual(diff, "-$0.0200")
- self.assertEqual(pct_str, "-20.0%")
-
- def test_compare_costs_increase(self):
- diff, pct_str = workflow_compare.compare_costs("$0.1000", "$0.1150")
- self.assertEqual(diff, "+$0.0150")
- self.assertEqual(pct_str, "+15.0%")
-
- def test_compare_costs_equal(self):
- diff, pct_str = workflow_compare.compare_costs("$0.1000", "$0.1000")
- self.assertEqual(diff, "$0.0000")
- self.assertEqual(pct_str, "0.0%")
-
- def test_compare_metrics(self):
- m1 = {"min": "10.0", "max": "50.0", "avg": "30.0 MB"}
- m2 = {"min": "12.0", "max": "45.0", "avg": "28.5 MB"}
-
- diff = workflow_compare.compare_metrics(m1, m2)
- # We expect it to format average delta: e.g. "avg: 30.0 -> 28.5 MB (-1.5 MB)" or similar
- self.assertIn("avg: 30.0 MB -> 28.5 MB", diff)
- self.assertIn("-1.5 MB", diff)
-
- def test_generate_comparison(self):
- data1 = {
- "run": {"id": 123, "runtime": "0:10:00", "status": "completed", "conclusion": "success"},
- "logs_dir": "/tmp/logs1",
- "jobs": [
- {
- "name": "build",
- "status": "completed",
- "conclusion": "success",
- "runner": {"labels": ["runs-on"], "name": "runner-1"},
- "duration": "0:03:20",
- "metrics": {
- "Instance Type": "c6in.4xlarge",
- "Cost": "$0.1000",
- "system.cpu.load_average.1m": {"min": "0.1", "max": "10.0", "avg": "5.0"}
- }
- }
- ]
- }
- data2 = {
- "run": {"id": 124, "runtime": "0:09:00", "status": "completed", "conclusion": "success"},
- "logs_dir": "/tmp/logs2",
- "jobs": [
- {
- "name": "build",
- "status": "completed",
- "conclusion": "success",
- "runner": {"labels": ["runs-on"], "name": "runner-2"},
- "duration": "0:02:40",
- "metrics": {
- "Instance Type": "c7i-flex.8xlarge",
- "Cost": "$0.0800",
- "system.cpu.load_average.1m": {"min": "0.2", "max": "12.0", "avg": "6.0"}
- }
- }
- ]
- }
-
- report = workflow_compare.generate_comparison(data1, data2)
- self.assertIn("# Workflow Trial Comparison", report)
- self.assertIn("c6in.4xlarge", report)
- self.assertIn("c7i-flex.8xlarge", report)
- self.assertIn("-0:00:40", report) # duration delta
- self.assertIn("-$0.0200", report) # cost delta
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/.github/AGENTS.md b/.github/AGENTS.md
index deba5daa58b..0026229460b 100644
--- a/.github/AGENTS.md
+++ b/.github/AGENTS.md
@@ -6,7 +6,6 @@ GitHub Actions in the Chainlink Go monorepo.
- Prefer runs-on runners when ubuntu-latest is insufficient.
- Minimize YAML and shell in workflows.
- Resolve smartcontractkit/.github from a local clone. Ask the user for the path if you cannot find it.
-- Do not fetch smartcontractkit/.github from the web unless no local copy exists or local copy does not match required CI behavior.
diff --git a/.github/actions/setup-go/action.yml b/.github/actions/setup-go/action.yml
index 03bc69f52ff..559fb26bc1d 100644
--- a/.github/actions/setup-go/action.yml
+++ b/.github/actions/setup-go/action.yml
@@ -49,7 +49,7 @@ runs:
echo "version=$version" >> "$GITHUB_OUTPUT"
- name: Set up Go
- uses: actions/setup-go@v6
+ uses: actions/setup-go@v7
with:
go-version: ${{ steps.go-version.outputs.version }}
cache: false
diff --git a/.github/actions/start-local-cre-environment/action.yml b/.github/actions/start-local-cre-environment/action.yml
index 5a6cd4b06d3..3d9eaada999 100644
--- a/.github/actions/start-local-cre-environment/action.yml
+++ b/.github/actions/start-local-cre-environment/action.yml
@@ -59,12 +59,26 @@ runs:
set +e
# GitHub invokes bash with errexit (-e); disable it so a failed `env start`
# does not abort the script before we can retry.
+
+ CRE_BIN=""
+ if [[ -x "$GITHUB_WORKSPACE/system-tests/tests/bin/cre-env" ]]; then
+ CRE_BIN="$GITHUB_WORKSPACE/system-tests/tests/bin/cre-env"
+ echo "Using pre-compiled CRE environment CLI binary: ${CRE_BIN}"
+ elif [[ -x "./bin/cre-env" ]]; then
+ CRE_BIN="./bin/cre-env"
+ echo "Using pre-compiled CRE environment CLI binary: ${CRE_BIN}"
+ fi
+
last_exit=1
attempt=1
while [[ "$attempt" -le "$MAX_ATTEMPTS" ]]; do
echo "Starting local CRE (attempt ${attempt}/${MAX_ATTEMPTS})..."
# shellcheck disable=SC2086
- go run . env start ${ENV_START_EXTRA_ARGS} --cleanup-on-error="${CLEANUP_ON_ERROR}"
+ if [[ -n "$CRE_BIN" ]]; then
+ "$CRE_BIN" env start ${ENV_START_EXTRA_ARGS} --cleanup-on-error="${CLEANUP_ON_ERROR}"
+ else
+ go run . env start ${ENV_START_EXTRA_ARGS} --cleanup-on-error="${CLEANUP_ON_ERROR}"
+ fi
last_exit=$?
if [[ "$last_exit" -eq 0 ]]; then
exit 0
@@ -72,7 +86,11 @@ runs:
echo "env start failed with exit code ${last_exit}"
if [[ "$attempt" -lt "$MAX_ATTEMPTS" ]]; then
echo "Running env stop before retry..."
- go run . env stop || true
+ if [[ -n "$CRE_BIN" ]]; then
+ "$CRE_BIN" env stop || true
+ else
+ go run . env stop || true
+ fi
sleep "${RETRY_DELAY_SECONDS}"
fi
attempt=$((attempt + 1))
diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml
deleted file mode 100644
index 0d3d62d3357..00000000000
--- a/.github/e2e-tests.yml
+++ /dev/null
@@ -1,231 +0,0 @@
-# This file specifies the GitHub runner for each E2E test and is utilized by all E2E CI workflows.
-#
-# Each entry in this file includes the following:
-# - The GitHub runner (runs_on field) that will execute tests.
-# - The tests that will be run by the runner.
-# - The triggers (e.g., Run PR E2E Tests, Nightly E2E Tests) that should trigger these tests.
-#
-runner-test-matrix:
- # START: CCIPv1.6 tests
-
- - id: smoke/ccip/ccip_reorg_test.go:LessThanFinalityTests
- path: integration-tests/smoke/ccip/ccip_reorg_test.go
- test_env_type: docker
- runs_on: ubuntu-latest
- runs_on_self_hosted: runs-on/cpu=16/ram=64/family=m7i+m8i/extras=s3-cache+tmpfs
- triggers:
- - Nightly E2E Tests
- - Push E2E CCIP v1.6 Tests
- - Workflow Dispatch E2E CCIP v1.6 Tests
- test_cmd: |
- cd integration-tests/smoke/ccip && \
- gotestsum \
- --junitfile=/tmp/junit.xml \
- --jsonfile=/tmp/gotest.log \
- --format=github-actions \
- -- -v -run "Test_CCIPReorg_BelowFinality_OnSource|Test_CCIPReorg_BelowFinality_OnDest" -timeout 25m -parallel=1 -count=1
- pyroscope_env: ci-smoke-ccipv1_6-evm-simulated
- test_env_vars:
- E2E_TEST_SELECTED_NETWORK: SIMULATED_1_DEEPER_FINALITY,SIMULATED_2_DEEPER_FINALITY
- E2E_JD_VERSION: 0.9.0
- CCIP_V16_TEST_ENV: docker
-
- - id: smoke/ccip/ccip_reorg_test.go:GreaterThanFinalityTests
- path: integration-tests/smoke/ccip/ccip_reorg_test.go
- test_env_type: docker
- runs_on: ubuntu-latest
- runs_on_self_hosted: runs-on/cpu=16/ram=64/family=m7i+m8i/extras=s3-cache+tmpfs
- triggers:
- - Nightly E2E Tests
- - Push E2E CCIP v1.6 Tests
- - Workflow Dispatch E2E CCIP v1.6 Tests
- test_cmd: |
- cd integration-tests/smoke/ccip && \
- gotestsum \
- --junitfile=/tmp/junit.xml \
- --jsonfile=/tmp/gotest.log \
- --format=github-actions \
- -- -v -run "Test_CCIPReorg_GreaterThanFinality_OnSource|Test_CCIPReorg_GreaterThanFinality_OnDest" -timeout 25m -parallel=1 -count=1
- pyroscope_env: ci-smoke-ccipv1_6-evm-simulated
- test_env_vars:
- E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2,SIMULATED_3
- E2E_JD_VERSION: 0.9.0
- CCIP_V16_TEST_ENV: docker
-
- - id: smoke/ccip/ccip_token_price_updates_test.go:*
- path: integration-tests/smoke/ccip/ccip_token_price_updates_test.go
- test_env_type: docker
- runs_on: ubuntu-latest
- runs_on_self_hosted: runs-on/cpu=16/ram=64/family=m7i+m8i/extras=s3-cache+tmpfs
- triggers:
- - PR E2E CCIP v1.6 Tests
- - Merge Queue E2E CCIP v1.6 Tests
- - Nightly E2E Tests
- - Push E2E CCIP v1.6 Tests
- - Workflow Dispatch E2E CCIP v1.6 Tests
- test_cmd: |
- cd smoke/ccip && \
- gotestsum \
- --junitfile=/tmp/junit.xml \
- --jsonfile=/tmp/gotest.log \
- --format=github-actions \
- -- -v -run "^Test_CCIPTokenPriceUpdates$" -timeout 18m -count=1 -parallel=1 github.com/smartcontractkit/chainlink/integration-tests/smoke/ccip
- pyroscope_env: ci-smoke-ccipv1_6-evm-simulated
- test_env_vars:
- E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2
- E2E_JD_VERSION: 0.9.0
- CCIP_V16_TEST_ENV: docker
- test_go_project_path: integration-tests
- free_disk_space: true
-
- - id: smoke/ccip/ccip_gas_price_updates_test.go:^Test_CCIPGasPriceUpdatesWriteFrequency$
- path: integration-tests/smoke/ccip/ccip_gas_price_updates_test.go
- test_env_type: docker
- runs_on: ubuntu-latest
- runs_on_self_hosted: runs-on/cpu=16/ram=64/family=m7i+m8i/extras=s3-cache+tmpfs
- triggers:
- - PR E2E CCIP v1.6 Tests
- - Merge Queue E2E CCIP v1.6 Tests
- - Nightly E2E Tests
- - Push E2E CCIP v1.6 Tests
- - Workflow Dispatch E2E CCIP v1.6 Tests
- test_cmd: |
- cd smoke/ccip && \
- gotestsum \
- --junitfile=/tmp/junit.xml \
- --jsonfile=/tmp/gotest.log \
- --format=github-actions \
- -- -v -run "^Test_CCIPGasPriceUpdatesWriteFrequency$" -timeout 18m -count=1 -parallel=1 github.com/smartcontractkit/chainlink/integration-tests/smoke/ccip
- pyroscope_env: ci-smoke-ccipv1_6-evm-simulated
- test_env_vars:
- E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2
- E2E_JD_VERSION: 0.9.0
- CCIP_V16_TEST_ENV: docker
- test_go_project_path: integration-tests
-
- - id: smoke/ccip/ccip_gas_price_updates_test.go:^Test_CCIPGasPriceUpdatesDeviation$
- path: integration-tests/smoke/ccip/ccip_gas_price_updates_test.go
- test_env_type: docker
- runs_on: ubuntu-latest
- runs_on_self_hosted: runs-on/cpu=16/ram=64/family=m7i+m8i/extras=s3-cache+tmpfs
- triggers:
- - PR E2E CCIP v1.6 Tests
- - Merge Queue E2E CCIP v1.6 Tests
- - Nightly E2E Tests
- - Push E2E CCIP v1.6 Tests
- - Workflow Dispatch E2E CCIP v1.6 Tests
- test_cmd: |
- cd smoke/ccip && \
- gotestsum \
- --junitfile=/tmp/junit.xml \
- --jsonfile=/tmp/gotest.log \
- --format=github-actions \
- -- -v -run "^Test_CCIPGasPriceUpdatesDeviation$" -timeout 18m -count=1 -parallel=1 github.com/smartcontractkit/chainlink/integration-tests/smoke/ccip
- pyroscope_env: ci-smoke-ccipv1_6-evm-simulated
- test_env_vars:
- E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2
- E2E_JD_VERSION: 0.9.0
- CCIP_V16_TEST_ENV: docker
- test_go_project_path: integration-tests
-
- - id: smoke/ccip/ccip_rmn_test.go:^TestRMN_TwoMessagesOneSourceChainCursed$
- path: integration-tests/smoke/ccip/ccip_rmn_test.go
- test_env_type: docker
- runs_on: ubuntu24.04-16cores-64GB
- runs_on_self_hosted: runs-on/cpu=16/ram=64/family=m7i+m8i/extras=s3-cache+tmpfs
- triggers:
- # Push/Nightly triggers removed: test consistently times out after 30m (RMN curse-revocation flake). Re-add once fixed.
- test_cmd: |
- cd smoke/ccip && \
- gotestsum \
- --junitfile=/tmp/junit.xml \
- --jsonfile=/tmp/gotest.log \
- --format=github-actions \
- -- -v -run "^TestRMN_TwoMessagesOneSourceChainCursed$" -timeout 30m -parallel=1 -count=1
- pyroscope_env: ci-smoke-ccipv1_6-evm-simulated
- test_env_vars:
- E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2
- E2E_JD_VERSION: 0.9.0
- E2E_RMN_RAGEPROXY_VERSION: master-amd6416f5d86
- E2E_RMN_AFN2PROXY_VERSION: master-amd64-10b42b2
- CCIP_V16_TEST_ENV: docker
- test_go_project_path: integration-tests
-
- - id: smoke/ccip/ccip_rmn_test.go:^TestRMN_GlobalCurseTwoMessagesOnTwoLanes$
- path: integration-tests/smoke/ccip/ccip_rmn_test.go
- test_env_type: docker
- runs_on: ubuntu24.04-16cores-64GB
- runs_on_self_hosted: runs-on/cpu=16/ram=64/family=m7i+m8i/extras=s3-cache+tmpfs
- triggers:
- - PR E2E CCIP v1.6 Tests
- - Nightly E2E Tests
- - Push E2E CCIP v1.6 Tests
- - Workflow Dispatch E2E CCIP v1.6 Tests
- test_cmd: |
- cd smoke/ccip && \
- gotestsum \
- --junitfile=/tmp/junit.xml \
- --jsonfile=/tmp/test.json \
- --format=github-actions \
- -- -v -run "^TestRMN_GlobalCurseTwoMessagesOnTwoLanes$" -timeout 30m -parallel=1 -count=1
- pyroscope_env: ci-smoke-ccipv1_6-evm-simulated
- test_env_vars:
- E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2
- E2E_JD_VERSION: 0.9.0
- E2E_RMN_RAGEPROXY_VERSION: master-amd6416f5d86
- E2E_RMN_AFN2PROXY_VERSION: master-amd64-10b42b2
- CCIP_V16_TEST_ENV: docker
- test_go_project_path: integration-tests
-
- - id: smoke/ccip/ccip_jobspec_test.go.go:TestDeleteCCIPJobs
- path: integration-tests/smoke/ccip/ccip_jobspec_test.go
- test_env_type: docker
- runs_on: ubuntu-latest
- runs_on_self_hosted: runs-on/cpu=16/ram=64/family=m7i+m8i/extras=s3-cache+tmpfs
- triggers:
- - PR E2E CCIP v1.6 Tests
- - Merge Queue E2E CCIP v1.6 Tests
- - Nightly E2E Tests
- - Push E2E CCIP v1.6 Tests
- - Workflow Dispatch E2E CCIP v1.6 Tests
- test_cmd: |
- cd smoke/ccip && \
- gotestsum \
- --junitfile=/tmp/junit.xml \
- --jsonfile=/tmp/gotest.log \
- --format=github-actions \
- -- -v -run "^TestDeleteCCIPJobs$" -count=1 -parallel=1 github.com/smartcontractkit/chainlink/integration-tests/smoke/ccip
- pyroscope_env: ci-smoke-ccipv1_6-evm-simulated
- test_env_vars:
- E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2
- E2E_JD_VERSION: 0.9.0
- CCIP_V16_TEST_ENV: docker
- test_go_project_path: integration-tests
- free_disk_space: true
-
- - id: smoke/ccip/ccip_jobspec_test.go.go:TestRevokeJobs
- path: integration-tests/smoke/ccip/ccip_jobspec_test.go
- test_env_type: docker
- runs_on: ubuntu-latest
- runs_on_self_hosted: runs-on/cpu=16/ram=64/family=m7i+m8i/extras=s3-cache+tmpfs
- triggers:
- - PR E2E CCIP v1.6 Tests
- - Merge Queue E2E CCIP v1.6 Tests
- - Nightly E2E Tests
- - Push E2E CCIP v1.6 Tests
- - Workflow Dispatch E2E CCIP v1.6 Tests
- test_cmd: |
- cd smoke/ccip && \
- gotestsum \
- --junitfile=/tmp/junit.xml \
- --jsonfile=/tmp/gotest.log \
- --format=github-actions \
- -- -v -run "^TestRevokeJobs$" -count=1 -parallel=1 github.com/smartcontractkit/chainlink/integration-tests/smoke/ccip
- pyroscope_env: ci-smoke-ccipv1_6-evm-simulated
- test_env_vars:
- E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2
- E2E_JD_VERSION: 0.9.0
- CCIP_V16_TEST_ENV: docker
- test_go_project_path: integration-tests
- free_disk_space: true
- # END: CCIPv1.6 tests
diff --git a/.github/workflows/ccip-chaos-tests.yml b/.github/workflows/ccip-chaos-tests.yml
deleted file mode 100644
index a2a8aad9d8a..00000000000
--- a/.github/workflows/ccip-chaos-tests.yml
+++ /dev/null
@@ -1,55 +0,0 @@
-name: CCIP Chaos Tests
-on:
- # Disabled until TT-1771 is resolved
- # workflow_run:
- # workflows: [ CCIP Load Test ]
- # types: [ completed ]
- # branches: [ develop ]
- workflow_dispatch:
- inputs:
- team:
- description: Team to run the tests for (e.g. BIX, CCIP)
- required: true
- default: "ccip"
- type: string
-
-# Only run 1 of this workflow at a time per PR
-concurrency:
- group: chaos-ccip-tests-chainlink-${{ github.ref }}
- cancel-in-progress: true
-
-jobs:
- run-e2e-tests-workflow:
- name: Run E2E Tests
- uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@9c49ffcf252efbedd7ec280e1993ba29dcbc9443 # 2026-04-28
- with:
- test_path: .github/e2e-tests.yml
- chainlink_version: ${{ github.sha }}
- require_chainlink_image_versions_in_qa_ecr: ${{ github.sha }}
- test_trigger: E2E CCIP Chaos Tests
- test_log_level: debug
- slack_notification_after_tests: on_failure
- slack_notification_after_tests_channel_id: "#ccip-testing"
- slack_notification_after_tests_name: CCIP Chaos E2E Tests
- team: ${{ inputs.team }}
- secrets:
- QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
- QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
- QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
- PROD_AWS_ACCOUNT_NUMBER: ${{ secrets.AWS_ACCOUNT_ID_PROD }}
- QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
- QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
- GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
- GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
- GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
- GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
- LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
- LOKI_URL: ${{ secrets.LOKI_URL }}
- LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- AWS_REGION: ${{ secrets.QA_AWS_REGION }}
- AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
- AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
- SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
- MAIN_DNS_ZONE_PUBLIC_SDLC: ${{ secrets.MAIN_DNS_ZONE_PUBLIC_SDLC }}
- AWS_K8S_CLUSTER_NAME_SDLC: ${{ secrets.AWS_K8S_CLUSTER_NAME_SDLC }}
diff --git a/.github/workflows/ccip-system-tests.yaml b/.github/workflows/ccip-system-tests.yaml
new file mode 100644
index 00000000000..00f4f853084
--- /dev/null
+++ b/.github/workflows/ccip-system-tests.yaml
@@ -0,0 +1,271 @@
+name: CCIP System Tests
+
+on:
+ workflow_dispatch:
+ inputs:
+ chainlink_image_repository_path:
+ description:
+ "ECR repository name used to compose image with chainlink_image_tag
+ (for example: chainlink-integration-tests or chainlink or
+ chainlink/chainlink)."
+ required: true
+ type: string
+ chainlink_image_tag:
+ required: true
+ type: string
+ description: "Chainlink image tag to use."
+ ecr:
+ required: false
+ type: choice
+ options:
+ - "sdlc"
+ - "public"
+ default: "sdlc"
+ description:
+ "Whether to use the SDLC or public ECR registry for the Chainlink
+ image."
+ chainlink_version:
+ required: false
+ type: string
+ description:
+ "The version of Chainlink repository to use for the tests. If
+ empty, defaults to github.sha."
+ default: ""
+ workflow_call:
+ inputs:
+ chainlink_image_repository_path:
+ description:
+ "ECR repository name used to compose image with chainlink_image_tag
+ (for example: chainlink-integration-tests or chainlink or
+ chainlink/chainlink)."
+ required: true
+ type: string
+ chainlink_image_tag:
+ required: true
+ type: string
+ description: "Chainlink image tag to use."
+ ecr:
+ type: string
+ required: true
+ description:
+ "Whether to use the SDLC (sdlc) or public ECR registry (public) for
+ the Chainlink image."
+ chainlink_version:
+ required: false
+ type: string
+ description:
+ "The version of Chainlink repository to use for the tests. If
+ empty, defaults to github.sha."
+ default: ""
+
+jobs:
+ define-test-matrix:
+ runs-on: ubuntu-latest
+ outputs:
+ matrix: ${{ steps.define-matrix.outputs.matrix }}
+ permissions:
+ contents: read
+ steps:
+ - name: Define test matrix
+ id: define-matrix
+ shell: bash
+ run: |
+ TESTS_JSON='[
+ {"test_name":"Test_CCIPReorg_BelowFinality_OnSource","timeout":"25m","selected_network":"SIMULATED_1_DEEPER_FINALITY,SIMULATED_2_DEEPER_FINALITY"},
+ {"test_name":"Test_CCIPReorg_BelowFinality_OnDest","timeout":"25m","selected_network":"SIMULATED_1_DEEPER_FINALITY,SIMULATED_2_DEEPER_FINALITY"},
+ {"test_name":"Test_CCIPReorg_GreaterThanFinality_OnSource","timeout":"25m","selected_network":"SIMULATED_1,SIMULATED_2,SIMULATED_3"},
+ {"test_name":"Test_CCIPReorg_GreaterThanFinality_OnDest","timeout":"25m","selected_network":"SIMULATED_1,SIMULATED_2,SIMULATED_3"},
+ {"test_name":"Test_CCIPGasPriceUpdatesWriteFrequency","timeout":"18m","selected_network":"SIMULATED_1,SIMULATED_2","runner_spec":"cpu=4/ram=16"},
+ {"test_name":"TestRMN_GlobalCurseTwoMessagesOnTwoLanes","timeout":"30m","selected_network":"SIMULATED_1,SIMULATED_2","rmn_rageproxy_version":"master-amd6416f5d86","rmn_afn2proxy_version":"master-amd64-10b42b2"},
+ {"test_name":"TestDeleteCCIPJobs|TestRevokeJobs","timeout":"50m","selected_network":"SIMULATED_1,SIMULATED_2","runner_spec":"cpu=4/ram=16","job_timeout":60}
+ ]'
+
+ tests=$(echo "$TESTS_JSON" | jq -c \
+ --argjson run_id "${{ github.run_id }}" \
+ --arg run_attempt "${{ github.run_attempt }}" '
+ to_entries | map(.value + {
+ test_id: .key,
+ runs_on: ("runs-on=\($run_id)-\(.key)-\($run_attempt)/" + (.value.runner_spec // "cpu=8/ram=64") + "/family=r6i+r7i+r8i/spot=co/image=ubuntu24-full-x64/extras=s3-cache+tmpfs")
+ })
+ ')
+
+ echo "matrix=$tests" | tee -a "${GITHUB_OUTPUT}"
+
+ run-ccip-tests:
+ name: ${{ matrix.tests.test_name }}
+ permissions:
+ contents: read
+ id-token: write
+ strategy:
+ fail-fast: false
+ matrix:
+ tests: ${{ fromJson(needs.define-test-matrix.outputs.matrix) }}
+ needs: [define-test-matrix]
+ runs-on: ${{ matrix.tests.runs_on }}
+ environment:
+ # http://docs.github.com/en/actions/how-tos/deploy/configure-and-manage-deployments/control-deployments#using-environments-without-deployments
+ name: integration
+ deployment: false
+ timeout-minutes: ${{ matrix.tests.job_timeout || 45 }}
+ env:
+ ENABLE_AUTO_QUARANTINE: "true"
+ INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
+ E2E_JD_IMAGE: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/job-distributor
+ E2E_RMN_RAGEPROXY_IMAGE: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/rageproxy
+ E2E_RMN_AFN2PROXY_IMAGE: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/afn2proxy
+ CHAINLINK_IMAGE: ${{ inputs.ecr == 'public' && format('public.ecr.aws/{0}', inputs.chainlink_image_repository_path) || format('{0}.dkr.ecr.{1}.amazonaws.com/{2}', secrets.QA_AWS_ACCOUNT_NUMBER, secrets.QA_AWS_REGION, inputs.chainlink_image_repository_path) }}
+ E2E_TEST_CHAINLINK_IMAGE: ${{ inputs.ecr == 'public' && format('public.ecr.aws/{0}', inputs.chainlink_image_repository_path) || format('{0}.dkr.ecr.{1}.amazonaws.com/{2}', secrets.QA_AWS_ACCOUNT_NUMBER, secrets.QA_AWS_REGION, inputs.chainlink_image_repository_path) }}
+
+ steps:
+ - name: Enable S3 Cache for Self-Hosted Runners
+ uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc # v2.2.0
+
+ - name: Checkout
+ uses: actions/checkout@v7
+ with:
+ ref: ${{ inputs.chainlink_version || github.sha }}
+ persist-credentials: false
+
+ - name: Setup GitHub token using GATI
+ id: github-token
+ uses: smartcontractkit/.github/actions/setup-github-token@setup-github-token/v1
+ with:
+ aws-role-arn: ${{ secrets.AWS_OIDC_CHAINLINK_READ_ONLY_TOKEN_ISSUER_ROLE_ARN }}
+ aws-lambda-url: ${{ secrets.AWS_INFRA_RELENG_TOKEN_ISSUER_LAMBDA_URL }}
+ aws-region: us-west-2
+ aws-role-duration-seconds: "1800"
+ set-git-config: "true"
+
+ - name: Set up Go
+ id: setup-go
+ uses: smartcontractkit/.github/actions/ctf-setup-go@fa1d48a33e24f9b3b9f8c52e99a578a4597cb2a5 # v0.4.0
+ with:
+ go_mod_path: integration-tests/go.mod
+ cache_key_id: integration-tests-v1
+ cache_builds: true
+ cache_restore_only: "true"
+ should_tidy: false
+ no_cache: false
+ gati_token: ${{ steps.github-token.outputs.access-token }}
+
+ - name: Restore Pre-Compiled Test Binaries from S3 Cache
+ uses: actions/cache/restore@v6
+ with:
+ path: integration-tests/bin/
+ key: test-binaries-${{ inputs.chainlink_version || github.sha }}
+
+ - name: Configure AWS Credentials
+ uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3
+ with:
+ aws-region: ${{ secrets.QA_AWS_REGION }}
+ role-to-assume: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+ role-duration-seconds: 1800
+ mask-aws-account-id: true
+
+ - name: Login to Amazon ECR
+ id: login-ecr
+ uses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64 # v2.1.6
+ with:
+ registries: ${{ format('{0},{1}', secrets.QA_AWS_ACCOUNT_NUMBER, secrets.AWS_ACCOUNT_ID_PROD) }}
+ env:
+ AWS_REGION: ${{ secrets.QA_AWS_REGION }}
+
+ - name: Cache gotestsum
+ id: cache-gotestsum
+ uses: actions/cache@v6
+ with:
+ path: ~/go/bin/gotestsum
+ key: gotestsum-1.13.0
+
+ - name: Set up gotestsum
+ shell: bash
+ run: |
+ if [ -f ~/go/bin/gotestsum ]; then
+ echo "Using cached gotestsum"
+ else
+ echo "::group::Install gotestsum"
+ go install gotest.tools/gotestsum@v1.13.0
+ echo "::endgroup::"
+ fi
+
+ - name: Run CCIP smoke tests
+ id: run-tests
+ shell: bash
+ working-directory: integration-tests
+ continue-on-error: ${{ env.ENABLE_AUTO_QUARANTINE == 'true' }}
+ env:
+ TEST_NAME: ${{ matrix.tests.test_name }}
+ TEST_TIMEOUT: ${{ matrix.tests.timeout }}
+ E2E_TEST_SELECTED_NETWORK: ${{ matrix.tests.selected_network }}
+ E2E_JD_VERSION: "0.9.0"
+ CCIP_V16_TEST_ENV: "docker"
+ E2E_RMN_RAGEPROXY_VERSION: ${{ matrix.tests.rmn_rageproxy_version || '' }}
+ E2E_RMN_AFN2PROXY_VERSION: ${{ matrix.tests.rmn_afn2proxy_version || '' }}
+ E2E_TEST_CHAINLINK_VERSION: ${{ inputs.chainlink_version || github.sha }}
+ GITHUB_TOKEN: ${{ steps.github-token.outputs.access-token || '' }}
+ run: |
+ echo "Starting test: '${TEST_NAME}'"
+ if [ -f "./bin/ccip-smoke.test" ]; then
+ echo "Using precompiled binary"
+ (
+ cd smoke/ccip && \
+ gotestsum \
+ --jsonfile=/tmp/gotest.log \
+ --junitfile=/tmp/junit.xml \
+ --format=github-actions \
+ --raw-command -- \
+ go tool test2json -t -p github.com/smartcontractkit/chainlink/integration-tests/smoke/ccip ../../bin/ccip-smoke.test -test.v -test.run "^(${TEST_NAME})$" -test.timeout "${TEST_TIMEOUT}" -test.parallel=1 -test.count=1
+ )
+ else
+ echo "No precompiled binary found. Building from scratch..."
+ gotestsum \
+ --jsonfile=/tmp/gotest.log \
+ --junitfile=/tmp/junit.xml \
+ --format=github-actions \
+ -- \
+ -v -tags embed -run "^(${TEST_NAME})$" -timeout "${TEST_TIMEOUT}" -count=1 -parallel=1 \
+ github.com/smartcontractkit/chainlink/integration-tests/smoke/ccip
+ fi
+
+ exit_code="$?"
+ if [ "$exit_code" -eq 0 ]; then
+ echo "tests_result=✅ Tests passed" >> "${GITHUB_OUTPUT}"
+ fi
+
+ - name: Analyze and upload test results (${{ steps.run-tests.outputs.tests_result }})
+ if: ${{ !cancelled() }}
+ uses: smartcontractkit/.github/actions/branch-out-upload@branch-out-upload/v1
+ with:
+ junit-file-path: "/tmp/junit.xml"
+ trunk-org-slug: chainlink
+ trunk-token: ${{ secrets.TRUNK_API_KEY }}
+ trunk-previous-step-outcome: ${{ steps.run-tests.outcome }}
+ # when auto-quarantine is enabled, allow this to determine test failures
+ trunk-upload-only: ${{ env.ENABLE_AUTO_QUARANTINE != 'true' }}
+ # unique name per matrix job to avoid 409 conflict when multiple jobs upload in the same workflow run
+ artifact-name: ${{ matrix.tests.test_id }}_test_logs
+
+ - name: Show Docker containers status
+ if: failure() || cancelled()
+ shell: bash
+ run: docker ps -a
+
+ - name: Save Docker logs
+ if: failure() || cancelled()
+ shell: bash
+ working-directory: integration-tests/smoke/ccip
+ run: |
+ mkdir -p logs
+ for c in $(docker ps -a --format '{{.Names}}'); do
+ docker logs "$c" > "logs/${c}.log" 2>&1
+ done
+
+ - name: Upload all artifacts as single package
+ if: failure() || cancelled()
+ uses: actions/upload-artifact@v7
+ with:
+ name: test-logs-${{ matrix.tests.test_name }}
+ path: |
+ ./integration-tests/smoke/ccip/logs/
+ /tmp/gotest.log
+ /tmp/junit.xml
diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml
index 216fed06329..e1222368fc1 100644
--- a/.github/workflows/ci-core.yml
+++ b/.github/workflows/ci-core.yml
@@ -179,7 +179,7 @@ jobs:
modules: ${{ fromJson(needs.filter.outputs.affected-modules) }}
steps:
- name: Enable S3 Cache for Self-Hosted Runners
- uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2
+ uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc # v2.2.0
- name: Checkout
uses: actions/checkout@v6
@@ -257,7 +257,7 @@ jobs:
type:
- cmd: go_core_tests
os: runs-on=${{ github.run_id
- }}-unit/cpu=48/ram=96/family=c6id+c5ad/spot=false/image=ubuntu24-full-x64/extras=s3-cache
+ }}-unit/cpu=48/ram=96/family=c7id+c8id/spot=false/image=ubuntu24-full-x64/extras=s3-cache
should-run: ${{ needs.filter.outputs.should-run-core-tests }}
trunk-auto-quarantine: "true"
go-mod-directory: ""
@@ -268,7 +268,7 @@ jobs:
- cmd: go_core_tests_integration
os: runs-on=${{ github.run_id
- }}-integ/cpu=48/ram=96/family=c6i/spot=false/image=ubuntu24-full-x64/extras=s3-cache+tmpfs
+ }}-integ/cpu=48/ram=96/family=c7i+c8i/spot=false/image=ubuntu24-full-x64/extras=s3-cache+tmpfs
should-run: ${{ needs.filter.outputs.should-run-core-tests }}
trunk-auto-quarantine: "true"
setup-solana: "true"
@@ -279,7 +279,7 @@ jobs:
- cmd: go_core_fuzz
os: runs-on=${{
- github.run_id}}-fuzz/cpu=8/ram=32/family=m6id+m6idn/spot=false/image=ubuntu24-full-x64/extras=s3-cache
+ github.run_id}}-fuzz/cpu=8/ram=32/family=m7id+m7idn+m8id+m8idn/spot=false/image=ubuntu24-full-x64/extras=s3-cache
should-run: ${{ needs.filter.outputs.should-run-core-tests }}
trunk-auto-quarantine: "false"
go-mod-directory: ""
@@ -290,7 +290,7 @@ jobs:
- cmd: go_core_race_tests
os: runs-on=${{
- github.run_id}}-race/cpu=64/ram=128/family=c7i/volume=80gb/spot=false/image=ubuntu24-full-x64/extras=s3-cache
+ github.run_id}}-race/cpu=64/ram=128/family=c7i+c8i/volume=80gb/spot=false/image=ubuntu24-full-x64/extras=s3-cache
should-run: ${{ needs.filter.outputs.should-run-core-tests }}
trunk-auto-quarantine: "false"
go-mod-directory: ""
@@ -313,11 +313,13 @@ jobs:
actions: read
steps:
- name: Enable S3 Cache for Self-Hosted Runners
- uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2
+ uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc # v2.2.0
+ with:
+ metrics: cpu,network,memory,disk,io
- name: Checkout the repo
if: ${{ matrix.type.should-run == 'true' }}
- uses: actions/checkout@v6
+ uses: actions/checkout@v7
with:
persist-credentials: false
@@ -492,7 +494,7 @@ jobs:
if: ${{ needs.filter.outputs.should-run-core-tests == 'true' }}
steps:
- name: Checkout
- uses: actions/checkout@v6
+ uses: actions/checkout@v7
- name: Setup Go
uses: ./.github/actions/setup-go
@@ -523,7 +525,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout the repo
- uses: actions/checkout@v6
+ uses: actions/checkout@v7
with:
persist-credentials: false
fetch-depth: 0 # fetches all history for all tags and branches to provide more metadata for sonar reports
@@ -630,7 +632,7 @@ jobs:
run:
shell: bash
steps:
- - uses: actions/checkout@v6
+ - uses: actions/checkout@v7
with:
persist-credentials: false
- name: Setup Go
@@ -692,7 +694,7 @@ jobs:
name: Misc
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v6
+ - uses: actions/checkout@v7
with:
persist-credentials: false
- name: Setup go
diff --git a/.github/workflows/ci-deployments.yml b/.github/workflows/ci-deployments.yml
index 287063cf7ad..0b3430b73ad 100644
--- a/.github/workflows/ci-deployments.yml
+++ b/.github/workflows/ci-deployments.yml
@@ -143,7 +143,7 @@ jobs:
actions: read
steps:
- name: Enable S3 Cache for Self-Hosted Runners
- uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2
+ uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc # v2.2.0
- name: Checkout the repo
uses: actions/checkout@v6
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
index f265efc2126..38e5388d7f9 100644
--- a/.github/workflows/codeql.yml
+++ b/.github/workflows/codeql.yml
@@ -43,8 +43,7 @@ jobs:
build-mode: none
steps:
- name: Enable S3 Cache for Self-Hosted Runners
- uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2
-
+ uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc # v2.2.0
- name: Checkout repository
uses: actions/checkout@v6
diff --git a/.github/workflows/cre-local-env-tests.yaml b/.github/workflows/cre-local-env-tests.yaml
index 5f9b54b0e09..043afbd4ae1 100644
--- a/.github/workflows/cre-local-env-tests.yaml
+++ b/.github/workflows/cre-local-env-tests.yaml
@@ -77,8 +77,7 @@ jobs:
ref: ${{ github.event_name == 'pull_request' && github.sha || inputs.chainlink_version }}
- name: Enable S3 Cache for Self-Hosted Runners
- uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2
-
+ uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc # v2.2.0
- name: Set up Go
id: setup-go
@@ -108,7 +107,7 @@ jobs:
# We need to login to ECR to allow the test to pull the Job Distributor and Chainlink images
- name: Configure AWS Credentials
- uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0
+ uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3
with:
aws-region: ${{ secrets.QA_AWS_REGION }}
role-to-assume: ${{ secrets.AWS_CTF_READ_ACCESS_ROLE_ARN }}
diff --git a/.github/workflows/cre-regression-system-tests.yaml b/.github/workflows/cre-regression-system-tests.yaml
index 25c2d46dd10..2f9528a18b3 100644
--- a/.github/workflows/cre-regression-system-tests.yaml
+++ b/.github/workflows/cre-regression-system-tests.yaml
@@ -91,7 +91,7 @@ jobs:
| map({
test_name: .value,
test_id: .key,
- runs_on: "runs-on=\($run_id)-\(.key)-\($run_attempt)/cpu=16/ram=64/family=m7i+m8i/spot=co/image=ubuntu24-full-x64/extras=s3-cache+tmpfs",
+ runs_on: "runs-on=\($run_id)-\(.key)-\($run_attempt)/cpu=8/ram=64/family=r6i+r7i+r8i/spot=co/image=ubuntu24-full-x64/extras=s3-cache+tmpfs",
configs: ($per[.value] // "configs/workflow-gateway-capabilities-don.toml")
})
')
@@ -99,6 +99,7 @@ jobs:
echo "matrix=$tests" | tee -a "${GITHUB_OUTPUT}"
run-system-tests:
+ name: ${{ matrix.tests.test_name }} ${{ matrix.tests.topology != '' && format(' ({0})', matrix.tests.topology) || '' }}
permissions:
contents: read
id-token: write
@@ -130,6 +131,7 @@ jobs:
BILLING_PLATFORM_SERVICE_IMAGE:
${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{
secrets.QA_AWS_REGION }}.amazonaws.com/billing-platform-service:v1.45.0
+ CHAINLINK_IMAGE_FULL: ${{ inputs.ecr == 'public' && format('public.ecr.aws/{0}:{1}', inputs.chainlink_image_repository_path, inputs.chainlink_image_tag) || format('{0}.dkr.ecr.{1}.amazonaws.com/{2}:{3}', secrets.QA_AWS_ACCOUNT_NUMBER, secrets.QA_AWS_REGION, inputs.chainlink_image_repository_path, inputs.chainlink_image_tag) }}
steps:
- name: Enable S3 Cache for Self-Hosted Runners
@@ -143,15 +145,23 @@ jobs:
- name: Set up Go
id: setup-go
- uses: actions/setup-go@v7
+ uses: smartcontractkit/.github/actions/ctf-setup-go@fa1d48a33e24f9b3b9f8c52e99a578a4597cb2a5 # v0.4.0
with:
- go-version-file: system-tests/tests/go.mod
- cache: true
+ go_mod_path: system-tests/tests/go.mod
+ cache_key_id: integration-tests-v1
+ cache_builds: true
+ cache_restore_only: "true"
+
+ - name: Restore Pre-Compiled Test Binaries from S3 Cache
+ uses: actions/cache/restore@v6
+ with:
+ path: system-tests/tests/bin/
+ key: test-binaries-cre-${{ hashFiles('system-tests/tests/go.sum', 'system-tests/tests/**/*.go', 'system-tests/lib/cre/**/*.go', 'core/scripts/cre/environment/**/*.go') }}
# Required to pull private ECR images such as Job Distributor (main) and Chip Ingress (main),
# and also the Chainlink image when inputs.ecr is "sdlc".
- name: Configure AWS Credentials
- uses: aws-actions/configure-aws-credentials@517a711dbcd0e402f90c77e7e2f81e849156e31d # v6.2.2
+ uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3
with:
aws-region: ${{ secrets.QA_AWS_REGION }}
role-to-assume: ${{ secrets.AWS_CTF_READ_ACCESS_ROLE_ARN }}
@@ -179,9 +189,16 @@ jobs:
- name: Set up gotestsum
shell: bash
run: |
- echo "::group::Install gotestsum"
- go install gotest.tools/gotestsum@v1.12.3
- echo "::endgroup::"
+ if [ -f "$GITHUB_WORKSPACE/system-tests/tests/bin/gotestsum" ]; then
+ echo "Using baked gotestsum from test binaries cache"
+ echo "$GITHUB_WORKSPACE/system-tests/tests/bin" >> $GITHUB_PATH
+ elif [ -f ~/go/bin/gotestsum ]; then
+ echo "Using cached gotestsum"
+ else
+ echo "::group::Install gotestsum"
+ go install gotest.tools/gotestsum@v1.13.0
+ echo "::endgroup::"
+ fi
- name: Setup Stellar CRE contracts
if: ${{ contains(matrix.tests.test_name, 'Stellar') }}
@@ -189,28 +206,13 @@ jobs:
with:
working-directory: system-tests/lib
- - name: Resolve Chainlink image
- id: resolve-chainlink-image
- working-directory: .github/scripts
- shell: bash
- env:
- ECR_TYPE: ${{ inputs.ecr }}
- CHAINLINK_IMAGE_REPO_PATH: ${{ inputs.chainlink_image_repository_path }}
- CHAINLINK_IMAGE_TAG: ${{ inputs.chainlink_image_tag }}
- AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
- AWS_REGION: ${{ secrets.QA_AWS_REGION }}
- run: |
- resolved_image="$(bash resolve-chainlink-image.sh)"
- echo "$resolved_image"
- echo "resolved_image=${resolved_image}" >> "${GITHUB_OUTPUT}"
-
- name: Start local CRE${{ matrix.tests.cre_version }}
id: start-local-cre
uses: ./.github/actions/start-local-cre-environment
with:
jd-image: "${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{
secrets.QA_AWS_REGION }}.amazonaws.com/job-distributor:0.28.0"
- chainlink-image: "${{ steps.resolve-chainlink-image.outputs.resolved_image }}"
+ chainlink-image: "${{ env.CHAINLINK_IMAGE_FULL }}"
chip-router-image: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{
secrets.QA_AWS_REGION }}.amazonaws.com/local-cre-chip-router:v1.0.1"
ctf-configs: ${{ matrix.tests.configs }}
@@ -228,17 +230,32 @@ jobs:
TEST_NAME: ${{ matrix.tests.test_name }}
TEST_TIMEOUT: 7m # let's leave 3 minutes for other steps (the whole job times out after 10 minutes)
CRE_TEST_PARALLEL_ENABLED: "true"
+ PARALLEL_COUNT: "10"
run: |
echo "Starting test: '${TEST_NAME}'"
echo "⚠️⚠️⚠️ Add 'skip-e2e-regression' label to skip this step if necessary ⚠️⚠️⚠️"
- gotestsum \
- --jsonfile=/tmp/gotest-regression.log \
- --junitfile=/tmp/junit-report-regression.xml \
- --format=github-actions \
- -- \
- -v -run "^(${TEST_NAME})$" -timeout "${TEST_TIMEOUT}" -count=1 \
- github.com/smartcontractkit/chainlink/system-tests/tests/regression/cre
+ if [ -f "./bin/cre-regression.test" ]; then
+ echo "Using precompiled binary"
+ (
+ cd regression/cre && \
+ gotestsum \
+ --jsonfile=/tmp/gotest-regression.log \
+ --junitfile=/tmp/junit-report-regression.xml \
+ --format=github-actions \
+ --raw-command -- \
+ go tool test2json -t -p github.com/smartcontractkit/chainlink/system-tests/tests/regression/cre ../../bin/cre-regression.test -test.v -test.run "^(${TEST_NAME})$" -test.timeout "${TEST_TIMEOUT}" -test.count=1 -test.parallel="${PARALLEL_COUNT}"
+ )
+ else
+ echo "No precompiled binary found. Building from scratch..."
+ gotestsum \
+ --jsonfile=/tmp/gotest-regression.log \
+ --junitfile=/tmp/junit-report-regression.xml \
+ --format=github-actions \
+ -- \
+ -v -run "^(${TEST_NAME})$" -timeout "${TEST_TIMEOUT}" -count=1 -parallel="${PARALLEL_COUNT}" \
+ github.com/smartcontractkit/chainlink/system-tests/tests/regression/cre
+ fi
exit_code="$?"
if [ "$exit_code" -eq 0 ]; then
diff --git a/.github/workflows/cre-soak-memory-leak.yml b/.github/workflows/cre-soak-memory-leak.yml
index cd48352f093..c7758a63954 100644
--- a/.github/workflows/cre-soak-memory-leak.yml
+++ b/.github/workflows/cre-soak-memory-leak.yml
@@ -37,7 +37,7 @@ jobs:
steps:
- name: Enable S3 Cache for Self-Hosted Runners
- uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2
+ uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc # v2.2.0
- name: Checkout
uses: actions/checkout@v6
@@ -53,7 +53,7 @@ jobs:
cache: true
- name: Configure AWS Credentials
- uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0
+ uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3
with:
aws-region: ${{ secrets.QA_AWS_REGION }}
role-to-assume: ${{ secrets.AWS_CTF_READ_ACCESS_ROLE_ARN }}
@@ -96,7 +96,8 @@ jobs:
- name: Start local CRE
uses: ./.github/actions/start-local-cre-environment
with:
- jd-image: "${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION
+ jd-image:
+ "${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION
}}.amazonaws.com/job-distributor:0.28.0"
chainlink-image: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{
secrets.QA_AWS_REGION }}.amazonaws.com/${{ inputs.ecr_name ||
@@ -120,7 +121,8 @@ jobs:
working-directory: system-tests/tests
env:
CRE_SOAK_DURATION: "2h"
- CTF_CHIP_INGRESS_IMAGE: "${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{
+ CTF_CHIP_INGRESS_IMAGE:
+ "${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{
secrets.QA_AWS_REGION
}}.amazonaws.com/atlas-chip-ingress:da84cb72d3a160e02896247d46ab4b9\
806ebee2f"
@@ -159,7 +161,7 @@ jobs:
name: Notify about test Failure
#if: failure()
if: false # TODO: Silence for now
- needs: [ soak ]
+ needs: [soak]
environment:
name: integration
deployment: false
diff --git a/.github/workflows/cre-system-tests.yaml b/.github/workflows/cre-system-tests.yaml
index d1f110c667a..a7d9cd8e7e1 100644
--- a/.github/workflows/cre-system-tests.yaml
+++ b/.github/workflows/cre-system-tests.yaml
@@ -115,6 +115,8 @@ jobs:
test_names=$(grep -rh -oP '^func \K(Test|Example)[^(]+' system-tests/tests/smoke/cre/*_test.go | grep -v Test_Upgrade)
tests=$(echo "$test_names" | jq -c -R -s \
+ --argjson run_id "${{ github.run_id }}" \
+ --arg run_attempt "${{ github.run_attempt }}" \
--argjson tops "$TOPOLOGIES_JSON" \
--argjson per "$PER_TEST_TOPOLOGIES_JSON" '
(split("\n") | map(select(length>0))) as $names
@@ -125,12 +127,16 @@ jobs:
] as $extra
| ($base + $extra)
| to_entries
- | map(.value + {test_id: .key})
+ | map(.value + {
+ test_id: .key,
+ runs_on: "runs-on=\($run_id)-\(.key)-\($run_attempt)/cpu=8/ram=64/family=r6i+r7i+r8i/spot=co/image=ubuntu24-full-x64/extras=s3-cache+tmpfs"
+ })
')
echo "matrix=$tests" | tee -a "${GITHUB_OUTPUT}"
run-system-tests:
+ name: ${{ matrix.tests.test_name }} ${{ matrix.tests.topology != '' && format(' ({0})', matrix.tests.topology) || '' }}
permissions:
contents: read
id-token: write
@@ -141,9 +147,7 @@ jobs:
needs: [define-test-matrix]
# we need a `test_id` and `run_attempt` here to stop runner stealing
# see: https://runs-on.com/guides/troubleshoot/#runner-stealing-and-matrix-jobs
- runs-on: runs-on=${{ github.run_id }}-${{ matrix.tests.test_id }}-${{
- github.run_attempt
- }}/cpu=16/ram=64/family=m7i+m8i/spot=co/image=ubuntu24-full-x64/extras=s3-cache+tmpfs
+ runs-on: ${{ matrix.tests.runs_on }}
environment:
# http://docs.github.com/en/actions/how-tos/deploy/configure-and-manage-deployments/control-deployments#using-environments-without-deployments
name: integration
@@ -154,6 +158,9 @@ jobs:
BILLING_PLATFORM_SERVICE_IMAGE:
${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{
secrets.QA_AWS_REGION }}.amazonaws.com/billing-platform-service:v1.57.1
+ CHAINLINK_IMAGE_FULL: ${{ inputs.ecr == 'public' && format('public.ecr.aws/{0}:{1}', inputs.chainlink_image_repository_path, inputs.chainlink_image_tag) || format('{0}.dkr.ecr.{1}.amazonaws.com/{2}:{3}', secrets.QA_AWS_ACCOUNT_NUMBER, secrets.QA_AWS_REGION, inputs.chainlink_image_repository_path, inputs.chainlink_image_tag) }}
+ CTF_CHIP_INGRESS_IMAGE: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/atlas-chip-ingress:da84cb72d3a160e02896247d46ab4b9806ebee2f
+ CTF_CHIP_CONFIG_IMAGE: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/atlas-chip-config:7b4e9ee68fd1c737dd3480b5a3ced0188f29b969
steps:
- name: Enable S3 Cache for Self-Hosted Runners
@@ -167,14 +174,22 @@ jobs:
- name: Set up Go
id: setup-go
- uses: actions/setup-go@v7
+ uses: smartcontractkit/.github/actions/ctf-setup-go@fa1d48a33e24f9b3b9f8c52e99a578a4597cb2a5 # v0.4.0
with:
- go-version-file: system-tests/tests/go.mod
- cache: true
+ go_mod_path: system-tests/tests/go.mod
+ cache_key_id: integration-tests-v1
+ cache_builds: true
+ cache_restore_only: "true"
+
+ - name: Restore Pre-Compiled Test Binaries from S3 Cache
+ uses: actions/cache/restore@v6
+ with:
+ path: system-tests/tests/bin/
+ key: test-binaries-cre-${{ hashFiles('system-tests/tests/go.sum', 'system-tests/tests/**/*.go', 'system-tests/lib/cre/**/*.go', 'core/scripts/cre/environment/**/*.go') }}
# Required to pull Job Distributor (main), Chip Ingress (main) and Chainlink (sdlc) private images
- name: Configure AWS Credentials
- uses: aws-actions/configure-aws-credentials@517a711dbcd0e402f90c77e7e2f81e849156e31d # v6.2.2
+ uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3
with:
aws-region: ${{ secrets.QA_AWS_REGION }}
role-to-assume: ${{ secrets.AWS_CTF_READ_ACCESS_ROLE_ARN }}
@@ -202,9 +217,16 @@ jobs:
- name: Set up gotestsum
shell: bash
run: |
- echo "::group::Install gotestsum"
- go install gotest.tools/gotestsum@v1.13.0
- echo "::endgroup::"
+ if [ -f "$GITHUB_WORKSPACE/system-tests/tests/bin/gotestsum" ]; then
+ echo "Using baked gotestsum from test binaries cache"
+ echo "$GITHUB_WORKSPACE/system-tests/tests/bin" >> $GITHUB_PATH
+ elif [ -f ~/go/bin/gotestsum ]; then
+ echo "Using cached gotestsum"
+ else
+ echo "::group::Install gotestsum"
+ go install gotest.tools/gotestsum@v1.13.0
+ echo "::endgroup::"
+ fi
- name: Install Aptos CLI
if: ${{ matrix.tests.test_name == 'Test_CRE_V2_Aptos_Suite' }}
@@ -218,21 +240,6 @@ jobs:
with:
working-directory: system-tests/lib
- - name: Resolve Chainlink image
- id: resolve-chainlink-image
- working-directory: .github/scripts
- shell: bash
- env:
- CHAINLINK_IMAGE_REPO_PATH: ${{ inputs.chainlink_image_repository_path }}
- CHAINLINK_IMAGE_TAG: ${{ inputs.chainlink_image_tag }}
- ECR_TYPE: ${{ inputs.ecr }}
- AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
- AWS_REGION: ${{ secrets.QA_AWS_REGION }}
- run: |
- resolved_image="$(bash resolve-chainlink-image.sh)"
- echo "$resolved_image"
- echo "resolved_image=${resolved_image}" >> "${GITHUB_OUTPUT}"
-
- name: Start observability stack (Beholder suite)
if: contains(matrix.tests.test_name, 'Beholder_Suite')
shell: bash
@@ -241,12 +248,6 @@ jobs:
TEST_NAME: ${{ matrix.tests.test_name }}
OBS_MAX_ATTEMPTS: "3"
OBS_RETRY_DELAY_SECONDS: "15"
- CTF_CHIP_INGRESS_IMAGE: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{
- secrets.QA_AWS_REGION
- }}.amazonaws.com/atlas-chip-ingress:da84cb72d3a160e02896247d46ab4b9806ebee2f
- CTF_CHIP_CONFIG_IMAGE: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{
- secrets.QA_AWS_REGION
- }}.amazonaws.com/atlas-chip-config:7b4e9ee68fd1c737dd3480b5a3ced0188f29b969
run: |
set -u
set +e
@@ -323,7 +324,7 @@ jobs:
jd-image:
"${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION
}}.amazonaws.com/job-distributor:0.28.0"
- chainlink-image: "${{ steps.resolve-chainlink-image.outputs.resolved_image }}"
+ chainlink-image: "${{ env.CHAINLINK_IMAGE_FULL }}"
chip-router-image: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{
secrets.QA_AWS_REGION }}.amazonaws.com/local-cre-chip-router:v1.0.1"
ctf-configs: ${{ matrix.tests.configs }}
@@ -355,21 +356,29 @@ jobs:
GITHUB_TOKEN: ${{ steps.github-token.outputs.access-token || '' }} # to avoid rate limiting when downloading protobuf files from GitHub
PARALLEL_COUNT: "10"
CRE_TEST_PARALLEL_ENABLED: "true"
- CTF_CHIP_INGRESS_IMAGE: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{
- secrets.QA_AWS_REGION
- }}.amazonaws.com/atlas-chip-ingress:da84cb72d3a160e02896247d46ab4b9806ebee2f
- CTF_CHIP_CONFIG_IMAGE: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{
- secrets.QA_AWS_REGION
- }}.amazonaws.com/atlas-chip-config:7b4e9ee68fd1c737dd3480b5a3ced0188f29b969
run: |
echo "Starting test: '${TEST_NAME}'"
- gotestsum \
- --jsonfile=/tmp/gotest.log \
- --junitfile=/tmp/junit-report.xml \
- --format=github-actions \
- -- \
- -v -run "^(${TEST_NAME})$" -timeout "${TEST_TIMEOUT}" -count=1 -parallel="${PARALLEL_COUNT}" \
- github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre
+ if [ -f "./bin/cre-smoke.test" ]; then
+ echo "Using precompiled binary"
+ (
+ cd smoke/cre && \
+ gotestsum \
+ --jsonfile=/tmp/gotest.log \
+ --junitfile=/tmp/junit-report.xml \
+ --format=github-actions \
+ --raw-command -- \
+ go tool test2json -t -p github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre ../../bin/cre-smoke.test -test.v -test.run "^(${TEST_NAME})$" -test.timeout "${TEST_TIMEOUT}" -test.count=1 -test.parallel="${PARALLEL_COUNT}"
+ )
+ else
+ echo "No precompiled binary found. Building from scratch..."
+ gotestsum \
+ --jsonfile=/tmp/gotest.log \
+ --junitfile=/tmp/junit-report.xml \
+ --format=github-actions \
+ -- \
+ -v -run "^(${TEST_NAME})$" -timeout "${TEST_TIMEOUT}" -count=1 -parallel="${PARALLEL_COUNT}" \
+ github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre
+ fi
exit_code="$?"
if [ "$exit_code" -eq 0 ]; then
diff --git a/.github/workflows/cre-wf-caching-test.yml b/.github/workflows/cre-wf-caching-test.yml
index 920d3a48568..5c1aa52363a 100644
--- a/.github/workflows/cre-wf-caching-test.yml
+++ b/.github/workflows/cre-wf-caching-test.yml
@@ -37,7 +37,7 @@ jobs:
steps:
- name: Enable S3 Cache for Self-Hosted Runners
- uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2
+ uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc # v2.2.0
with:
metrics: cpu,network,memory,disk,io
@@ -55,7 +55,7 @@ jobs:
cache: true
- name: Configure AWS Credentials
- uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0
+ uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3
with:
aws-region: ${{ secrets.QA_AWS_REGION }}
role-to-assume: ${{ secrets.AWS_CTF_READ_ACCESS_ROLE_ARN }}
diff --git a/.github/workflows/cre-workflow-don-benchmark.yaml b/.github/workflows/cre-workflow-don-benchmark.yaml
index 9a2fa288d45..422f82d6a80 100644
--- a/.github/workflows/cre-workflow-don-benchmark.yaml
+++ b/.github/workflows/cre-workflow-don-benchmark.yaml
@@ -51,7 +51,7 @@ jobs:
# We need to login to ECR to allow the test to pull the Job Distributor and Chainlink images
- name: Configure AWS Credentials
- uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0
+ uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3
with:
aws-region: ${{ secrets.QA_AWS_REGION }}
role-to-assume: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
diff --git a/.github/workflows/devenv-compat.yml b/.github/workflows/devenv-compat.yml
index 8e5237cbd68..acd045be3f7 100644
--- a/.github/workflows/devenv-compat.yml
+++ b/.github/workflows/devenv-compat.yml
@@ -17,7 +17,8 @@ on:
type: boolean
default: false
refs:
- description: "Git refs to test, should be used only for tool testing purposes.
+ description:
+ "Git refs to test, should be used only for tool testing purposes.
In all other cases refs are detected automatically either from Git or
RANE SOT"
required: false
@@ -59,7 +60,8 @@ on:
default: "2"
type: string
tag-version-ceiling:
- description: "Version ceiling for compatibility testing (tags newer than this
+ description:
+ "Version ceiling for compatibility testing (tags newer than this
version will be excluded)"
required: false
type: string
@@ -128,7 +130,7 @@ jobs:
# We need to login to ECR to allow the test to pull the Job Distributor (main) and Chainlink (sdlc) images
- name: Configure AWS Credentials
- uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0
+ uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3
with:
aws-region: ${{ secrets.QA_AWS_REGION }}
role-to-assume: ${{ secrets.AWS_CTF_READ_ACCESS_ROLE_ARN }}
diff --git a/.github/workflows/find-flaky-tests.yml b/.github/workflows/find-flaky-tests.yml
deleted file mode 100644
index 064578a8b97..00000000000
--- a/.github/workflows/find-flaky-tests.yml
+++ /dev/null
@@ -1,53 +0,0 @@
-name: Find Flaky Tests
-
-on:
- workflow_dispatch:
- inputs:
- dir:
- description: The directory to find flaky tests in (core/services/...)
- required: true
- type: string
- default: "./core/..."
- iterations:
- description: The number of iterations to run the tests for
- required: true
- type: number
- default: 20 # Should take 3.3 hours max
- parallel_iterations:
- description: The number of iterations to run in parallel
- required: false
- type: number
- default: 1
-
-permissions:
- contents: read
-
-env:
- ITERATIONS: ${{ inputs.iterations || 20 }}
- PARALLEL_ITERATIONS: ${{ inputs.parallel_iterations || 1 }}
- DIR: ${{ inputs.dir || './core/...' }}
-
-jobs:
- find-flaky-tests:
- name: Find Flaky Tests
- runs-on: runs-on=${{ github.run_id
- }}-diagnose/cpu=48/ram=96/family=c6id+c5ad/spot=false/image=ubuntu24-full-x64/extras=s3-cache
- timeout-minutes: 270 # 4.5 hours
- steps:
- - name: Checkout
- uses: actions/checkout@v6
- with:
- persist-credentials: false
- repository: smartcontractkit/chainlink
- ref: ${{ github.sha }}
- - name: Enable S3 Cache for Self-Hosted Runners
- uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2
- - name: Run `diagnose`
- run: |
- go -C tools/test run . diagnose --iterations "${ITERATIONS}" --parallel-iterations "${PARALLEL_ITERATIONS}" -- "${DIR}"
- - name: Upload Results
- uses: actions/upload-artifact@v7
- with:
- name: flaky-tests-results
- path: ./diagnose-*
- if-no-files-found: error
diff --git a/.github/workflows/go-mod-cache.yml b/.github/workflows/go-mod-cache.yml
index fb21e86ff66..e6147e5bcea 100644
--- a/.github/workflows/go-mod-cache.yml
+++ b/.github/workflows/go-mod-cache.yml
@@ -53,7 +53,7 @@ jobs:
pull-requests: read
steps:
- name: Enable S3 Cache for Self-Hosted Runners
- uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2
+ uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc # v2.2.0
- name: Checkout the repo
uses: actions/checkout@v6
diff --git a/.github/workflows/integration-in-memory-tests.yml b/.github/workflows/integration-in-memory-tests.yml
index 09208f367f1..c99d459ee55 100644
--- a/.github/workflows/integration-in-memory-tests.yml
+++ b/.github/workflows/integration-in-memory-tests.yml
@@ -101,7 +101,7 @@ jobs:
contents: read
needs: changes
if: github.event_name == 'pull_request' && needs.changes.outputs.run-tests == 'true'
- uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@9c49ffcf252efbedd7ec280e1993ba29dcbc9443 # 2026-04-28
+ uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@run-e2e-tests/v1
with:
workflow_name: Run CCIP Integration Tests For PR
chainlink_version: ${{ inputs.cl_ref || github.sha }}
@@ -133,7 +133,7 @@ jobs:
contents: read
needs: changes
if: github.event_name == 'merge_group' && needs.changes.outputs.run-tests == 'true'
- uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@9c49ffcf252efbedd7ec280e1993ba29dcbc9443 # 2026-04-28
+ uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@run-e2e-tests/v1
with:
workflow_name: Run CCIP Integration Tests For Merge Queue
chainlink_version: ${{ inputs.cl_ref || github.sha }}
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 2fcbf6016c2..c9a24a87182 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -94,6 +94,7 @@ jobs:
general-changes: ${{ steps.changes.outputs.general_changes }}
core-changes: ${{ steps.changes.outputs.core_changes }}
cre-changes: ${{ steps.changes.outputs.cre_changes }}
+ ccip-changes: ${{ steps.changes.outputs.ccip_changes }}
steps:
- name: Checkout the repo
uses: actions/checkout@v7
@@ -108,9 +109,9 @@ jobs:
filters: |
general_changes:
- '.github/workflows/integration-tests.yml'
- - '.github/workflows/run-e2e-tests-reusable-workflow.yml'
- '.github/workflows/cre-system-tests.yaml'
- - '.github/e2e-tests.yml'
+ - '.github/workflows/cre-regression-system-tests.yaml'
+ - '.github/workflows/ccip-system-tests.yaml'
- 'GNUmakefile'
- 'core/chainlink.Dockerfile'
- 'plugins/chainlink.Dockerfile'
@@ -128,6 +129,12 @@ jobs:
- 'system-tests/**'
- 'plugins/plugins.private.yaml'
- 'plugins/plugins.public.yaml'
+ ccip_changes:
+ - '.github/workflows/integration-tests.yml'
+ - '.github/workflows/ccip-system-tests.yaml'
+ - '.github/actions/**'
+ - 'integration-tests/**'
+ - 'core/capabilities/ccip/**'
- name: Decide which tests to run (rollout-only)
# To validate that this properly tests, we will run this beside the dorny/paths-filter actions
@@ -148,6 +155,7 @@ jobs:
- "core/**/config/**/*.toml"
workflow-files:
- ".github/workflows/integration-tests.yml"
+ - ".github/workflows/ccip-system-tests.yaml"
- ".github/actions/**/*.y*ml"
docker-files:
- "**/*Dockerfile"
@@ -155,7 +163,6 @@ jobs:
- "plugins/plugins.private.yaml"
legacy-integ-test-files:
- "integration-tests/**"
- - ".github/e2e-tests.yml"
system-test-files:
- "system-tests/**"
- "core/scripts/cre/**"
@@ -190,6 +197,7 @@ jobs:
- "!deployment/**"
- "**/*ccip*"
- "**/*ccip*/**"
+ - ".github/workflows/ccip-system-tests.yaml"
labels:
name: Get PR labels and set runner labels
@@ -206,6 +214,7 @@ jobs:
skip-e2e-regression-label-found: ${{ steps.label-skip-e2e-regression.outputs.check-label-found || 'false' }}
steps:
- name: Get PR Labels (runs-on-opt-out)
+ if: github.event_name == 'pull_request'
id: label-runs-on-opt-out
uses: smartcontractkit/.github/actions/get-pr-labels@get-pr-labels/v1
with:
@@ -231,8 +240,8 @@ jobs:
OPT_OUT: ${{ steps.label-runs-on-opt-out.outputs.check-label-found || 'false' }}
GH_BUILDER_RUNNER: ubuntu22.04-8cores-32GB
# include unique label (core/plugins) to ensure jobs are not competing for the same runner
- SH_BUILDER_RUNNER_CORE: runs-on=${{ github.run_id }}-core/cpu=16/memory=32/family=c7i+c8i/spot=co/extras=s3-cache+tmpfs
- SH_BUILDER_RUNNER_PLUGINS: runs-on=${{ github.run_id }}-plugins/cpu=16/memory=32/family=c7i+c8i/spot=co/extras=s3-cache+tmpfs
+ SH_BUILDER_RUNNER_CORE: runs-on=${{ github.run_id }}-core/cpu=32/memory=64/family=c7i+c8i+c6i+m7i/spot=co/extras=s3-cache+tmpfs
+ SH_BUILDER_RUNNER_PLUGINS: runs-on=${{ github.run_id }}-plugins/cpu=32/memory=64/family=c7i+c8i+c6i+m7i/spot=co/extras=s3-cache+tmpfs
run: |
if [[ "${OPT_OUT}" == "true" ]]; then
echo "builder-runner-label=${GH_BUILDER_RUNNER}" | tee -a "$GITHUB_OUTPUT"
@@ -249,13 +258,7 @@ jobs:
name: integration
deployment: false
runs-on: ${{ matrix.image.runner }}
- needs:
- [
- labels,
- enforce-ctf-version,
- run-core-cre-e2e-tests-setup,
- run-ccip-v1-6-e2e-tests-setup,
- ]
+ needs: [labels, enforce-ctf-version, changes]
permissions:
id-token: write
contents: read
@@ -267,11 +270,14 @@ jobs:
dockerfile: core/chainlink.Dockerfile
tag-suffix: ""
cache-scope: core
- # todo: optimize this conditional
any-should-run: >-
${{
- needs.run-ccip-v1-6-e2e-tests-setup.outputs.should-run == 'true' ||
- needs.run-core-cre-e2e-tests-setup.outputs.should-run == 'true'
+ github.event_name == 'workflow_dispatch' ||
+ needs.changes.outputs.general-changes == 'true' ||
+ needs.changes.outputs.core-changes == 'true' ||
+ needs.changes.outputs.cre-changes == 'true' ||
+ needs.changes.outputs.ccip-changes == 'true' ||
+ needs.labels.outputs.run-e2e-tests-label-found == 'true'
}}
- name: (plugins)
@@ -279,15 +285,17 @@ jobs:
dockerfile: plugins/chainlink.Dockerfile
tag-suffix: -plugins
cache-scope: plugins
- # todo: optimize this conditional
any-should-run: >-
${{
- needs.run-ccip-v1-6-e2e-tests-setup.outputs.should-run == 'true' ||
- needs.run-core-cre-e2e-tests-setup.outputs.should-run == 'true'
+ github.event_name == 'workflow_dispatch' ||
+ needs.changes.outputs.general-changes == 'true' ||
+ needs.changes.outputs.core-changes == 'true' ||
+ needs.changes.outputs.cre-changes == 'true' ||
+ needs.labels.outputs.run-e2e-tests-label-found == 'true'
}}
steps:
- name: Enable S3 Cache for Self-Hosted Runners
- uses: runs-on/action@bdccf4a8c118feaa74578d6fb51012875a860567 # v2.2.0
+ uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc # v2.2.0
- name: Check if image exists in ECR
id: check-image-exists
@@ -339,6 +347,7 @@ jobs:
aws-account-number: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
aws-region: ${{ secrets.QA_AWS_REGION }}
aws-role-arn: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
+ free-disk-space: "false"
docker-additional-build-args: |
CL_IS_PROD_BUILD=false
# go-get-overrides is empty when inputs.evm-ref is unset so ctf-build-image skips an unnecessary setup-go / go mod tidy.
@@ -423,7 +432,7 @@ jobs:
run-core-cre-e2e-tests:
name: Run Core CRE E2E Tests
- needs: [build-chainlink, run-core-cre-e2e-tests-setup]
+ needs: [build-chainlink, run-core-cre-e2e-tests-setup, compile-tests]
permissions:
actions: read
checks: write
@@ -441,7 +450,7 @@ jobs:
run-core-cre-e2e-regression-tests:
name: Run Core CRE E2E Regression Tests
- needs: [build-chainlink, run-core-cre-e2e-tests-setup]
+ needs: [build-chainlink, run-core-cre-e2e-tests-setup, compile-tests]
permissions:
actions: read
checks: write
@@ -465,56 +474,193 @@ jobs:
contents: read
outputs:
should-run: ${{ steps.form-inputs.outputs.should-run }}
- workflow-name: ${{ steps.form-inputs.outputs.workflow-name }}
- test-trigger: ${{ steps.form-inputs.outputs.test-trigger }}
steps:
- name: Form Inputs for CCIP v1.6 E2E Tests
id: form-inputs
env:
GITHUB_EVENT_NAME: ${{ github.event_name }}
GITHUB_REF_TYPE: ${{ github.ref_type }}
- CONTAINS_CHANGES: ${{ needs.changes.outputs.general-changes == 'true' || needs.changes.outputs.core-changes == 'true' }}
+ CONTAINS_CHANGES: ${{ needs.changes.outputs.ccip-changes == 'true' }}
RUN_E2E_TESTS_LABEL_FOUND: ${{ needs.labels.outputs.run-e2e-tests-label-found || 'false' }}
run: |
if [[ "${GITHUB_EVENT_NAME}" == 'pull_request' ]]; then
- # Run CCIP v1.6 E2E Tests on PRs only if the label "run-e2e-tests" is present, and there are relevant changes
- echo "workflow-name=Run CCIP v1.6 E2E Tests For PR" | tee -a "$GITHUB_OUTPUT"
- echo "test-trigger=PR E2E CCIP v1.6 Tests" | tee -a "$GITHUB_OUTPUT"
-
- if [[ "${RUN_E2E_TESTS_LABEL_FOUND}" == 'true' ]]; then
- # only run if the PR has the label "run-e2e-tests"
- echo "should-run=true" | tee -a "$GITHUB_OUTPUT"
- fi
+ # CCIP v1.6 E2E Tests run on merge queue (merge_group), not on PRs
+ echo "should-run=false" | tee -a "$GITHUB_OUTPUT"
elif [[ "${GITHUB_EVENT_NAME}" == 'merge_group' ]]; then
# Run CCIP v1.6 E2E Tests in the merge queue, if there are relevant changes
- echo "workflow-name=Run CCIP v1.6 E2E Tests For Merge Queue" | tee -a "$GITHUB_OUTPUT"
- echo "test-trigger=Merge Queue E2E CCIP v1.6 Tests" | tee -a "$GITHUB_OUTPUT"
echo "should-run=${CONTAINS_CHANGES}" | tee -a "$GITHUB_OUTPUT"
elif [[ "${GITHUB_EVENT_NAME}" == 'workflow_dispatch' ]]; then
# Always Run CCIP v1.6 E2E Tests on workflow dispatch
- echo "workflow-name=Run CCIP v1.6 E2E Tests For Workflow Dispatch" | tee -a "$GITHUB_OUTPUT"
- echo "test-trigger=Workflow Dispatch E2E CCIP v1.6 Tests" | tee -a "$GITHUB_OUTPUT"
echo "should-run=true" | tee -a "$GITHUB_OUTPUT"
elif [[ "${GITHUB_EVENT_NAME}" == 'push' ]]; then
# Run CCIP v1.6 E2E Tests on push events, only if there are relevant changes or if it's a tag push
- echo "workflow-name=Run CCIP v1.6 E2E Tests For Push" | tee -a "$GITHUB_OUTPUT"
- echo "test-trigger=Push E2E CCIP v1.6 Tests" | tee -a "$GITHUB_OUTPUT"
if [[ "${CONTAINS_CHANGES}" == 'true' || "${GITHUB_REF_TYPE}" == 'tag' ]]; then
echo "should-run=true" | tee -a "$GITHUB_OUTPUT"
+ else
+ echo "should-run=false" | tee -a "$GITHUB_OUTPUT"
fi
else
- echo "workflow-name=Run CCIP v1.6 E2E Tests For Unknown Event" | tee -a "$GITHUB_OUTPUT"
- echo "test-trigger=Unknown CCIP v1.6 E2E Tests" | tee -a "$GITHUB_OUTPUT"
echo "should-run=false" | tee -a "$GITHUB_OUTPUT"
fi
+ # Central writer for the unified `integration-tests-v1` build cache consumed (restore-only) by
+ # CRE smoke + regression matrices and CCIP test suites.
+ # Keyed on go.sum hash to optimize parallel PR cache hits (Strategy A + B + C).
+ compile-tests:
+ name: Compile CRE & CCIP Tests
+ if: >-
+ ${{
+ github.actor != 'dependabot[bot]' &&
+ (needs.run-core-cre-e2e-tests-setup.outputs.should-run == 'true' ||
+ needs.run-ccip-v1-6-e2e-tests-setup.outputs.should-run == 'true')
+ }}
+ needs:
+ - run-core-cre-e2e-tests-setup
+ - run-ccip-v1-6-e2e-tests-setup
+ runs-on: runs-on=${{ github.run_id }}-compile/cpu=32/ram=64/family=c6i+c7i+c8i/spot=co/volume=100GB/extras=s3-cache
+ environment:
+ name: integration
+ deployment: false
+ permissions:
+ id-token: write
+ contents: read
+ steps:
+ - name: Enable S3 Cache for Self-Hosted Runners
+ uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc # v2.2.0
+
+ - name: Setup GitHub token using GATI
+ id: github-token
+ uses: smartcontractkit/.github/actions/setup-github-token@setup-github-token/v1
+ with:
+ aws-role-arn: ${{ secrets.AWS_OIDC_CHAINLINK_READ_ONLY_TOKEN_ISSUER_ROLE_ARN }}
+ aws-lambda-url: ${{ secrets.AWS_INFRA_RELENG_TOKEN_ISSUER_LAMBDA_URL }}
+ aws-region: us-west-2
+ aws-role-duration-seconds: "1800"
+ set-git-config: "true"
+
+ - name: Checkout the repo
+ uses: actions/checkout@v7
+ with:
+ persist-credentials: false
+ repository: smartcontractkit/chainlink
+ ref: ${{ env.CHAINLINK_REF }}
+
+ - name: Set up Go for compilation
+ id: setup-go
+ uses: smartcontractkit/.github/actions/ctf-setup-go@fa1d48a33e24f9b3b9f8c52e99a578a4597cb2a5 # v0.4.0
+ with:
+ # Key the cache on go.sum hash; restore-only on PRs to eliminate upload overhead and cache thrashing.
+ go_mod_path: ./system-tests/tests/go.mod
+ cache_key_id: integration-tests-v1
+ cache_builds: true
+ cache_restore_only: ${{ github.ref_name != 'develop' }}
+ should_tidy: false
+ no_cache: false
+ gati_token: ${{ steps.github-token.outputs.access-token }}
+
+ - name: Restore Pre-Compiled CRE Test Binaries
+ if: needs.run-core-cre-e2e-tests-setup.outputs.should-run == 'true'
+ uses: actions/cache/restore@v6
+ id: cache-cre-binaries
+ with:
+ path: system-tests/tests/bin/
+ key: test-binaries-cre-${{ hashFiles('system-tests/tests/go.sum', 'system-tests/tests/**/*.go', 'system-tests/lib/cre/**/*.go', 'core/scripts/cre/environment/**/*.go') }}
+
+ # No cache-hit short-circuit: ctf-setup-go@v0.4.0 exposes no `cache-hit` output,
+ # and this job's purpose is to (re)populate the shared build cache regardless.
+ - name: Compile E2E tests
+ shell: bash
+ env:
+ SHOULD_RUN_CRE: ${{ needs.run-core-cre-e2e-tests-setup.outputs.should-run }}
+ SHOULD_RUN_CCIP: ${{ needs.run-ccip-v1-6-e2e-tests-setup.outputs.should-run }}
+ run: |
+ set -euo pipefail
+ mkdir -p "$GITHUB_WORKSPACE/.gotmp"
+ export GOTMPDIR="$GITHUB_WORKSPACE/.gotmp"
+
+ mkdir -p "$GITHUB_WORKSPACE/system-tests/tests/bin"
+ mkdir -p "$GITHUB_WORKSPACE/integration-tests/bin"
+
+ cre_pid=""
+ ccip_pid=""
+
+ if [[ "${SHOULD_RUN_CRE}" == "true" ]]; then
+ if [[ ! -f "$GITHUB_WORKSPACE/system-tests/tests/bin/cre-smoke.test" || ! -f "$GITHUB_WORKSPACE/system-tests/tests/bin/cre-regression.test" || ! -f "$GITHUB_WORKSPACE/system-tests/tests/bin/cre-env" ]]; then
+ (
+ set -euo pipefail
+ echo "Compiling CRE tests and CLI in parallel..."
+ cd "$GITHUB_WORKSPACE/system-tests/tests"
+ go mod download
+ go test -c -ldflags="-s -w" -o ./bin/cre-smoke.test ./smoke/cre &
+ p1=$!
+ go test -c -ldflags="-s -w" -o ./bin/cre-regression.test ./regression/cre &
+ p2=$!
+ (cd "$GITHUB_WORKSPACE/core/scripts/cre/environment" && go build -ldflags="-s -w" -o "$GITHUB_WORKSPACE/system-tests/tests/bin/cre-env" .) &
+ p3=$!
+ set +e
+ wait "$p1"; p1_exit=$?
+ wait "$p2"; p2_exit=$?
+ wait "$p3"; p3_exit=$?
+ set -e
+ if [[ "$p1_exit" -ne 0 || "$p2_exit" -ne 0 || "$p3_exit" -ne 0 ]]; then
+ echo "Compilation failed (cre-smoke=$p1_exit, cre-regression=$p2_exit, cre-env=$p3_exit)" >&2
+ exit 1
+ fi
+ ) &
+ cre_pid=$!
+ else
+ echo "CRE test binaries already exist, skipping compilation..."
+ fi
+ fi
+
+ if [[ "${SHOULD_RUN_CCIP}" == "true" ]]; then
+ (
+ set -euo pipefail
+ echo "Compiling CCIP tests in parallel..."
+ cd "$GITHUB_WORKSPACE/integration-tests"
+ go mod download
+ go test -c -tags embed -ldflags="-s -w" -o ./bin/ccip-smoke.test ./smoke/ccip
+ ) &
+ ccip_pid=$!
+ fi
+
+ if [ -n "$cre_pid" ]; then wait "$cre_pid"; fi
+ if [ -n "$ccip_pid" ]; then wait "$ccip_pid"; fi
+
+ echo "Installing gotestsum..."
+ go install gotest.tools/gotestsum@v1.13.0
+ cp "$(go env GOPATH)/bin/gotestsum" "$GITHUB_WORKSPACE/system-tests/tests/bin/"
+
+ rm -rf "$GITHUB_WORKSPACE/.gotmp"
+
+ - name: Cache Pre-Compiled CRE Test Binaries
+ if: needs.run-core-cre-e2e-tests-setup.outputs.should-run == 'true'
+ uses: actions/cache/save@v6
+ with:
+ path: system-tests/tests/bin/
+ key: test-binaries-cre-${{ hashFiles('system-tests/tests/go.sum', 'system-tests/tests/**/*.go', 'system-tests/lib/cre/**/*.go', 'core/scripts/cre/environment/**/*.go') }}
+
+ - name: Cache Pre-Compiled CRE Test Binaries (Warm Forks)
+ if: needs.run-core-cre-e2e-tests-setup.outputs.should-run == 'true' && github.ref_name == 'develop'
+ uses: actions/cache/save@v6
+ with:
+ path: system-tests/tests/bin/
+ key: test-binaries-cre-${{ hashFiles('system-tests/tests/go.sum', 'system-tests/tests/**/*.go', 'system-tests/lib/cre/**/*.go', 'core/scripts/cre/environment/**/*.go') }}-${{ github.sha }}
+
+ - name: Cache Pre-Compiled CCIP Test Binaries
+ if: needs.run-ccip-v1-6-e2e-tests-setup.outputs.should-run == 'true'
+ uses: actions/cache/save@v6
+ with:
+ path: integration-tests/bin/
+ key: test-binaries-${{ inputs.evm-ref || inputs.cl_ref || github.sha }}
+
run-ccip-v1-6-e2e-tests:
- needs: [run-ccip-v1-6-e2e-tests-setup, build-chainlink, changes, labels]
- name: ${{ needs.run-ccip-v1-6-e2e-tests-setup.outputs.workflow-name }}
+ needs: [run-ccip-v1-6-e2e-tests-setup, build-chainlink, compile-tests]
+ name: Run CCIP v1.6 E2E Tests
permissions:
actions: read
checks: write
@@ -522,39 +668,13 @@ jobs:
id-token: write
contents: read
if: needs.run-ccip-v1-6-e2e-tests-setup.outputs.should-run == 'true'
- uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@9c49ffcf252efbedd7ec280e1993ba29dcbc9443 # 2026-04-28
+ uses: ./.github/workflows/ccip-system-tests.yaml
with:
- workflow_name: ${{ needs.run-ccip-v1-6-e2e-tests-setup.outputs.workflow-name }}
+ ecr: "sdlc"
+ chainlink_image_repository_path: ${{ inputs.ecr_name || 'chainlink-integration-tests' }}
chainlink_version: ${{ inputs.evm-ref || inputs.cl_ref || github.sha }}
- test_path: .github/e2e-tests.yml
- test_trigger: ${{ needs.run-ccip-v1-6-e2e-tests-setup.outputs.test-trigger }}
- upload_cl_node_coverage_artifact: true
- enable_otel_traces_for_ocr2_plugins: ${{ contains(join(github.event.pull_request.labels.*.name, ' '), 'enable tracing') }}
- use-self-hosted-runners: ${{ needs.labels.outputs.should-use-self-hosted-runners }}
- ecr_name: ${{ inputs.ecr_name || 'chainlink-integration-tests' }}
- quarantine: "true"
- secrets:
- QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
- QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
- QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
- PROD_AWS_ACCOUNT_NUMBER: ${{ secrets.AWS_ACCOUNT_ID_PROD }}
- QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
- QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
- GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
- GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
- GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
- GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
- LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
- LOKI_URL: ${{ secrets.LOKI_URL }}
- LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- AWS_REGION: ${{ secrets.QA_AWS_REGION }}
- AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
- AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
- SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
- OPTIONAL_GATI_AWS_ROLE_ARN: ${{ secrets.AWS_OIDC_CHAINLINK_READ_ONLY_TOKEN_ISSUER_ROLE_ARN }}
- OPTIONAL_GATI_LAMBDA_URL: ${{ secrets.AWS_INFRA_RELENG_TOKEN_ISSUER_LAMBDA_URL}}
- TRUNK_API_KEY: ${{ secrets.TRUNK_API_KEY }}
+ chainlink_image_tag: ${{ inputs.evm-ref && format('{0}', inputs.evm-ref) || inputs.cl_ref && format('{0}', inputs.cl_ref) || format('{0}', github.sha) }}
+ secrets: inherit
check-e2e-test-results:
if: always()
@@ -568,71 +688,38 @@ jobs:
run-core-cre-e2e-regression-tests,
]
steps:
- - name: Check CCIP v1.6 E2E test results
- id: check_ccip_v1_6_results
- env:
- TEST_RESULTS: ${{ needs.run-ccip-v1-6-e2e-tests.outputs.test_results }}
- run: |
- results="$TEST_RESULTS"
- echo "CCIP v1.6 E2E test results:"
- echo "$results" | jq .
-
- - name: Fail the job if CCIP v1.6 E2E tests were not successful
- if: always()
- env:
- JOB_RESULT: ${{ needs.run-ccip-v1-6-e2e-tests.result }}
- run: |
- if [[ "${JOB_RESULT}" == "failure" ]]; then
- echo "::error::CCIP v1.6 E2E tests failed."
- exit 1
- elif [[ "${JOB_RESULT}" == "cancelled" ]]; then
- echo "::error::CCIP v1.6 E2E tests were cancelled."
- exit 1
- elif [[ "${JOB_RESULT}" == "skipped" ]]; then
- echo "::warning::CCIP v1.6 E2E tests were skipped."
- fi
-
- - name: Fail the job if core CRE tests were not successful
+ - name: Check all E2E test results
if: always()
env:
- JOB_RESULT: ${{ needs.run-core-cre-e2e-tests.result }}
+ CCIP_RESULT: ${{ needs.run-ccip-v1-6-e2e-tests.result }}
+ CRE_RESULT: ${{ needs.run-core-cre-e2e-tests.result }}
+ CRE_REGRESSION_RESULT: ${{ needs.run-core-cre-e2e-regression-tests.result }}
+ BUILD_RESULT: ${{ needs.build-chainlink.result }}
run: |
- if [[ "${JOB_RESULT}" == "failure" ]]; then
- echo "::error::Core CRE E2E tests failed."
- exit 1
- elif [[ "${JOB_RESULT}" == "cancelled" ]]; then
- echo "::error::Core CRE E2E tests were cancelled."
- exit 1
- elif [[ "${JOB_RESULT}" == "skipped" ]]; then
- echo "::warning::Core CRE E2E tests were skipped."
- fi
-
- - name: Fail the job if core CRE regression tests were not successful
- if: always()
- env:
- JOB_RESULT: ${{ needs.run-core-cre-e2e-regression-tests.result }}
- run: |
- if [[ "${JOB_RESULT}" == "failure" ]]; then
- echo "::error::Core CRE E2E regression tests failed."
- exit 1
- elif [[ "${JOB_RESULT}" == "cancelled" ]]; then
- echo "::error::Core CRE E2E regression tests were cancelled."
- exit 1
- elif [[ "${JOB_RESULT}" == "skipped" ]]; then
- echo "::warning::Core CRE E2E regression tests were skipped."
- fi
-
- - name: Fail the job if Chainlink image was cancelled or failed to build
- if: always()
- env:
- JOB_RESULT: ${{ needs.build-chainlink.result }}
- run: |
- if [[ "${JOB_RESULT}" == "failure" ]]; then
- echo "::error::Chainlink image build failed."
- exit 1
- elif [[ "${JOB_RESULT}" == "cancelled" ]]; then
- echo "::error::Chainlink image build was cancelled."
+ check_result() {
+ local name="$1" result="$2"
+ case "${result}" in
+ failure)
+ echo "::error::${name} failed."
+ return 1
+ ;;
+ cancelled)
+ echo "::error::${name} cancelled."
+ return 1
+ ;;
+ skipped)
+ echo "::warning::${name} skipped."
+ ;;
+ esac
+ return 0
+ }
+
+ failed=0
+ check_result "CCIP v1.6 E2E tests" "${CCIP_RESULT}" || failed=1
+ check_result "Core CRE E2E tests" "${CRE_RESULT}" || failed=1
+ check_result "Core CRE E2E regression tests" "${CRE_REGRESSION_RESULT}" || failed=1
+ check_result "Chainlink image build" "${BUILD_RESULT}" || failed=1
+
+ if [[ "$failed" -eq 1 ]]; then
exit 1
- elif [[ "${JOB_RESULT}" == "skipped" ]]; then
- echo "::warning::Chainlink image build was skipped."
fi
diff --git a/.github/workflows/legacy-non-functional-tests.yml b/.github/workflows/legacy-non-functional-tests.yml
index 045c45b36e1..69d2910a307 100644
--- a/.github/workflows/legacy-non-functional-tests.yml
+++ b/.github/workflows/legacy-non-functional-tests.yml
@@ -133,7 +133,7 @@ jobs:
just-version: "1.40.0"
- name: Configure AWS credentials using OIDC
- uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0
+ uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3
with:
role-to-assume: ${{ secrets.AWS_CTF_READ_ACCESS_ROLE_ARN }}
aws-region: us-west-2
diff --git a/.github/workflows/legacy-system-tests.yml b/.github/workflows/legacy-system-tests.yml
index 3d803371bbc..301a27d91e2 100644
--- a/.github/workflows/legacy-system-tests.yml
+++ b/.github/workflows/legacy-system-tests.yml
@@ -256,7 +256,7 @@ jobs:
just-version: "1.40.0"
- name: Configure AWS credentials using OIDC
- uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0
+ uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3
with:
role-to-assume: ${{ secrets.AWS_CTF_READ_ACCESS_ROLE_ARN }}
aws-region: us-west-2
diff --git a/.github/workflows/run-nightly-in-memory-integration-tests.yml b/.github/workflows/run-nightly-in-memory-integration-tests.yml
index c3b1c01b248..7e5f582dafe 100644
--- a/.github/workflows/run-nightly-in-memory-integration-tests.yml
+++ b/.github/workflows/run-nightly-in-memory-integration-tests.yml
@@ -36,7 +36,7 @@ on:
jobs:
call-nightly-run-e2e-tests-workflow:
name: Run Integration In-Memory Tests
- uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@9c49ffcf252efbedd7ec280e1993ba29dcbc9443 # 2026-04-28
+ uses: smartcontractkit/.github/.github/workflows/run-e2e-tests.yml@run-e2e-tests/v1
with:
chainlink_version: ${{ inputs.chainlink_version || 'develop' }}
test_path: .github/integration-in-memory-tests.yml
diff --git a/.gitignore b/.gitignore
index bb9f433a047..bdf9df52c4e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -26,7 +26,7 @@ tools/clroot/db.sqlite3-wal
.env*
.dbenv
!.github/actions/setup-postgres/.env
-/.github/.agents/skills/right-size-runners/trials/
+/.github/.agents/skills/**/trials/
.direnv
.idea
.vscode/
@@ -35,8 +35,6 @@ debug.env
operator_ui/install
.devenv
event_dump.ndjson
-.cursor/
-.claude/
# neovim
.nvim.lua
@@ -135,3 +133,9 @@ core/scripts/cre/environment/logs/
core/scripts/cre/environment/cron
core/scripts/cre/environment/binaries/*
*.br.b64
+
+# AI Agent settings
+.claude/
+.cursor/
+.gemini/
+opencode.json
\ No newline at end of file
diff --git a/core/chainlink.Dockerfile b/core/chainlink.Dockerfile
index 1cc14f0f038..c68ca264de9 100644
--- a/core/chainlink.Dockerfile
+++ b/core/chainlink.Dockerfile
@@ -88,9 +88,9 @@ RUN --mount=type=cache,target=/root/.cache/go-build,id=go-build-chainlink \
mkdir -p /gobins && \
if [ "$CL_IS_PROD_BUILD" = "false" ]; then \
GOBIN=/gobins make install-chainlink-dev; \
- else \
+ else \
GOBIN=/gobins make install-chainlink; \
- fi
+ fi
##
# Final Image
diff --git a/plugins/chainlink.Dockerfile b/plugins/chainlink.Dockerfile
index 676f5b52a8c..d7c5636c781 100644
--- a/plugins/chainlink.Dockerfile
+++ b/plugins/chainlink.Dockerfile
@@ -88,9 +88,9 @@ RUN --mount=type=cache,target=/root/.cache/go-build,id=go-build-chainlink \
mkdir -p /gobins && \
if [ "$CL_IS_PROD_BUILD" = "false" ]; then \
GOBIN=/gobins make install-chainlink-dev; \
- else \
+ else \
GOBIN=/gobins make install-chainlink; \
- fi
+ fi
##
# Final Image
diff --git a/system-tests/lib/cre/environment/blockchains/blockchains.go b/system-tests/lib/cre/environment/blockchains/blockchains.go
index 5f5e015a428..0fb5c555e92 100644
--- a/system-tests/lib/cre/environment/blockchains/blockchains.go
+++ b/system-tests/lib/cre/environment/blockchains/blockchains.go
@@ -6,6 +6,7 @@ import (
pkgerrors "github.com/pkg/errors"
"github.com/rs/zerolog"
+ "golang.org/x/sync/errgroup"
"github.com/smartcontractkit/chainlink-common/pkg/logger"
cldf_chain "github.com/smartcontractkit/chainlink-deployments-framework/chain"
@@ -55,28 +56,36 @@ func Start(
inputs []*blockchain.Input,
deployers map[blockchain.ChainFamily]Deployer,
) (*DeployedBlockchains, error) {
- outputs := make([]Blockchain, 0, len(inputs))
+ outputs := make([]Blockchain, len(inputs))
+ g, gctx := errgroup.WithContext(ctx)
+
+ for i, input := range inputs {
+ g.Go(func() error {
+ chainFamily, chErr := blockchain.TypeToFamily(input.Type)
+ if chErr != nil {
+ return chErr
+ }
- for _, input := range inputs {
- chainFamily, chErr := blockchain.TypeToFamily(input.Type)
- if chErr != nil {
- return nil, chErr
- }
+ deployer, ok := deployers[chainFamily]
+ if !ok {
+ if err := framework.PrintFailedContainerLogs(30); err != nil {
+ testLogger.Error().Err(err).Msg("failed to print failed Docker container logs")
+ }
+ return fmt.Errorf("no deployer found for blockchain type %s", input.Type)
+ }
- deployer, ok := deployers[chainFamily]
- if !ok {
- if err := framework.PrintFailedContainerLogs(30); err != nil {
- testLogger.Error().Err(err).Msg("failed to print failed Docker container logs")
+ deployedBlockchain, deployErr := deployer.Deploy(gctx, input)
+ if deployErr != nil {
+ return pkgerrors.Wrapf(deployErr, "failed to deploy blockchain of type %s", input.Type)
}
- return nil, fmt.Errorf("no deployer found for blockchain type %s", input.Type)
- }
- deployedBlockchain, deployErr := deployer.Deploy(ctx, input)
- if deployErr != nil {
- return nil, pkgerrors.Wrapf(deployErr, "failed to deploy blockchain of type %s", input.Type)
- }
+ outputs[i] = deployedBlockchain
+ return nil
+ })
+ }
- outputs = append(outputs, deployedBlockchain)
+ if err := g.Wait(); err != nil {
+ return nil, err
}
cldfBlockchains := make([]cldf_chain.BlockChain, 0, len(outputs))
diff --git a/system-tests/tests/regression/cre/consensus_regression_test.go b/system-tests/tests/regression/cre/consensus_regression_test.go
index 8dfea4a5694..cee5ada68f3 100644
--- a/system-tests/tests/regression/cre/consensus_regression_test.go
+++ b/system-tests/tests/regression/cre/consensus_regression_test.go
@@ -8,12 +8,11 @@ import (
commonevents "github.com/smartcontractkit/chainlink-protos/workflows/go/common"
workflowevents "github.com/smartcontractkit/chainlink-protos/workflows/go/events"
+ "github.com/smartcontractkit/chainlink-testing-framework/framework"
consensus_negative_config "github.com/smartcontractkit/chainlink/system-tests/tests/regression/cre/consensus/config"
t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers"
ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration"
-
- "github.com/smartcontractkit/chainlink-testing-framework/framework"
)
// regression
diff --git a/system-tests/tests/regression/cre/cre_regression_suite_test.go b/system-tests/tests/regression/cre/cre_regression_suite_test.go
index f41a465de38..c61b14fb60d 100644
--- a/system-tests/tests/regression/cre/cre_regression_suite_test.go
+++ b/system-tests/tests/regression/cre/cre_regression_suite_test.go
@@ -5,9 +5,9 @@ import (
"strings"
"testing"
- t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers"
-
"github.com/smartcontractkit/chainlink-testing-framework/framework"
+
+ t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers"
)
var (
diff --git a/system-tests/tests/regression/cre/cron_chip_ingress_stack_regression_test.go b/system-tests/tests/regression/cre/cron_chip_ingress_stack_regression_test.go
index cbe4db77aa2..c769ed904b2 100644
--- a/system-tests/tests/regression/cre/cron_chip_ingress_stack_regression_test.go
+++ b/system-tests/tests/regression/cre/cron_chip_ingress_stack_regression_test.go
@@ -9,7 +9,6 @@ import (
"github.com/smartcontractkit/chainlink-testing-framework/framework"
crontypes "github.com/smartcontractkit/chainlink/core/scripts/cre/environment/examples/workflows/cron/types"
-
t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers"
ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration"
)
@@ -45,7 +44,7 @@ func CronChipIngressStackFailsWithInvalidScheduleTest(t *testing.T, testEnv *tty
testLogger.Warn().Msgf("Expecting Cron workflow to fail with invalid schedule: %s", invalidSchedule)
// Not matched via UserLogs; engine init failure path ends the assertion with an error.
unusedExpectedUserLog := "__unused_expected_user_log_for_negative_test__"
- timeout := 75 * time.Second
+ timeout := 45 * time.Second
expectedError := t_helpers.AssertChipIngressStackMessage(listenerCtx, t, unusedExpectedUserLog, testLogger, messageChan, kafkaErrChan, timeout)
require.Error(t, expectedError, "Cron (Chip Ingress stack) test failed. This test expects to fail with an error, but did not.")
testLogger.Info().Msg("Cron (Chip Ingress stack) fail test completed")
diff --git a/system-tests/tests/regression/cre/evm_regression_test.go b/system-tests/tests/regression/cre/evm_regression_test.go
index 689560a8516..ebaf3274fc4 100644
--- a/system-tests/tests/regression/cre/evm_regression_test.go
+++ b/system-tests/tests/regression/cre/evm_regression_test.go
@@ -14,19 +14,17 @@ import (
commonevents "github.com/smartcontractkit/chainlink-protos/workflows/go/common"
workflowevents "github.com/smartcontractkit/chainlink-protos/workflows/go/events"
- keystone_changeset "github.com/smartcontractkit/chainlink/deployment/keystone/changeset"
+ "github.com/smartcontractkit/chainlink-testing-framework/framework"
+ keystone_changeset "github.com/smartcontractkit/chainlink/deployment/keystone/changeset"
"github.com/smartcontractkit/chainlink/system-tests/lib/cre"
"github.com/smartcontractkit/chainlink/system-tests/lib/cre/contracts"
"github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment/blockchains/evm"
-
evm_negative_config "github.com/smartcontractkit/chainlink/system-tests/tests/regression/cre/evm/evmread-negative/config"
evm_write_negative_config "github.com/smartcontractkit/chainlink/system-tests/tests/regression/cre/evm/evmwrite-negative/config"
evm_logtrigger_negative_config "github.com/smartcontractkit/chainlink/system-tests/tests/regression/cre/evm/logtrigger-negative/config"
t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers"
ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration"
-
- "github.com/smartcontractkit/chainlink-testing-framework/framework"
)
// regression
@@ -253,7 +251,7 @@ func EVMReadFailsTest(t *testing.T, testEnv *ttypes.TestEnvironment, evmNegative
baseMessageCh,
t_helpers.WorkflowEngineInitErrorLog,
evmNegativeTest.expectedError,
- 2*time.Minute,
+ 60*time.Second,
t_helpers.WithUserLogWorkflowID(workflowID),
)
testLogger.Info().Msgf("EVM Read Fail test successfully completed for test case %s and chain %s", evmNegativeTest.name, chainID)
@@ -317,7 +315,7 @@ func EVMLogTriggerFailsTest(t *testing.T, testEnv *ttypes.TestEnvironment, evmNe
testLogger,
baseMessageCh,
t_helpers.WorkflowEngineInitErrorLog,
- 2*time.Minute,
+ 60*time.Second,
t_helpers.WithBaseMessageWorkflowID(workflowID),
t_helpers.WithBaseMessageLabelContains("err", evmNegativeTest.expectedError),
)
@@ -418,7 +416,7 @@ func EVMWriteFailsTest(t *testing.T, testEnv *ttypes.TestEnvironment, evmNegativ
baseMessageCh,
t_helpers.WorkflowEngineInitErrorLog,
evmNegativeTest.expectedError,
- 2*time.Minute,
+ 60*time.Second,
t_helpers.WithUserLogWorkflowID(workflowID),
)
testLogger.Info().Msg("EVM Write Regression test successfully completed")
diff --git a/system-tests/tests/regression/cre/http_action_regression_test.go b/system-tests/tests/regression/cre/http_action_regression_test.go
index 7c147c55971..68a850da1ec 100644
--- a/system-tests/tests/regression/cre/http_action_regression_test.go
+++ b/system-tests/tests/regression/cre/http_action_regression_test.go
@@ -8,7 +8,6 @@ import (
commonevents "github.com/smartcontractkit/chainlink-protos/workflows/go/common"
workflowevents "github.com/smartcontractkit/chainlink-protos/workflows/go/events"
-
"github.com/smartcontractkit/chainlink-testing-framework/framework"
httpaction_config "github.com/smartcontractkit/chainlink/system-tests/tests/regression/cre/httpaction-negative/config"
diff --git a/system-tests/tests/regression/cre/http_trigger_regression_test.go b/system-tests/tests/regression/cre/http_trigger_regression_test.go
index dd219726a4f..8bc7ce0b2ee 100644
--- a/system-tests/tests/regression/cre/http_trigger_regression_test.go
+++ b/system-tests/tests/regression/cre/http_trigger_regression_test.go
@@ -18,6 +18,7 @@ import (
"github.com/google/uuid"
"github.com/stretchr/testify/require"
+ "github.com/smartcontractkit/chainlink-common/keystore/corekeys/dkgrecipientkey"
jsonrpc "github.com/smartcontractkit/chainlink-common/pkg/jsonrpc2"
gateway_common "github.com/smartcontractkit/chainlink-common/pkg/types/gateway"
commonevents "github.com/smartcontractkit/chainlink-protos/workflows/go/common"
@@ -25,14 +26,12 @@ import (
"github.com/smartcontractkit/chainlink-testing-framework/framework"
"github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake"
- "github.com/smartcontractkit/chainlink-common/keystore/corekeys/dkgrecipientkey"
- "github.com/smartcontractkit/chainlink/v2/core/utils"
-
"github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment/blockchains/evm"
libcrypto "github.com/smartcontractkit/chainlink/system-tests/lib/crypto"
http_config "github.com/smartcontractkit/chainlink/system-tests/tests/regression/cre/http/config"
t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers"
ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration"
+ "github.com/smartcontractkit/chainlink/v2/core/utils"
)
// regression - HTTP trigger negative test cases
@@ -150,7 +149,7 @@ func HTTPTriggerFailsTest(t *testing.T, testEnv *ttypes.TestEnvironment, httpNeg
testLogger,
baseMessageCh,
t_helpers.WorkflowEngineInitErrorLog,
- 2*time.Minute,
+ 30*time.Second,
t_helpers.WithBaseMessageWorkflowID(workflowID),
t_helpers.WithBaseMessageLabelContains("err", httpNegativeTest.expectedError),
)
@@ -181,8 +180,8 @@ func executeHTTPTriggerRequestExpectingFailure(t *testing.T, testEnv *ttypes.Tes
// Retry logic to wait for workflow to be loaded, then expect auth failure
var authFailureDetected bool
- tick := 5 * time.Second
- timeout := 3 * time.Minute
+ tick := t_helpers.DefaultPollInterval
+ timeout := 45 * time.Second
require.Eventually(t, func() bool {
// Create HTTP trigger request with unauthorized key
diff --git a/system-tests/tests/smoke/cre/aptos_capability_test.go b/system-tests/tests/smoke/cre/aptos_capability_test.go
index a747f3b9a5e..2a63e13fa9b 100644
--- a/system-tests/tests/smoke/cre/aptos_capability_test.go
+++ b/system-tests/tests/smoke/cre/aptos_capability_test.go
@@ -25,11 +25,10 @@ import (
aptosbind "github.com/smartcontractkit/chainlink-aptos/bindings/bind"
aptosdatafeeds "github.com/smartcontractkit/chainlink-aptos/bindings/data_feeds"
aptosplatformsecondary "github.com/smartcontractkit/chainlink-aptos/bindings/platform_secondary"
- "github.com/smartcontractkit/chainlink-testing-framework/framework"
- "github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain"
-
commonevents "github.com/smartcontractkit/chainlink-protos/workflows/go/common"
workflowevents "github.com/smartcontractkit/chainlink-protos/workflows/go/events"
+ "github.com/smartcontractkit/chainlink-testing-framework/framework"
+ "github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain"
crelib "github.com/smartcontractkit/chainlink/system-tests/lib/cre"
crecontracts "github.com/smartcontractkit/chainlink/system-tests/lib/cre/contracts"
diff --git a/system-tests/tests/smoke/cre/billing_helpers.go b/system-tests/tests/smoke/cre/billing_helpers.go
index 3937279a995..a6fde9d9b22 100644
--- a/system-tests/tests/smoke/cre/billing_helpers.go
+++ b/system-tests/tests/smoke/cre/billing_helpers.go
@@ -21,9 +21,9 @@ import (
"github.com/smartcontractkit/chainlink-testing-framework/framework"
"github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain"
"github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake"
+
libcre "github.com/smartcontractkit/chainlink/system-tests/lib/cre"
"github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment/config"
-
ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration"
)
diff --git a/system-tests/tests/smoke/cre/consensus_capability_test.go b/system-tests/tests/smoke/cre/consensus_capability_test.go
index 16fafb3b497..a298da8c6e0 100644
--- a/system-tests/tests/smoke/cre/consensus_capability_test.go
+++ b/system-tests/tests/smoke/cre/consensus_capability_test.go
@@ -5,10 +5,9 @@ import (
"testing"
"time"
- "github.com/smartcontractkit/chainlink-testing-framework/framework"
-
commonevents "github.com/smartcontractkit/chainlink-protos/workflows/go/common"
workflowevents "github.com/smartcontractkit/chainlink-protos/workflows/go/events"
+ "github.com/smartcontractkit/chainlink-testing-framework/framework"
t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers"
ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration"
diff --git a/system-tests/tests/smoke/cre/cre_suite_test.go b/system-tests/tests/smoke/cre/cre_suite_test.go
index 6391386e248..351a6b67c84 100644
--- a/system-tests/tests/smoke/cre/cre_suite_test.go
+++ b/system-tests/tests/smoke/cre/cre_suite_test.go
@@ -7,10 +7,9 @@ import (
"github.com/stretchr/testify/require"
- solana_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/solana/solread/config"
-
suite_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/config"
evm_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/evm/evmread/config"
+ solana_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/solana/solread/config"
t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers"
)
@@ -90,14 +89,15 @@ func runSuiteScenario(t *testing.T, topology string, scenario suite_config.Suite
allowlistSubtestName := "allowlist_auth_when_jwt_auth_disabled"
jwtSubtestName := "jwt_auth_rejected_when_jwt_auth_disabled"
vaultConfig := getVaultDefaultTestConfig(t)
- if isVaultJWTAuthEnabledTopology(topology) {
+ switch {
+ case isVaultJWTAuthEnabledTopology(topology):
vaultConfig = getVaultJWTAuthEnabledTestConfig(t)
allowlistSubtestName = "allowlist_auth_when_jwt_auth_enabled"
jwtSubtestName = "jwt_auth_when_jwt_auth_enabled"
- } else if isVaultOptimizationsEnabledTopology(topology) {
+ case isVaultOptimizationsEnabledTopology(topology):
vaultConfig = getVaultOptimizationsEnabledTestConfig(t)
allowlistSubtestName = "allowlist_auth_when_vault_optimizations_enabled"
- } else if isVaultWorkflowDONBindingEnabledTopology(topology) {
+ case isVaultWorkflowDONBindingEnabledTopology(topology):
vaultConfig = getVaultWorkflowDONBindingEnabledTestConfig(t)
allowlistSubtestName = "allowlist_auth_when_workflow_don_binding_enabled"
}
@@ -274,7 +274,6 @@ func Test_CRE_V2_Aptos_Suite(t *testing.T) {
})
}
-//nolint:paralleltest // isolate local cre env run
func Test_CRE_V2_Stellar_Suite(t *testing.T) {
testEnv := t_helpers.SetupTestEnvironmentWithConfig(t, t_helpers.GetTestConfig(t, "/configs/workflow-gateway-don-stellar.toml"))
diff --git a/system-tests/tests/smoke/cre/cron_chip_ingress_stack_test.go b/system-tests/tests/smoke/cre/cron_chip_ingress_stack_test.go
index 390b67a9e9b..8adae8ee312 100644
--- a/system-tests/tests/smoke/cre/cron_chip_ingress_stack_test.go
+++ b/system-tests/tests/smoke/cre/cron_chip_ingress_stack_test.go
@@ -9,7 +9,6 @@ import (
"github.com/smartcontractkit/chainlink-testing-framework/framework"
crontypes "github.com/smartcontractkit/chainlink/core/scripts/cre/environment/examples/workflows/cron/types"
-
t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers"
ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration"
)
diff --git a/system-tests/tests/smoke/cre/dontime_test.go b/system-tests/tests/smoke/cre/dontime_test.go
index f1b9c29b71f..ab215b91ee0 100644
--- a/system-tests/tests/smoke/cre/dontime_test.go
+++ b/system-tests/tests/smoke/cre/dontime_test.go
@@ -7,11 +7,9 @@ import (
commonevents "github.com/smartcontractkit/chainlink-protos/workflows/go/common"
workflowevents "github.com/smartcontractkit/chainlink-protos/workflows/go/events"
-
"github.com/smartcontractkit/chainlink-testing-framework/framework"
crontypes "github.com/smartcontractkit/chainlink/core/scripts/cre/environment/examples/workflows/cron/types"
-
t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers"
ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration"
)
diff --git a/system-tests/tests/smoke/cre/evm_capability_test.go b/system-tests/tests/smoke/cre/evm_capability_test.go
index f8b2d0e13ff..8b9eb3e7e48 100644
--- a/system-tests/tests/smoke/cre/evm_capability_test.go
+++ b/system-tests/tests/smoke/cre/evm_capability_test.go
@@ -13,24 +13,21 @@ import (
"github.com/rs/zerolog"
"github.com/stretchr/testify/require"
+ forwarder "github.com/smartcontractkit/chainlink-evm/gethwrappers/keystone/generated/forwarder_1_0_0"
commonevents "github.com/smartcontractkit/chainlink-protos/workflows/go/common"
workflowevents "github.com/smartcontractkit/chainlink-protos/workflows/go/events"
+ "github.com/smartcontractkit/chainlink-testing-framework/framework"
- "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/evmread/contracts"
-
+ keystonechangeset "github.com/smartcontractkit/chainlink/deployment/keystone/changeset"
crecontracts "github.com/smartcontractkit/chainlink/system-tests/lib/cre/contracts"
"github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment/blockchains"
"github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment/blockchains/evm"
evm_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/evm/evmread/config"
evmreadcontracts "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/evm/evmread/contracts"
evm_logTrigger_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/evm/logtrigger/config"
+ "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/evmread/contracts"
t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers"
ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration"
-
- forwarder "github.com/smartcontractkit/chainlink-evm/gethwrappers/keystone/generated/forwarder_1_0_0"
- "github.com/smartcontractkit/chainlink-testing-framework/framework"
-
- keystonechangeset "github.com/smartcontractkit/chainlink/deployment/keystone/changeset"
)
// smoke
diff --git a/system-tests/tests/smoke/cre/grpc_source_test.go b/system-tests/tests/smoke/cre/grpc_source_test.go
index 4ddaa9276b6..336f5f481fa 100644
--- a/system-tests/tests/smoke/cre/grpc_source_test.go
+++ b/system-tests/tests/smoke/cre/grpc_source_test.go
@@ -13,17 +13,14 @@ import (
_ "github.com/lib/pq"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/proto"
-
"gopkg.in/yaml.v3"
+ "github.com/smartcontractkit/chainlink-common/pkg/workflows"
+ "github.com/smartcontractkit/chainlink-common/pkg/workflows/privateregistry"
workflowsv2 "github.com/smartcontractkit/chainlink-protos/workflows/go/v2"
-
"github.com/smartcontractkit/chainlink-testing-framework/framework"
ns "github.com/smartcontractkit/chainlink-testing-framework/framework/components/simple_node_set"
- "github.com/smartcontractkit/chainlink-common/pkg/workflows"
- "github.com/smartcontractkit/chainlink-common/pkg/workflows/privateregistry"
-
crontypes "github.com/smartcontractkit/chainlink/core/scripts/cre/environment/examples/workflows/cron/types"
grpcsourcemock "github.com/smartcontractkit/chainlink/system-tests/lib/cre/grpc_source_mock"
creworkflow "github.com/smartcontractkit/chainlink/system-tests/lib/cre/workflow"
diff --git a/system-tests/tests/smoke/cre/http_action_test.go b/system-tests/tests/smoke/cre/http_action_test.go
index c4f107ce9be..105b953ebc3 100644
--- a/system-tests/tests/smoke/cre/http_action_test.go
+++ b/system-tests/tests/smoke/cre/http_action_test.go
@@ -14,6 +14,7 @@ import (
workflowevents "github.com/smartcontractkit/chainlink-protos/workflows/go/events"
"github.com/smartcontractkit/chainlink-testing-framework/framework"
"github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake"
+
httpactionconfig "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/httpaction/config"
t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers"
ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration"
diff --git a/system-tests/tests/smoke/cre/http_trigger_action_test.go b/system-tests/tests/smoke/cre/http_trigger_action_test.go
index 483bee2eb64..b2cff0e7acb 100644
--- a/system-tests/tests/smoke/cre/http_trigger_action_test.go
+++ b/system-tests/tests/smoke/cre/http_trigger_action_test.go
@@ -22,13 +22,11 @@ import (
"github.com/smartcontractkit/chainlink-testing-framework/framework"
"github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake"
- "github.com/smartcontractkit/chainlink/v2/core/utils"
-
"github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment/blockchains/evm"
libcrypto "github.com/smartcontractkit/chainlink/system-tests/lib/crypto"
-
t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers"
ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration"
+ "github.com/smartcontractkit/chainlink/v2/core/utils"
)
const (
@@ -97,7 +95,7 @@ func executeHTTPTriggerRequest(t *testing.T, testEnv *ttypes.TestEnvironment, ga
var finalResponse jsonrpc.Response[json.RawMessage]
var triggerRequest jsonrpc.Request[json.RawMessage]
- tick := 5 * time.Second
+ tick := t_helpers.DefaultPollInterval
require.Eventually(t, func() bool {
triggerRequest = createHTTPTriggerRequestWithKey(t, workflowName, workflowID, workflowOwnerAddress, singingKey)
triggerRequestBody, err := json.Marshal(triggerRequest)
@@ -161,7 +159,7 @@ func executeHTTPTriggerRequest(t *testing.T, testEnv *ttypes.TestEnvironment, ga
// validateHTTPWorkflowRequest validates that the workflow made the expected HTTP request
func validateHTTPWorkflowRequest(t *testing.T, testEnv *ttypes.TestEnvironment) {
- tick := 5 * time.Second
+ tick := t_helpers.DefaultPollInterval
require.Eventually(t, func() bool {
records, err := fake.R.Get("POST", "/orders")
return err == nil && len(records) > 0
diff --git a/system-tests/tests/smoke/cre/log_streaming_test.go b/system-tests/tests/smoke/cre/log_streaming_test.go
index 04957fa13e1..a8d0543146f 100644
--- a/system-tests/tests/smoke/cre/log_streaming_test.go
+++ b/system-tests/tests/smoke/cre/log_streaming_test.go
@@ -13,9 +13,9 @@ import (
"github.com/stretchr/testify/require"
+ testutils "github.com/smartcontractkit/chainlink-common/pkg/utils/tests"
"github.com/smartcontractkit/chainlink-testing-framework/framework"
- testutils "github.com/smartcontractkit/chainlink-common/pkg/utils/tests"
ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration"
)
diff --git a/system-tests/tests/smoke/cre/multi_gateway_http_action_test.go b/system-tests/tests/smoke/cre/multi_gateway_http_action_test.go
index 3fbdff749b4..474224b88f2 100644
--- a/system-tests/tests/smoke/cre/multi_gateway_http_action_test.go
+++ b/system-tests/tests/smoke/cre/multi_gateway_http_action_test.go
@@ -14,6 +14,7 @@ import (
workflowevents "github.com/smartcontractkit/chainlink-protos/workflows/go/events"
"github.com/smartcontractkit/chainlink-testing-framework/framework"
"github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake"
+
"github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment/blockchains/evm"
envconfig "github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment/config"
stvault "github.com/smartcontractkit/chainlink/system-tests/lib/cre/vault"
diff --git a/system-tests/tests/smoke/cre/por_helpers.go b/system-tests/tests/smoke/cre/por_helpers.go
index 5da677e9cc3..13818708531 100644
--- a/system-tests/tests/smoke/cre/por_helpers.go
+++ b/system-tests/tests/smoke/cre/por_helpers.go
@@ -19,22 +19,17 @@ import (
cldf_tron "github.com/smartcontractkit/chainlink-deployments-framework/chain/tron"
"github.com/smartcontractkit/chainlink-deployments-framework/datastore"
"github.com/smartcontractkit/chainlink-evm/gethwrappers/data-feeds/generated/data_feeds_cache"
+ corevm "github.com/smartcontractkit/chainlink-evm/pkg/relay"
"github.com/smartcontractkit/chainlink-testing-framework/framework"
"github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain"
-
- tron_df_changeset "github.com/smartcontractkit/chainlink/deployment/data-feeds/changeset/tron"
- df_changeset_types "github.com/smartcontractkit/chainlink/deployment/data-feeds/changeset/types"
-
cldchangeset "github.com/smartcontractkit/cld-changesets/pkg/cldfutil/changeset"
+ portypes "github.com/smartcontractkit/chainlink/core/scripts/cre/environment/examples/workflows/proof-of-reserve/cron-based/types"
df_changeset "github.com/smartcontractkit/chainlink/deployment/data-feeds/changeset"
+ tron_df_changeset "github.com/smartcontractkit/chainlink/deployment/data-feeds/changeset/tron"
+ df_changeset_types "github.com/smartcontractkit/chainlink/deployment/data-feeds/changeset/types"
keystone_changeset "github.com/smartcontractkit/chainlink/deployment/keystone/changeset"
tron_keystone_changeset "github.com/smartcontractkit/chainlink/deployment/keystone/changeset/tron"
-
- corevm "github.com/smartcontractkit/chainlink-evm/pkg/relay"
-
- portypes "github.com/smartcontractkit/chainlink/core/scripts/cre/environment/examples/workflows/proof-of-reserve/cron-based/types"
-
"github.com/smartcontractkit/chainlink/system-tests/lib/cre"
crecontracts "github.com/smartcontractkit/chainlink/system-tests/lib/cre/contracts"
"github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment/blockchains"
@@ -511,7 +506,7 @@ func validatePoRPrices(t *testing.T, testEnv *ttypes.TestEnvironment, priceProvi
startTime := time.Now()
waitFor := 5 * time.Minute
- tick := 5 * time.Second
+ tick := t_helpers.DefaultPollInterval
switch bcOutput.CtfOutput().Family {
case blockchain.FamilyTron:
diff --git a/system-tests/tests/smoke/cre/sharding_test.go b/system-tests/tests/smoke/cre/sharding_test.go
index 87e62bbafff..3de94d28236 100644
--- a/system-tests/tests/smoke/cre/sharding_test.go
+++ b/system-tests/tests/smoke/cre/sharding_test.go
@@ -13,7 +13,6 @@ import (
"github.com/Masterminds/semver/v3"
"github.com/ethereum/go-ethereum/common"
"github.com/rs/zerolog"
- cldchangeset "github.com/smartcontractkit/cld-changesets/pkg/cldfutil/changeset"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
@@ -25,6 +24,7 @@ import (
commonevents "github.com/smartcontractkit/chainlink-protos/workflows/go/common"
workflowevents "github.com/smartcontractkit/chainlink-protos/workflows/go/events"
"github.com/smartcontractkit/chainlink-testing-framework/framework"
+ cldchangeset "github.com/smartcontractkit/cld-changesets/pkg/cldfutil/changeset"
crontypes "github.com/smartcontractkit/chainlink/core/scripts/cre/environment/examples/workflows/cron/types"
deployment_contracts "github.com/smartcontractkit/chainlink/deployment/cre/contracts"
@@ -115,7 +115,7 @@ func ExecuteShardingTest(t *testing.T, testEnv *ttypes.TestEnvironment) {
const numWorkflows = 5
workflowFileLocation := "../../../../core/scripts/cre/environment/examples/workflows/cron/main.go"
- var workflowIDs []string
+ workflowIDs := make([]string, 0, numWorkflows)
for i := range numWorkflows {
workflowName := fmt.Sprintf("shardtest%d", i)
workflowConfig := crontypes.WorkflowConfig{
diff --git a/system-tests/tests/smoke/cre/solana_capability_test.go b/system-tests/tests/smoke/cre/solana_capability_test.go
index 419cf4a47d3..b740caf7471 100644
--- a/system-tests/tests/smoke/cre/solana_capability_test.go
+++ b/system-tests/tests/smoke/cre/solana_capability_test.go
@@ -16,17 +16,16 @@ import (
solgo "github.com/gagliardetto/solana-go"
"github.com/gagliardetto/solana-go/rpc"
"github.com/rs/zerolog"
- chainselectors "github.com/smartcontractkit/chain-selectors"
"github.com/stretchr/testify/require"
+ chainselectors "github.com/smartcontractkit/chain-selectors"
solCommonUtil "github.com/smartcontractkit/chainlink-ccip/chains/solana/utils/common"
- commonevents "github.com/smartcontractkit/chainlink-protos/workflows/go/common"
- workflowevents "github.com/smartcontractkit/chainlink-protos/workflows/go/events"
- solana_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/solana/solread/config"
-
"github.com/smartcontractkit/chainlink-deployments-framework/datastore"
cldf "github.com/smartcontractkit/chainlink-deployments-framework/deployment"
+ commonevents "github.com/smartcontractkit/chainlink-protos/workflows/go/common"
+ workflowevents "github.com/smartcontractkit/chainlink-protos/workflows/go/events"
"github.com/smartcontractkit/chainlink-testing-framework/framework"
+
commonchangeset "github.com/smartcontractkit/chainlink/deployment/common/changeset"
ks_sol "github.com/smartcontractkit/chainlink/deployment/cre/forwarder/solana"
df_sol "github.com/smartcontractkit/chainlink/deployment/data-feeds/changeset/solana"
@@ -35,6 +34,7 @@ import (
"github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment/blockchains/evm"
"github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment/blockchains/solana"
sollogtrigger_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/solana/sollogtrigger/config"
+ solana_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/solana/solread/config"
"github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/solana/solwrite/config"
t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers"
"github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration"
diff --git a/system-tests/tests/smoke/cre/stellar_capability_test.go b/system-tests/tests/smoke/cre/stellar_capability_test.go
index fd90a7a0bee..0c5dd9dc595 100644
--- a/system-tests/tests/smoke/cre/stellar_capability_test.go
+++ b/system-tests/tests/smoke/cre/stellar_capability_test.go
@@ -6,14 +6,12 @@ import (
"testing"
"time"
+ "github.com/stellar/go-stellar-sdk/xdr"
"github.com/stretchr/testify/require"
- "github.com/smartcontractkit/chainlink-testing-framework/framework"
-
commonevents "github.com/smartcontractkit/chainlink-protos/workflows/go/common"
workflowevents "github.com/smartcontractkit/chainlink-protos/workflows/go/events"
-
- "github.com/stellar/go-stellar-sdk/xdr"
+ "github.com/smartcontractkit/chainlink-testing-framework/framework"
crelib "github.com/smartcontractkit/chainlink/system-tests/lib/cre"
"github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment/blockchains"
diff --git a/system-tests/tests/smoke/cre/v2_durable_emitter_test.go b/system-tests/tests/smoke/cre/v2_durable_emitter_test.go
index 6dd7bd61cef..b7a009bb836 100644
--- a/system-tests/tests/smoke/cre/v2_durable_emitter_test.go
+++ b/system-tests/tests/smoke/cre/v2_durable_emitter_test.go
@@ -13,7 +13,6 @@ import (
"github.com/smartcontractkit/chainlink-testing-framework/framework"
crontypes "github.com/smartcontractkit/chainlink/core/scripts/cre/environment/examples/workflows/cron/types"
-
"github.com/smartcontractkit/chainlink/system-tests/lib/cre"
t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers"
ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration"
diff --git a/system-tests/tests/smoke/cre/v2_module_cache_test.go b/system-tests/tests/smoke/cre/v2_module_cache_test.go
index d0c41a0beee..c08c4c026da 100644
--- a/system-tests/tests/smoke/cre/v2_module_cache_test.go
+++ b/system-tests/tests/smoke/cre/v2_module_cache_test.go
@@ -11,7 +11,6 @@ import (
"github.com/smartcontractkit/chainlink-testing-framework/framework"
crontypes "github.com/smartcontractkit/chainlink/core/scripts/cre/environment/examples/workflows/cron/types"
-
t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers"
ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration"
)
diff --git a/system-tests/tests/smoke/cre/vault_don_test.go b/system-tests/tests/smoke/cre/vault_don_test.go
index c92ebe5cbe3..f0e1b83ce5d 100644
--- a/system-tests/tests/smoke/cre/vault_don_test.go
+++ b/system-tests/tests/smoke/cre/vault_don_test.go
@@ -14,38 +14,34 @@ import (
"testing"
"time"
+ retry "github.com/avast/retry-go/v4"
"github.com/ethereum/go-ethereum/common"
"github.com/golang-jwt/jwt/v5"
"github.com/google/uuid"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/encoding/protojson"
- retry "github.com/avast/retry-go/v4"
-
vault_helpers "github.com/smartcontractkit/chainlink-common/pkg/capabilities/actions/vault"
jsonrpc "github.com/smartcontractkit/chainlink-common/pkg/jsonrpc2"
"github.com/smartcontractkit/chainlink-common/pkg/settings/cresettings"
+ workflow_registry_v2_wrapper "github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/workflow_registry_wrapper_v2"
commonevents "github.com/smartcontractkit/chainlink-protos/workflows/go/common"
workflowevents "github.com/smartcontractkit/chainlink-protos/workflows/go/events"
+ "github.com/smartcontractkit/chainlink-testing-framework/framework"
+ "github.com/smartcontractkit/chainlink-testing-framework/seth"
+
keystone_changeset "github.com/smartcontractkit/chainlink/deployment/keystone/changeset"
crecontracts "github.com/smartcontractkit/chainlink/system-tests/lib/cre/contracts"
"github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment/blockchains/evm"
- t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers"
- vaultcap "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault"
- "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault/vaulttypes"
- "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault/vaultutils"
-
- workflow_registry_v2_wrapper "github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/workflow_registry_wrapper_v2"
-
envconfig "github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment/config"
"github.com/smartcontractkit/chainlink/system-tests/lib/cre/vault"
- ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration"
-
- "github.com/smartcontractkit/chainlink-testing-framework/framework"
- "github.com/smartcontractkit/chainlink-testing-framework/seth"
-
creworkflow "github.com/smartcontractkit/chainlink/system-tests/lib/cre/workflow"
vaultsecret_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/vaultsecret/config"
+ t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers"
+ ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration"
+ vaultcap "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault"
+ "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault/vaulttypes"
+ "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault/vaultutils"
)
// ExecuteVaultAllowListBasedTests covers vault gateway + workflows with allow-listed JSON-RPC auth
@@ -217,9 +213,6 @@ func ExecuteVaultAllowListBasedTests(t *testing.T, fixture *vaultScenarioFixture
if isVaultJWTAuthEnabledTopology(testEnv.TestConfig.EnvironmentConfigPath) {
t.Run("identifier_validation", func(t *testing.T) {
- if parallelEnabled {
- t.Parallel()
- }
subEnv := t_helpers.SetupTestEnvironmentWithPerTestKeys(t, testEnv.TestConfig)
sc := subEnv.CreEnvironment.Blockchains[0].(*evm.Blockchain).SethClient
owner := sc.MustGetRootKeyAddress().Hex()
diff --git a/system-tests/tests/smoke/cre/vault_don_test_helpers.go b/system-tests/tests/smoke/cre/vault_don_test_helpers.go
index bfd1010944b..145da45b1aa 100644
--- a/system-tests/tests/smoke/cre/vault_don_test_helpers.go
+++ b/system-tests/tests/smoke/cre/vault_don_test_helpers.go
@@ -20,13 +20,10 @@ import (
"github.com/ethereum/go-ethereum/accounts/abi/bind"
"github.com/ethereum/go-ethereum/common"
"github.com/google/uuid"
- "github.com/smartcontractkit/tdh2/go/tdh2/tdh2easy"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/proto"
- "github.com/smartcontractkit/chainlink-protos/cre/go/values"
-
vault_helpers "github.com/smartcontractkit/chainlink-common/pkg/capabilities/actions/vault"
capabilitiespb "github.com/smartcontractkit/chainlink-common/pkg/capabilities/pb"
jsonrpc "github.com/smartcontractkit/chainlink-common/pkg/jsonrpc2"
@@ -34,11 +31,14 @@ import (
"github.com/smartcontractkit/chainlink-common/pkg/settings/limits"
capabilities_registry_v2 "github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/capabilities_registry_wrapper_v2"
workflow_registry_v2_wrapper "github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/workflow_registry_wrapper_v2"
+ "github.com/smartcontractkit/chainlink-protos/cre/go/values"
commonevents "github.com/smartcontractkit/chainlink-protos/workflows/go/common"
workflowevents "github.com/smartcontractkit/chainlink-protos/workflows/go/events"
"github.com/smartcontractkit/chainlink-testing-framework/framework"
ctfblockchain "github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain"
"github.com/smartcontractkit/chainlink-testing-framework/seth"
+ "github.com/smartcontractkit/tdh2/go/tdh2/tdh2easy"
+
keystone_changeset "github.com/smartcontractkit/chainlink/deployment/keystone/changeset"
"github.com/smartcontractkit/chainlink/system-tests/lib/cre"
crecontracts "github.com/smartcontractkit/chainlink/system-tests/lib/cre/contracts"
diff --git a/system-tests/tests/soak/cre/workflow_caching_test.go b/system-tests/tests/soak/cre/workflow_caching_test.go
index 627ceab4035..2fd70a181d2 100644
--- a/system-tests/tests/soak/cre/workflow_caching_test.go
+++ b/system-tests/tests/soak/cre/workflow_caching_test.go
@@ -19,7 +19,6 @@ import (
"github.com/smartcontractkit/chainlink-testing-framework/framework"
crontypes "github.com/smartcontractkit/chainlink/core/scripts/cre/environment/examples/workflows/cron/types"
-
"github.com/smartcontractkit/chainlink/system-tests/lib/cre"
t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers"
)
diff --git a/system-tests/tests/test-helpers/before_suite.go b/system-tests/tests/test-helpers/before_suite.go
index 6b6b1f18b47..ec8bae9aa25 100644
--- a/system-tests/tests/test-helpers/before_suite.go
+++ b/system-tests/tests/test-helpers/before_suite.go
@@ -2,11 +2,11 @@ package helpers
import (
"context"
+ "crypto/ecdsa"
"encoding/hex"
"fmt"
"math/big"
"os"
- "os/exec"
"path/filepath"
"slices"
"strconv"
@@ -25,6 +25,7 @@ import (
cldf_evm "github.com/smartcontractkit/chainlink-deployments-framework/chain/evm"
"github.com/smartcontractkit/chainlink-deployments-framework/datastore"
cldf "github.com/smartcontractkit/chainlink-deployments-framework/deployment"
+ workflow_registry_v2_wrapper "github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/workflow_registry_wrapper_v2"
"github.com/smartcontractkit/chainlink-testing-framework/framework"
"github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain"
ctfchiprouter "github.com/smartcontractkit/chainlink-testing-framework/framework/components/chiprouter"
@@ -32,8 +33,6 @@ import (
keystone_changeset "github.com/smartcontractkit/chainlink/deployment/keystone/changeset"
cldlogger "github.com/smartcontractkit/chainlink/deployment/logger"
-
- workflow_registry_v2_wrapper "github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/workflow_registry_wrapper_v2"
"github.com/smartcontractkit/chainlink/system-tests/lib/cre/chiprouter"
crecontracts "github.com/smartcontractkit/chainlink/system-tests/lib/cre/contracts"
"github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment"
@@ -42,7 +41,6 @@ import (
envconfig "github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment/config"
crevault "github.com/smartcontractkit/chainlink/system-tests/lib/cre/vault"
crecrypto "github.com/smartcontractkit/chainlink/system-tests/lib/crypto"
-
ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration"
)
@@ -50,10 +48,16 @@ const (
perTestEVMFundingAmountWei uint64 = 1_000_000_000_000_000_000 // 1 ETH
)
+type preFundedKey struct {
+ addr common.Address
+ priv *ecdsa.PrivateKey
+}
+
type sharedEnvironmentEntry struct {
- once sync.Once
- env *ttypes.TestEnvironment
- err error
+ once sync.Once
+ env *ttypes.TestEnvironment
+ err error
+ keyPool chan *preFundedKey
}
var (
@@ -92,10 +96,10 @@ func SetupTestEnvironmentWithPerTestKeys(t *testing.T, tconf *ttypes.TestConfig,
func setupTestEnvironmentWithConfigMode(t *testing.T, tconf *ttypes.TestConfig, usePerTestKeys bool, flags ...string) *ttypes.TestEnvironment {
t.Helper()
- sharedEnv := getOrCreateSharedEnvironment(t, tconf, flags...)
- testEnv := cloneSharedEnvironmentForTest(sharedEnv, tconf)
+ entry := getOrCreateSharedEnvironmentEntry(t, tconf, flags...)
+ testEnv := cloneSharedEnvironmentForTest(entry.env, tconf)
if usePerTestKeys {
- testEnv.Execution = configurePerTestExecutionContext(t, sharedEnv, testEnv)
+ testEnv.Execution = configurePerTestExecutionContext(t, entry, testEnv)
}
t.Cleanup(func() {
@@ -118,12 +122,19 @@ func setupTestEnvironmentWithConfigMode(t *testing.T, tconf *ttypes.TestConfig,
func getOrCreateSharedEnvironment(t *testing.T, tconf *ttypes.TestConfig, flags ...string) *ttypes.TestEnvironment {
t.Helper()
+ return getOrCreateSharedEnvironmentEntry(t, tconf, flags...).env
+}
+
+func getOrCreateSharedEnvironmentEntry(t *testing.T, tconf *ttypes.TestConfig, flags ...string) *sharedEnvironmentEntry {
+ t.Helper()
key := sharedEnvironmentKey(tconf, flags)
sharedEnvMu.Lock()
entry, ok := sharedEnvironments[key]
if !ok {
- entry = &sharedEnvironmentEntry{}
+ entry = &sharedEnvironmentEntry{
+ keyPool: make(chan *preFundedKey, 24),
+ }
sharedEnvironments[key] = entry
}
sharedEnvMu.Unlock()
@@ -147,11 +158,40 @@ func getOrCreateSharedEnvironment(t *testing.T, tconf *ttypes.TestConfig, flags
CreEnvironment: creEnvironment,
Dons: dons,
}
+
+ numKeys := 24
+ var signers []common.Address
+ for range numKeys {
+ addr, priv, kerr := crecrypto.GenerateNewKeyPair()
+ require.NoError(t, kerr, "failed to generate key pair")
+ entry.keyPool <- &preFundedKey{addr: addr, priv: priv}
+ signers = append(signers, addr)
+ }
+
+ rootSignerNonceLock.Lock()
+ for _, bcOutput := range entry.env.CreEnvironment.Blockchains {
+ evmChain, ok := bcOutput.(*evm.Blockchain)
+ if !ok {
+ continue
+ }
+ for _, signer := range signers {
+ require.NoError(
+ t,
+ evmChain.Fund(t.Context(), signer.Hex(), perTestEVMFundingAmountWei),
+ "failed to fund pooled key %s on chain selector %d",
+ signer.Hex(),
+ evmChain.ChainSelector(),
+ )
+ }
+ }
+ rootSignerNonceLock.Unlock()
+
+ authorizePooledSigners(t, entry.env, signers)
})
require.NoError(t, entry.err, "failed to load environment")
require.NotNil(t, entry.env, "shared test environment was not initialized")
- return entry.env
+ return entry
}
func sharedEnvironmentKey(tconf *ttypes.TestConfig, flags []string) string {
@@ -179,12 +219,18 @@ func cloneSharedEnvironmentForTest(sharedEnv *ttypes.TestEnvironment, tconf *tty
// configurePerTestExecutionContext creates one funded, registry-authorized signer, swaps testEnv EVM blockchains
// to per-test seth clients, and sets the CLDF deployer key (SetupTestEnvironmentWithPerTestKeys).
-func configurePerTestExecutionContext(t *testing.T, sharedEnv *ttypes.TestEnvironment, testEnv *ttypes.TestEnvironment) *ttypes.ExecutionContext {
+func configurePerTestExecutionContext(t *testing.T, entry *sharedEnvironmentEntry, testEnv *ttypes.TestEnvironment) *ttypes.ExecutionContext {
t.Helper()
- ownerAddress, privateKey, addrErr := crecrypto.GenerateNewKeyPair()
- require.NoError(t, addrErr, "failed to generate per-test key pair")
- privateKeyHex := hex.EncodeToString(gethcrypto.FromECDSA(privateKey))
+ var key *preFundedKey
+ select {
+ case key = <-entry.keyPool:
+ default:
+ t.Fatal("key pool exhausted; increase preFundedKey pool size")
+ }
+
+ ownerAddress := key.addr
+ privateKeyHex := hex.EncodeToString(gethcrypto.FromECDSA(key.priv))
testID := deriveExecutionTestID(t)
execCtx := &ttypes.ExecutionContext{
@@ -194,7 +240,7 @@ func configurePerTestExecutionContext(t *testing.T, sharedEnv *ttypes.TestEnviro
registryChainSelector := testEnv.CreEnvironment.Blockchains[0].ChainSelector()
rootEVMChains := make(map[uint64]*evm.Blockchain)
- for _, bcOutput := range sharedEnv.CreEnvironment.Blockchains {
+ for _, bcOutput := range entry.env.CreEnvironment.Blockchains {
evmChain, ok := bcOutput.(*evm.Blockchain)
if !ok {
continue
@@ -220,18 +266,8 @@ func configurePerTestExecutionContext(t *testing.T, sharedEnv *ttypes.TestEnviro
Build()
require.NoErrorf(t, clientErr, "failed to create per-test seth client for selector %d", evmChain.ChainSelector())
- rootSignerNonceLock.Lock()
- require.NoError(
- t,
- rootChain.Fund(t.Context(), ownerAddress.Hex(), perTestEVMFundingAmountWei),
- "failed to fund per-test owner %s on chain selector %d",
- ownerAddress.Hex(),
- evmChain.ChainSelector(),
- )
- rootSignerNonceLock.Unlock()
-
testEnv.CreEnvironment.Blockchains[i] = evmChain.CloneWithSethClient(perTestClient)
- deployerKey, txOptsErr := bind.NewKeyedTransactorWithChainID(privateKey, big.NewInt(perTestClient.ChainID))
+ deployerKey, txOptsErr := bind.NewKeyedTransactorWithChainID(key.priv, big.NewInt(perTestClient.ChainID))
require.NoErrorf(t, txOptsErr, "failed to create deployer key for chain selector %d", evmChain.ChainSelector())
deployerKey.Context = t.Context()
require.NoErrorf(
@@ -246,7 +282,6 @@ func configurePerTestExecutionContext(t *testing.T, sharedEnv *ttypes.TestEnviro
}
}
- authorizePerTestWorkflowSignerIfNeeded(t, sharedEnv, ownerAddress)
return execCtx
}
@@ -258,7 +293,7 @@ func deriveExecutionTestID(t *testing.T) string {
return fmt.Sprintf("%s-%d", base, time.Now().UnixNano()%100000)
}
-func authorizePerTestWorkflowSignerIfNeeded(t *testing.T, sharedEnv *ttypes.TestEnvironment, signer common.Address) {
+func authorizePooledSigners(t *testing.T, sharedEnv *ttypes.TestEnvironment, signers []common.Address) {
t.Helper()
registryAddressRef := crecontracts.MustGetAddressRefFromDataStore(
@@ -278,17 +313,25 @@ func authorizePerTestWorkflowSignerIfNeeded(t *testing.T, sharedEnv *ttypes.Test
registry, err := workflow_registry_v2_wrapper.NewWorkflowRegistry(common.HexToAddress(registryAddressRef.Address), rootRegistryChain.SethClient.Client)
require.NoError(t, err, "failed to instantiate workflow registry v2 contract")
- allowed, err := registry.IsAllowedSigner(rootRegistryChain.SethClient.NewCallOpts(), signer)
- require.NoError(t, err, "failed to check signer allowlist status")
- if allowed {
+ var unallowedSigners []common.Address
+ for _, signer := range signers {
+ var allowed bool
+ allowed, err = registry.IsAllowedSigner(rootRegistryChain.SethClient.NewCallOpts(), signer)
+ require.NoError(t, err, "failed to check signer allowlist status")
+ if !allowed {
+ unallowedSigners = append(unallowedSigners, signer)
+ }
+ }
+
+ if len(unallowedSigners) == 0 {
return
}
rootSignerNonceLock.Lock()
defer rootSignerNonceLock.Unlock()
- _, err = rootRegistryChain.SethClient.Decode(registry.UpdateAllowedSigners(rootRegistryChain.SethClient.NewTXOpts(), []common.Address{signer}, true))
- require.NoError(t, err, "failed to authorize per-test signer")
+ _, err = rootRegistryChain.SethClient.Decode(registry.UpdateAllowedSigners(rootRegistryChain.SethClient.NewTXOpts(), unallowedSigners, true))
+ require.NoError(t, err, "failed to authorize per-test signers")
}
func GetDefaultTestConfig(t *testing.T) *ttypes.TestConfig {
@@ -348,11 +391,9 @@ func createEnvironmentIfNotExists(ctx context.Context, relativePathToRepoRoot, e
if !envconfig.LocalCREStateFileExists(relativePathToRepoRoot) {
framework.L.Info().Str("CTF_CONFIGS", os.Getenv("CTF_CONFIGS")).Str("local CRE state file", envconfig.MustLocalCREStateFileAbsPath(relativePathToRepoRoot)).Msg("Local CRE state file does not exist, starting environment...")
- args := []string{"run", ".", "env", "start"} //nolint:prealloc // prealloc here would read horribly
- args = append(args, flags...)
+ args := append([]string{"env", "start"}, flags...)
- cmd := exec.CommandContext(ctx, "go", args...)
- cmd.Dir = environmentDir
+ cmd := resolveCreEnvCommand(ctx, relativePathToRepoRoot, environmentDir, args...)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmdErr := cmd.Run()
diff --git a/system-tests/tests/test-helpers/chip-testsink/server.go b/system-tests/tests/test-helpers/chip-testsink/server.go
index 092c7173496..13fbb53555b 100644
--- a/system-tests/tests/test-helpers/chip-testsink/server.go
+++ b/system-tests/tests/test-helpers/chip-testsink/server.go
@@ -7,11 +7,10 @@ import (
"net"
"time"
+ "github.com/cloudevents/sdk-go/binding/format/protobuf/v2/pb"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
- "github.com/cloudevents/sdk-go/binding/format/protobuf/v2/pb"
-
chippb "github.com/smartcontractkit/chainlink-common/pkg/chipingress/pb"
)
diff --git a/system-tests/tests/test-helpers/chip_ingress_stack_provider.go b/system-tests/tests/test-helpers/chip_ingress_stack_provider.go
index 4df7c537236..018c4d0377e 100644
--- a/system-tests/tests/test-helpers/chip_ingress_stack_provider.go
+++ b/system-tests/tests/test-helpers/chip_ingress_stack_provider.go
@@ -6,6 +6,7 @@ import (
"math/rand"
"os"
"os/exec"
+ "path/filepath"
"strings"
"sync"
"time"
@@ -19,6 +20,7 @@ import (
commonevents "github.com/smartcontractkit/chainlink-protos/workflows/go/common"
workflowevents "github.com/smartcontractkit/chainlink-protos/workflows/go/events"
"github.com/smartcontractkit/chainlink-testing-framework/framework"
+
"github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment/config"
"github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration"
)
@@ -29,7 +31,7 @@ const (
defaultErrorBufferSize = 100
// Kafka timings
- chipIngressStackStartTimeout = 2 * time.Minute // timeout for starting Chip Ingress stack
+ chipIngressStackStartTimeout = 5 * time.Minute // timeout for starting Chip Ingress stack
maxConsumerConnectivityTimeout = 60 * time.Second // max timeout before Kafka consumer reconnection
kafkaSessionTimeoutMs = 20000 // keep it high enough to let Chip Ingress stack messages incoming
messageReadInterval = 50 * time.Millisecond
@@ -82,6 +84,27 @@ func NewChipIngressStack(lggr zerolog.Logger, testConfig *configuration.TestConf
return &ChipIngressStack{cfg: chipConfig, lggr: lggr}, nil
}
+// creEnvBinaryName is the precompiled CRE environment binary built by the CI compile-tests job.
+const creEnvBinaryName = "cre-env"
+
+// resolveCreEnvCommand returns the command to run the CRE environment tool.
+// If a precompiled binary exists at /system-tests/tests/bin/cre-env,
+// it is used instead of "go run ." to avoid recompilation overhead.
+func resolveCreEnvCommand(ctx context.Context, relativePathToRepoRoot, environmentDir string, args ...string) *exec.Cmd {
+ binaryPath := filepath.Join(relativePathToRepoRoot, "system-tests", "tests", "bin", creEnvBinaryName)
+ if info, err := os.Stat(binaryPath); err == nil && info.Mode().IsRegular() && info.Mode().Perm()&0o111 != 0 {
+ framework.L.Info().Str("binary", binaryPath).Msg("Using precompiled cre-env binary")
+ cmd := exec.CommandContext(ctx, binaryPath, args...)
+ cmd.Dir = environmentDir
+ return cmd
+ }
+ framework.L.Info().Msg("No precompiled cre-env binary found, falling back to go run")
+ goArgs := append([]string{"run", "."}, args...)
+ cmd := exec.CommandContext(ctx, "go", goArgs...)
+ cmd.Dir = environmentDir
+ return cmd
+}
+
// startChipIngressStackIfNotRunning starts the Chip Ingress stack if it's not already running.
func startChipIngressStackIfNotRunning(relativePathToRepoRoot, environmentDir string) error {
if config.ChipIngressStateFileExists(relativePathToRepoRoot) {
@@ -93,8 +116,7 @@ func startChipIngressStackIfNotRunning(relativePathToRepoRoot, environmentDir st
ctx, cancel := context.WithTimeout(context.Background(), chipIngressStackStartTimeout)
defer cancel()
- cmd := exec.CommandContext(ctx, "go", "run", ".", "env", "chip-ingress-stack", "start")
- cmd.Dir = environmentDir
+ cmd := resolveCreEnvCommand(ctx, relativePathToRepoRoot, environmentDir, "env", "chip-ingress-stack", "start")
cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
if err := cmd.Run(); err != nil {
@@ -118,8 +140,7 @@ func StopChipIngressStack(relativePathToRepoRoot, environmentDir string) error {
ctx, cancel := context.WithTimeout(context.Background(), chipIngressStackStartTimeout)
defer cancel()
- cmd := exec.CommandContext(ctx, "go", "run", ".", "env", "chip-ingress-stack", "stop")
- cmd.Dir = environmentDir
+ cmd := resolveCreEnvCommand(ctx, relativePathToRepoRoot, environmentDir, "env", "chip-ingress-stack", "stop")
cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
if err := cmd.Run(); err != nil {
diff --git a/system-tests/tests/test-helpers/chip_ingress_stack_provider_test.go b/system-tests/tests/test-helpers/chip_ingress_stack_provider_test.go
new file mode 100644
index 00000000000..9df93777a91
--- /dev/null
+++ b/system-tests/tests/test-helpers/chip_ingress_stack_provider_test.go
@@ -0,0 +1,74 @@
+package helpers
+
+import (
+ "context"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestResolveCreEnvCommand(t *testing.T) {
+ t.Parallel()
+
+ ctx := context.Background()
+
+ tests := []struct {
+ name string
+ createBinary bool
+ binaryMode os.FileMode
+ inputArgs []string
+ expectedPathSuffix string
+ expectedArgsContains string
+ }{
+ {
+ name: "fallback to go run when binary does not exist",
+ createBinary: false,
+ inputArgs: []string{"env", "start"},
+ expectedPathSuffix: "go",
+ expectedArgsContains: "run",
+ },
+ {
+ name: "uses precompiled binary when it exists and is executable",
+ createBinary: true,
+ binaryMode: 0700,
+ inputArgs: []string{"env", "start"},
+ expectedPathSuffix: "cre-env",
+ expectedArgsContains: "env",
+ },
+ {
+ name: "fallback to go run when binary exists but is not executable",
+ createBinary: true,
+ binaryMode: 0600,
+ inputArgs: []string{"env", "start"},
+ expectedPathSuffix: "go",
+ expectedArgsContains: "run",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+
+ tmpDir := t.TempDir()
+ environmentDir := filepath.Join(tmpDir, "core", "scripts", "cre", "environment")
+ require.NoError(t, os.MkdirAll(environmentDir, 0700))
+
+ if tt.createBinary {
+ binDir := filepath.Join(tmpDir, "system-tests", "tests", "bin")
+ require.NoError(t, os.MkdirAll(binDir, 0700))
+ binPath := filepath.Join(binDir, "cre-env")
+ require.NoError(t, os.WriteFile(binPath, []byte("#!/bin/sh\necho ok"), tt.binaryMode))
+ }
+
+ cmd := resolveCreEnvCommand(ctx, tmpDir, environmentDir, tt.inputArgs...)
+ require.NotNil(t, cmd)
+
+ assert.Equal(t, environmentDir, cmd.Dir)
+ assert.Equal(t, tt.expectedPathSuffix, filepath.Base(cmd.Path))
+ assert.Contains(t, cmd.Args, tt.expectedArgsContains)
+ })
+ }
+}
diff --git a/system-tests/tests/test-helpers/chip_testsink_helpers.go b/system-tests/tests/test-helpers/chip_testsink_helpers.go
index a772ecda9f9..cca44a018c5 100644
--- a/system-tests/tests/test-helpers/chip_testsink_helpers.go
+++ b/system-tests/tests/test-helpers/chip_testsink_helpers.go
@@ -19,11 +19,10 @@ import (
"google.golang.org/protobuf/proto"
chippb "github.com/smartcontractkit/chainlink-common/pkg/chipingress/pb"
- "github.com/smartcontractkit/chainlink-testing-framework/framework"
-
commonevents "github.com/smartcontractkit/chainlink-protos/workflows/go/common"
workflowevents "github.com/smartcontractkit/chainlink-protos/workflows/go/events"
workfloweventsv2 "github.com/smartcontractkit/chainlink-protos/workflows/go/v2"
+ "github.com/smartcontractkit/chainlink-testing-framework/framework"
"github.com/smartcontractkit/chainlink/system-tests/lib/cre/chiprouter"
chiptestsink "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/chip-testsink"
diff --git a/system-tests/tests/test-helpers/container_logs.go b/system-tests/tests/test-helpers/container_logs.go
index 82515a768d6..eae0d1c1947 100644
--- a/system-tests/tests/test-helpers/container_logs.go
+++ b/system-tests/tests/test-helpers/container_logs.go
@@ -14,6 +14,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/smartcontractkit/chainlink-testing-framework/framework"
+
ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration"
)
diff --git a/system-tests/tests/test-helpers/t_helpers.go b/system-tests/tests/test-helpers/t_helpers.go
index 6abf9ee9775..243f49c36d8 100644
--- a/system-tests/tests/test-helpers/t_helpers.go
+++ b/system-tests/tests/test-helpers/t_helpers.go
@@ -46,21 +46,6 @@ import (
"github.com/smartcontractkit/chainlink-common/keystore/corekeys/solkey"
commonevents "github.com/smartcontractkit/chainlink-protos/workflows/go/common"
workflowevents "github.com/smartcontractkit/chainlink-protos/workflows/go/events"
-
- solread_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/solana/solread/config"
-
- consensus_negative_config "github.com/smartcontractkit/chainlink/system-tests/tests/regression/cre/consensus/config"
- evmread_negative_config "github.com/smartcontractkit/chainlink/system-tests/tests/regression/cre/evm/evmread-negative/config"
- evmwrite_negative_config "github.com/smartcontractkit/chainlink/system-tests/tests/regression/cre/evm/evmwrite-negative/config"
- logtrigger_negative_config "github.com/smartcontractkit/chainlink/system-tests/tests/regression/cre/evm/logtrigger-negative/config"
- aptoswrite_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/aptos/aptoswrite/config"
- aptoswriteroundtrip_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/aptos/aptoswriteroundtrip/config"
- evmread_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/evm/evmread/config"
- logtrigger_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/evm/logtrigger/config"
- sollogtrigger_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/solana/sollogtrigger/config"
- solwrite_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/solana/solwrite/config"
- ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration"
-
"github.com/smartcontractkit/chainlink-testing-framework/framework"
"github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain"
ns "github.com/smartcontractkit/chainlink-testing-framework/framework/components/simple_node_set"
@@ -78,10 +63,22 @@ import (
"github.com/smartcontractkit/chainlink/system-tests/lib/cre/flags"
creworkflow "github.com/smartcontractkit/chainlink/system-tests/lib/cre/workflow"
crecrypto "github.com/smartcontractkit/chainlink/system-tests/lib/crypto"
+ consensus_negative_config "github.com/smartcontractkit/chainlink/system-tests/tests/regression/cre/consensus/config"
+ evmread_negative_config "github.com/smartcontractkit/chainlink/system-tests/tests/regression/cre/evm/evmread-negative/config"
+ evmwrite_negative_config "github.com/smartcontractkit/chainlink/system-tests/tests/regression/cre/evm/evmwrite-negative/config"
+ logtrigger_negative_config "github.com/smartcontractkit/chainlink/system-tests/tests/regression/cre/evm/logtrigger-negative/config"
http_config "github.com/smartcontractkit/chainlink/system-tests/tests/regression/cre/http/config"
httpaction_negative_config "github.com/smartcontractkit/chainlink/system-tests/tests/regression/cre/httpaction-negative/config"
+ aptoswrite_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/aptos/aptoswrite/config"
+ aptoswriteroundtrip_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/aptos/aptoswriteroundtrip/config"
+ evmread_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/evm/evmread/config"
+ logtrigger_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/evm/logtrigger/config"
httpaction_smoke_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/httpaction/config"
+ sollogtrigger_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/solana/sollogtrigger/config"
+ solread_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/solana/solread/config"
+ solwrite_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/solana/solwrite/config"
vaultsecret_config "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/vaultsecret/config"
+ ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration"
)
const WorkflowEngineInitErrorLog = "Workflow Engine initialization failed"
@@ -951,6 +948,11 @@ func ParallelEnabled() bool {
// for its expected log.
const StellarWorkflowTimeout = 4 * time.Minute
+// DefaultPollInterval is the standard tick for require.Eventually loops.
+// Lower than the historical 5s to reduce post-success latency; the loop's
+// timeout (not this tick) still bounds worst-case waits.
+const DefaultPollInterval = 2 * time.Second
+
// stellarCronScheduleEnvVar overrides the cron schedule for Stellar test workflows.
const stellarCronScheduleEnvVar = "CRE_STELLAR_CRON_SCHEDULE"
diff --git a/system-tests/tests/test-helpers/t_helpers_soak.go b/system-tests/tests/test-helpers/t_helpers_soak.go
index 1484c2ae0a8..d45e376bc7f 100644
--- a/system-tests/tests/test-helpers/t_helpers_soak.go
+++ b/system-tests/tests/test-helpers/t_helpers_soak.go
@@ -22,8 +22,8 @@ import (
"github.com/smartcontractkit/chainlink-testing-framework/framework"
ns "github.com/smartcontractkit/chainlink-testing-framework/framework/components/simple_node_set"
"github.com/smartcontractkit/chainlink-testing-framework/seth"
- keystone_changeset "github.com/smartcontractkit/chainlink/deployment/keystone/changeset"
+ keystone_changeset "github.com/smartcontractkit/chainlink/deployment/keystone/changeset"
"github.com/smartcontractkit/chainlink/system-tests/lib/cre"
crecontracts "github.com/smartcontractkit/chainlink/system-tests/lib/cre/contracts"
"github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment/blockchains/evm"
@@ -201,6 +201,7 @@ func configureAdditionalWorkflowSigners(t *testing.T, sharedEnv *ttypes.TestEnvi
}
out := make([]ttypes.PerTestDeployKey, 0, numSigners)
+ signers := make([]common.Address, 0, numSigners)
for keyIdx := range numSigners {
ownerAddress, privateKey, addrErr := crecrypto.GenerateNewKeyPair()
require.NoError(t, addrErr, "failed to generate workflow signer key pair")
@@ -235,13 +236,15 @@ func configureAdditionalWorkflowSigners(t *testing.T, sharedEnv *ttypes.TestEnvi
}
require.NotNil(t, registryClient, "failed to build registry chain seth client for signer %d", keyIdx)
- authorizePerTestWorkflowSignerIfNeeded(t, sharedEnv, ownerAddress)
+ signers = append(signers, ownerAddress)
out = append(out, ttypes.PerTestDeployKey{
OwnerAddress: ownerAddress,
RegistryClient: registryClient,
})
}
+ authorizePooledSigners(t, sharedEnv, signers)
+
return out
}