diff --git a/.gitignore b/.gitignore index 83b4dc2ec..48b7899d5 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ docs/sphinx/_toc.yml # Build distributions dist/ + +# Local sample output +sample_reports/ diff --git a/cvs/cli_plugins/list_plugin.py b/cvs/cli_plugins/list_plugin.py index 0ad0d66a4..2c3a7f695 100644 --- a/cvs/cli_plugins/list_plugin.py +++ b/cvs/cli_plugins/list_plugin.py @@ -44,7 +44,14 @@ def discover_tests(): test_map[pkg_name] = {} for root, dirs, files in os.walk(tests_dir): for file in files: - if file.endswith(".py") and file != "__init__.py": + # Skip pytest infra (conftest.py) and private helpers + # (e.g. _shared.py): they are not selectable suites. + if ( + file.endswith(".py") + and file != "__init__.py" + and file != "conftest.py" + and not file.startswith("_") + ): rel_path = os.path.relpath(os.path.join(root, file), tests_dir) module_parts = os.path.splitext(rel_path)[0].split(os.sep) # Module path: . diff --git a/cvs/cli_plugins/run_plugin.py b/cvs/cli_plugins/run_plugin.py index 72a8c8d11..ab9936b0f 100644 --- a/cvs/cli_plugins/run_plugin.py +++ b/cvs/cli_plugins/run_plugin.py @@ -23,8 +23,12 @@ def get_parser(self, subparsers): ) parser.add_argument( "--log-file", - default="/tmp/cvs/test.log", - help="Pytest: Path to file for logging output (default: /tmp/cvs/test.log)", + default=None, + metavar="PATH", + help=( + "Pytest: write logging output to this file (optional). " + "Parent directories are created automatically when set." + ), ) parser.add_argument( "--log-level", diff --git a/cvs/cli_plugins/unittests/test_run_plugin.py b/cvs/cli_plugins/unittests/test_run_plugin.py index ef73ad42c..57fbfa1a9 100644 --- a/cvs/cli_plugins/unittests/test_run_plugin.py +++ b/cvs/cli_plugins/unittests/test_run_plugin.py @@ -79,6 +79,35 @@ def test_run_test_multiple_functions(self, mock_exit, mock_pytest_main): mock_pytest_main.assert_called_once_with(expected_args) mock_exit.assert_called_once_with(0) + @patch("cvs.cli_plugins.run_plugin.pytest.main") + @patch("cvs.cli_plugins.run_plugin.sys.exit") + def test_run_test_omits_log_file_when_not_set(self, mock_exit, mock_pytest_main): + """No --log-file is passed to pytest when the user does not request file logging.""" + args = MagicMock() + args.test = "agfhc_cvs" + args.function = [] + args.cluster_file = "/path/to/cluster.json" + args.config_file = "/path/to/config.json" + args.html = None + args.self_contained_html = False + args.log_file = None + args.log_level = None + args.capture = None + args.extra_pytest_args = [] + + mock_pytest_main.return_value = 0 + + with patch.object(self.plugin, "get_test_file", return_value="/mock/path/test.py"): + self.plugin.run(args) + + expected_args = [ + "/mock/path/test.py", + "--cluster_file=/path/to/cluster.json", + "--config_file=/path/to/config.json", + ] + mock_pytest_main.assert_called_once_with(expected_args) + mock_exit.assert_called_once_with(0) + if __name__ == "__main__": unittest.main() diff --git a/cvs/conftest.py b/cvs/conftest.py index e46945821..0b739c0a4 100644 --- a/cvs/conftest.py +++ b/cvs/conftest.py @@ -6,6 +6,7 @@ """ import importlib.metadata +import logging import sys from pathlib import Path @@ -13,9 +14,11 @@ from cvs.lib.report_plugins import HtmlReportManager +log = logging.getLogger(__name__) -@pytest.hookimpl(tryfirst=True) -def pytest_configure(config): + +def _sync_suite_name_from_args(config): + """Derive suite stem from the first ``*.py`` target in ``config.args``.""" suite_name = "test" for arg in config.args: bare = arg.split("::")[0] @@ -24,7 +27,96 @@ def pytest_configure(config): break config._suite_name = suite_name config._test_html_dir = f"{suite_name}_html" + + +def _ensure_html_report_manager(config): + """Create ``HtmlReportManager`` once; safe if ``pytest_configure`` did not run.""" + _sync_suite_name_from_args(config) + mgr = getattr(config, "_html_report_manager", None) + if mgr is not None: + return mgr + config._html_report_manager = HtmlReportManager(config) + return config._html_report_manager + + +def _auto_register_inference_suite_report(config): + from cvs.lib.report.auto_register import try_auto_register_inference_suite_report + + _sync_suite_name_from_args(config) + return try_auto_register_inference_suite_report(config) + + +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config): + _ensure_html_report_manager(config) + _auto_register_inference_suite_report(config) + + +@pytest.fixture(scope="session", autouse=True) +def _cvs_inference_suite_report_session(request): + """Initialize the session report store when a suite preset is registered.""" + from cvs.lib.report.registry import clear_session_results, get_suite_report_config + from cvs.lib.report.types import InferenceReportConfig + + if not isinstance(get_suite_report_config(request.config), InferenceReportConfig): + yield + return + + clear_session_results() + yield + + +@pytest.fixture(scope="module", autouse=True) +def _cvs_inference_suite_report_bind_module(request, _cvs_inference_suite_report_session): + """Bind module-scoped suite fixtures into the session store at module teardown.""" + from cvs.lib.report.registry import bind_session_results, get_suite_report_config + from cvs.lib.report.types import InferenceReportConfig + + if not isinstance(get_suite_report_config(request.config), InferenceReportConfig): + yield + return + + inf_res_dict = None + variant_config = None + lifecycle = None + try: + inf_res_dict = request.getfixturevalue("inf_res_dict") + except pytest.FixtureLookupError: + log.warning( + "Inference suite report preset registered but inf_res_dict fixture is missing; " + "session-end report will be skipped" + ) + yield + return + try: + variant_config = request.getfixturevalue("variant_config") + except pytest.FixtureLookupError: + log.warning( + "Inference suite report preset registered but variant_config fixture is missing; " + "session-end report will be skipped" + ) + yield + return + try: + lifecycle = request.getfixturevalue("lifecycle") + except pytest.FixtureLookupError: + log.warning( + "Inference suite report preset registered but lifecycle fixture is missing; " + "session-end report will be skipped" + ) + yield + return + + def _bind_at_module_end(): + bind_session_results( + inf_res_dict=inf_res_dict, + variant_config=variant_config, + lifecycle=lifecycle, + ) + + request.addfinalizer(_bind_at_module_end) + yield # Add all additional cmd line arguments for the script @@ -93,7 +185,8 @@ def pytest_metadata(metadata): # Prepare a clean per-run log directory before tests start. def pytest_sessionstart(session): - session.config._html_report_manager.setup_log_dir() + _auto_register_inference_suite_report(session.config) + _ensure_html_report_manager(session.config).setup_log_dir() # Capture each test report and attach a per-test external log link. @@ -101,7 +194,19 @@ def pytest_sessionstart(session): def pytest_runtest_makereport(item, call): # noqa: ARG001 outcome = yield report = outcome.get_result() - report.extras = item.config._html_report_manager.write_test_log(report, item.originalname) + report.extras = _ensure_html_report_manager(item.config).write_test_log(report, item.originalname) + + from cvs.lib.report.registry import get_suite_report_config + from cvs.lib.report.types import InferenceReportConfig + + if isinstance(get_suite_report_config(item.config), InferenceReportConfig): + from cvs.lib.report.inference_wiring import ( + attach_inference_suite_lifecycle_table, + attach_inference_suite_report_row_extra, + ) + + attach_inference_suite_lifecycle_table(item, report) + attach_inference_suite_report_row_extra(item, report) # Replace inline pytest-html log content with a short externalized-log message. @@ -118,4 +223,6 @@ def pytest_html_results_summary(prefix, summary, postfix): @pytest.hookimpl(hookwrapper=True) def pytest_sessionfinish(session, exitstatus): # noqa: ARG001 yield # wait for pytest-html and all other plugins to finish writing the report - session.config._html_report_manager.create_zip_bundle(session) + mgr = _ensure_html_report_manager(session.config) + mgr.generate_suite_reports(session) + mgr.create_zip_bundle(session) diff --git a/cvs/core/orchestrators/container.py b/cvs/core/orchestrators/container.py index 0d1d2026b..a8a6f1881 100644 --- a/cvs/core/orchestrators/container.py +++ b/cvs/core/orchestrators/container.py @@ -618,6 +618,10 @@ def exec(self, cmd, hosts=None, timeout=None, detailed=False): return self.runtime.exec(self.container_id, cmd, hosts, timeout, detailed) + def exec_on_host(self, cmd, hosts=None, timeout=None, detailed=False): + """Execute command on the cluster host OS (SSH), not inside the container.""" + return super().exec(cmd, hosts=hosts, timeout=timeout, detailed=detailed) + def exec_on_head(self, cmd, timeout=None): """ Execute command directly on head node (baremetal). diff --git a/cvs/core/orchestrators/unittests/test_container.py b/cvs/core/orchestrators/unittests/test_container.py index ee9eccec2..5e4479090 100644 --- a/cvs/core/orchestrators/unittests/test_container.py +++ b/cvs/core/orchestrators/unittests/test_container.py @@ -15,7 +15,7 @@ # patched once in setUp (not per method); _make() returns a fresh orch + runtime mock. import unittest -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, call, patch from cvs.core.orchestrators.factory import OrchestratorConfig, _resolve_container_lifetime from cvs.core.orchestrators.container import ContainerOrchestrator @@ -258,6 +258,21 @@ def test_setup_sshd_multinode_attempts_setup(self): } self.assertTrue(orch.setup_sshd()) self.assertTrue(runtime.exec.called) + cmds = [call.args[0] for call in runtime.exec.call_args_list] + self.assertFalse(any("apt-get" in cmd for cmd in cmds)) + + @patch("time.sleep", lambda *_a, **_k: None) + def test_setup_sshd_multinode_fails_when_sshd_missing(self): + orch, runtime = self._make(lifetime="per_run") + orch.container_id = "cvs_iter_test" + + def _exec_side_effect(cmd, **kwargs): + if "command -v /usr/sbin/sshd" in cmd: + return {"10.0.0.1": {"exit_code": 127, "output": "missing sshd"}} + return {"10.0.0.1": {"exit_code": 0}} + + runtime.exec.side_effect = _exec_side_effect + self.assertFalse(orch.setup_sshd()) # ------------------------------------------------------------------ # teardown_containers lifetime branching diff --git a/cvs/core/runtimes/docker.py b/cvs/core/runtimes/docker.py index 3b902072f..a45f49b89 100644 --- a/cvs/core/runtimes/docker.py +++ b/cvs/core/runtimes/docker.py @@ -121,6 +121,14 @@ def setup_containers( else: self.log.info(f"Image {container_config['image']} already exists, skipping tar load") + if not self.check_image_exists(image): + self.log.info(f"Image {image} not present on all hosts; pulling before start") + pull_result = self.pull_image(image, timeout=600) + failed_pull = [host for host, res in pull_result.items() if res.get('exit_code') != 0] + if failed_pull: + self.log.error(f"Failed to pull image on hosts: {failed_pull}") + return False + cmd = f"sudo docker run -d --name {container_name} {all_args_str} {image} sleep infinity" self.log.info(f"Starting long-running containers on {len(self.orchestrator.hosts)} nodes: {container_name}") @@ -130,7 +138,7 @@ def setup_containers( remove_cmd = f"sudo docker rm -f {container_name} || true" self.orchestrator.all.exec(remove_cmd, timeout=30, print_console=False) - result = self.orchestrator.all.exec(cmd, timeout=60, detailed=True) + result = self.orchestrator.all.exec(cmd, timeout=120, detailed=True) # Check if all hosts started successfully success = all(output['exit_code'] == 0 for output in result.values()) @@ -278,6 +286,13 @@ def _build_runtime_args(runtime_args_config): return args + def pull_image(self, image_name, timeout=None): + """Pull container image on all hosts.""" + timeout = timeout or 600 + cmd = f"sudo docker pull {shlex.quote(image_name)}" + self.log.info(f"Pulling image on all hosts: {image_name}") + return self.orchestrator.all.exec(cmd, timeout=timeout, detailed=True) + def load_image(self, tar_path, timeout=None): """Load container image from tar file on all hosts.""" cmd = f"sudo docker load < {tar_path}" diff --git a/cvs/core/runtimes/unittests/test_docker.py b/cvs/core/runtimes/unittests/test_docker.py index 18e7dbfce..e9743a015 100644 --- a/cvs/core/runtimes/unittests/test_docker.py +++ b/cvs/core/runtimes/unittests/test_docker.py @@ -130,6 +130,29 @@ def test_cmd_never_contains_gpus_all(self): f"[{label}] '--gpus all' must never appear in docker cmd:\n{captured[0]}", ) + def test_pulls_image_when_missing_before_run(self): + calls = [] + + def _fake_exec(cmd, timeout=None, detailed=False, print_console=True): + calls.append(cmd) + if "docker images" in cmd and "grep" in cmd: + return {"host1": {"output": "", "exit_code": 1}} + return {"host1": {"output": "", "exit_code": 0}} + + orchestrator = MagicMock() + orchestrator.hosts = ["host1"] + orchestrator.all.exec.side_effect = _fake_exec + rt = DockerRuntime(MagicMock(), orchestrator) + + result = rt.setup_containers( + container_config=_container_config(), + container_name="cvs_iter_test", + volumes=["/home/u:/workspace"], + ) + self.assertTrue(result) + self.assertTrue(any("docker pull" in c for c in calls)) + self.assertTrue(any(c.startswith("sudo docker run") for c in calls)) + if __name__ == "__main__": unittest.main() diff --git a/cvs/input/cluster_file/inferencex_atom_cluster.json b/cvs/input/cluster_file/inferencex_atom_cluster.json new file mode 100644 index 000000000..d923988ec --- /dev/null +++ b/cvs/input/cluster_file/inferencex_atom_cluster.json @@ -0,0 +1,35 @@ +{ + "_comment": "InferenceX ATOM cluster template (container backend). Copy to ~/input/cluster_file/inferencex_atom_cluster.json and edit placeholders. head_node_dict.mgmt_ip MUST be the rank-0 GPU node VPC IP (same as variant params.master_addr for nnodes>1) — not the pytest jumphost. Trim node_dict to one host for single-node variants. Variant config overrides container.image/name/volumes; cluster container block is the fallback default.", + "orchestrator": "container", + "username": "{user-id}", + "priv_key_file": "/home/{user-id}/.ssh/cluster_id_ed25519", + "head_node_dict": { + "mgmt_ip": "{head-node-ip}" + }, + "env_vars": {}, + "_env_vars_comment": "Optional host env exported on each GPU node before container setup (e.g. ROCm install on host). Usually empty when the workload image carries ROCm/vLLM.", + "node_dict": { + "{head-node-ip}": { + "bmc_ip": "NA", + "vpc_ip": "{head-node-ip}" + }, + "{worker-node-ip}": { + "bmc_ip": "NA", + "vpc_ip": "{worker-node-ip}" + } + }, + "container": { + "lifetime": "per_run", + "image": "rocm/atom-dev:latest", + "name": "inferencex_atom", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G" + } + } + } +} diff --git a/cvs/input/config_file/inference/inferencemax/mi300x_inferencemax_gpt_oss_120b_single.json b/cvs/input/config_file/inference/inferencemax/mi300x_inferencemax_gpt_oss_120b_single.json deleted file mode 100644 index 18a4d5a8b..000000000 --- a/cvs/input/config_file/inference/inferencemax/mi300x_inferencemax_gpt_oss_120b_single.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "config": { - "container_image": "rocm/7.0:rocm7.0_ubuntu_22.04_vllm_0.10.1_instinct_20250927_rc1", - "container_name": "inference_max_rocm", - "_example_nnodes": "4", - "nnodes": "4", - "inferencemax_repo": "https://github.com/SemiAnalysisAI/InferenceX.git", - "benchmark_script_repo": "https://github.com/kimbochen/bench_serving.git", - "hf_token_file": "/home/{user-id}/.hf_token", - "shm_size": "128G", - "log_dir": "/home/{user-id}/LOGS", - "container_config": { - "device_list": [ - "/dev/dri", - "/dev/kfd" - ], - "volume_dict": { - "/home/{user-id}": "/home/{user-id}" - }, - "env_dict": {} - } - }, - "benchmark_params": { - "gpt-oss-120b": { - "backend": "vllm", - "base_url": "http://0.0.0.0", - "port_no": "8000", - "_example_dataset_name": "sharegpt|hf|random|sonnet|burstgpt", - "dataset_name": "random", - "max_concurrency": "64", - "model": "openai/gpt-oss-120b", - "num_prompts": "1000", - "input_sequence_length": "8192", - "output_sequence_length": "1024", - "burstiness": "1.0", - "seed": "0", - "max_model_length": "9216", - "random_range_ratio": "0.8", - "random_prefix_len": "0", - "tensor_parallelism": "8", - "_example_tokenizer_mode": "auto|slow|mistral|custom", - "tokenizer_mode": "auto", - "percentiles_metrics": "ttft,tpot,itl,e2el", - "metric_percentiles": "99", - "server_script": "gptoss_fp4_mi300x.sh", - "bench_serv_script": "benchmark_serving.py", - "result_dict": { - "output_throughput_per_sec": "4200", - "mean_ttft_ms": "500", - "mean_tpot_ms": "15" - } - } - } -} \ No newline at end of file diff --git a/cvs/input/config_file/inference/inferencemax/mi355x_inferencemax_gpt_oss_120b_single.json b/cvs/input/config_file/inference/inferencemax/mi355x_inferencemax_gpt_oss_120b_single.json deleted file mode 100644 index 516efca2b..000000000 --- a/cvs/input/config_file/inference/inferencemax/mi355x_inferencemax_gpt_oss_120b_single.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "config": { - "container_image": "rocm/7.0:rocm7.0_ubuntu_22.04_vllm_0.10.1_instinct_20250927_rc1", - "container_name": "inference_max_rocm", - "_example_nnodes": "4", - "nnodes": "4", - "inferencemax_repo": "https://github.com/SemiAnalysisAI/InferenceX.git", - "benchmark_script_repo": "https://github.com/kimbochen/bench_serving.git", - "hf_token_file": "/home/{user-id}/.hf_token", - "shm_size": "128G", - "log_dir": "/home/{user-id}/LOGS", - "container_config": { - "device_list": [ - "/dev/dri", - "/dev/kfd" - ], - "volume_dict": { - "/home/{user-id}": "/home/{user-id}" - }, - "env_dict": {} - } - }, - "benchmark_params": { - "gpt-oss-120b": { - "backend": "vllm", - "base_url": "http://0.0.0.0", - "port_no": "8000", - "_example_dataset_name": "sharegpt|hf|random|sonnet|burstgpt", - "dataset_name": "random", - "max_concurrency": "64", - "model": "openai/gpt-oss-120b", - "num_prompts": "1000", - "input_sequence_length": "8192", - "output_sequence_length": "1024", - "burstiness": "1.0", - "seed": "0", - "max_model_length": "9216", - "random_range_ratio": "0.8", - "random_prefix_len": "0", - "tensor_parallelism": "8", - "_example_tokenizer_mode": "auto|slow|mistral|custom", - "tokenizer_mode": "auto", - "percentiles_metrics": "ttft,tpot,itl,e2el", - "metric_percentiles": "99", - "server_script": "gptoss_fp4_mi355x.sh", - "bench_serv_script": "benchmark_serving.py", - "result_dict": { - "output_throughput_per_sec": "4200", - "mean_ttft_ms": "500", - "mean_tpot_ms": "15" - } - } - } -} \ No newline at end of file diff --git a/cvs/input/config_file/inference/inferencex_atom/README.md b/cvs/input/config_file/inference/inferencex_atom/README.md new file mode 100644 index 000000000..eac346cf9 --- /dev/null +++ b/cvs/input/config_file/inference/inferencex_atom/README.md @@ -0,0 +1,383 @@ +# InferenceX ATOM variants + +W1 **DeepSeek R1 FP8** on 8× GPU, ISL=OSL=1024, TP8. + +## Layout + +**In the CVS repo**, all variants live as flat sibling pairs in **this directory**: + +```text +{gpu}_inferencex-atom_{model}_{precision}[_{mode}].json +{gpu}_inferencex-atom_{model}_{precision}[_{mode}]_threshold.json +``` + +Same convention as ``inference/vllm/`` (for example ``mi300x_vllm_llama31-70b_fp8_single.json`` / ``…_distributed.json``): flat sibling pairs, no ``_config`` suffix on the main JSON. + +**On your lab machine** (`~/input/config_file/inference/inferencex_atom/`), copy each variant into its **own subdirectory** so only one `*threshold.json` sits next to the config you pass to `--config_file`. `substitute_config` globs the config's parent directory; multiple `*threshold.json` files there raises `ValueError: multiple *threshold.json files … (ambiguous)`. + +```text +~/input/.../inferencex_atom/single/ # single-node config + threshold only +~/input/.../inferencex_atom/distributed/ # vllm_atom PP=2 config + threshold only +~/input/.../inferencex_atom/sglang_distributed/ # sglang PP=2 config + threshold only +``` + +Each shipped config sets `"threshold_json"` to the sibling threshold filename (resolved relative to the config directory). You may also use an absolute path (vLLM-style). + +Legacy nested layouts (`deepseek_r1_fp8_mi300x_atom_perf/`, `inferencemax/`, etc.) are **removed** from the repo tree. Use only the flat stems below. + +**Config filename example:** `mi300x_inferencex-atom_deepseek-r1_fp8_single.json` + +| Variant | GPU | Driver | Notes | +|---------|-----|--------|-------| +| `mi300x_inferencex-atom_deepseek-r1_fp8_single` | MI300X | `atom` | W1 single-node, portable min-SLO thresholds, server reuse across sweep | +| `mi300x_inferencex-atom_deepseek-r1_fp8_baseline_sweep` | MI300X | `atom` | **DTNI baseline matrix:** 1K/1K + 8K/1K × C=4–256 (14 cells); `max_model_length=10240` | +| `mi300x_inferencex-atom_deepseek-r1_fp8_baseline_sweep_distributed` | MI300X | `vllm_atom` | **2-node** DTNI baseline (14 cells); `PP=2`, scaling gates | +| `mi300x_inferencex-atom_deepseek-r1_fp8_distributed` | MI300X | `vllm_atom` | W1 **2-node** PP=2; `enforce_thresholds: true` after lab recalibration | +| `mi300x_inferencex-atom_deepseek-r1_fp8_sglang_distributed` | MI300X | `sglang` | W1 **2-node** PP=2; `enforce_thresholds: false` until lab confirm | +| `mi300x_inferencex-atom_deepseek-r1_fp8_mtp3` | MI300X | `atom` | W1 FP8+MTP3 | +| `mi355x_inferencex-atom_deepseek-r1_fp8_single` | MI355X | `atom` | W1 single-node (CI seeds, `enforce_thresholds: false`) | +| `mi355x_inferencex-atom_deepseek-r1_fp8_baseline_sweep` | MI355X | `atom` | **DTNI baseline matrix:** 1K/1K + 8K/1K × C=4–256 (14 cells); threshold seeds, `enforce_thresholds: false` | +| `mi355x_inferencex-atom_deepseek-r1_fp8_distributed` | MI355X | `vllm_atom` | W1 **2-node** PP=2; `enforce_thresholds: false` until lab confirm | +| `mi355x_inferencex-atom_deepseek-r1_fp8_mtp3` | MI355X | `atom` | W1 FP8+MTP3 | +| `mi300x_inferencex-atom_gpt-oss-120b_bf16` | MI300X | `vllm` | GPT-OSS uplift placeholder | +| `mi355x_inferencex-atom_gpt-oss-120b_bf16` | MI355X | `vllm` | GPT-OSS uplift placeholder | + +**Removed:** `*_smoke` variant (use `-k` on `single`, `distributed`, or `sglang_distributed` for a one-cell smoke). **Removed:** bare `driver=atom` multinode PP — use `vllm_atom` or `sglang` distributed stems above. + +ATOM server CLI for **`driver=atom`** lives in `roles.server.atom_args`. Multinode **PP=2** variants use **`driver=vllm_atom`** (`roles.server.serve_args`) or **`driver=sglang`** (`roles.server.sglang_args`). MTP3 variants also set `params.bench_extra_args`. + +## Execution drivers (`params.driver`) + +Standalone ATOM has **no native pipeline parallel**. Multinode PP validation requires a framework coordinator: + +| Driver | When to use | Server | Multinode PP | +|--------|-------------|--------|--------------| +| `atom` | W1 single-node (`*_single`, baseline sweep, MTP3) | `atom.entrypoints.openai_server` | No — single host only | +| `vllm_atom` | **2-node PP=2** (shipped multinode stems) | `vllm serve` + ATOM ROCm env | Yes — vLLM `--pipeline-parallel-size`, `--node-rank` | +| `sglang` | **2-node PP=2** SGLang path | `sglang.launch_server` | Yes — `--pp-size`, `--dist-init-addr` | +| `vllm` | GPT-OSS uplift placeholder only | `vllm serve` | Same PP flags as `vllm_atom` when configured | + +**Before a multinode PP lab run**, set in the copied config: + +- `container.image` — vLLM+ATOM or SGLang-capable image (shipped configs use ``) +- `params.master_addr` — head node VPC IP (replace `{head-node-ip}`) + +Multinode fabric is probed once per run in `test_discover_topology` (or lazily on first `build_server_cmd` if that test is omitted from a smoke `-k` filter). Probes run on the **cluster host OS** (not inside the container), where `ip` and `ibv_devinfo` are available: + +- `roles.server.ib_hca_devices: "auto"` (default) → `NCCL_IB_HCA` from `ibv_devinfo -l` +- `roles.server.ib_netdev: "auto"` (default on distributed stems) → `GLOO_SOCKET_IFNAME` / `NCCL_SOCKET_IFNAME` from the cluster IP on each node + +Override `ib_netdev` only when auto-discovery fails (asymmetric interface names) or you need a non-default NIC. Do **not** set `mlx5_*` — those are IB HCAs, not IP netdevs. + +Threshold cell keys for multinode PP: `ISL=…,OSL=…,TP=8,PP=2,NNODES=2,CONC=…`. + +**Model cache path:** shipped configs set `paths.models_dir` to `/home/models` and mount `/home/models:/home/models` into the container. Logs and HF token paths still use `{shared_fs}` under the SSH user home. + +## Cluster file + +Ship one template: `cvs/input/cluster_file/inferencex_atom_cluster.json`. Copy it to `~/input/cluster_file/inferencex_atom_cluster.json` and edit IPs, SSH user, and key path. + +**Host count must match the variant:** `len(node_dict)` must equal `params.nnodes` in the config you pass to `--config_file`. + +| Variant type | `params.nnodes` | `node_dict` | +|--------------|-----------------|-------------| +| Single-node (`*_single`, baseline sweep, MTP3) | `1` (default) | **Head node only** — remove the worker entry | +| Multinode PP (`*_distributed`, `*_baseline_sweep_distributed`, `*_sglang_distributed`) | `2` | Head + worker; use lab subdirs `distributed/` or `sglang_distributed/` | + +For multinode PP variants, set `params.master_addr` to the head VPC IP and a coordinator-capable `container.image`. Fabric netdev/HCAs are discovered in `test_discover_topology` unless overridden. `test_setup_sshd` runs when `len(node_dict) > 1`. + +## Shared suite helpers (reusable by other inference suites) + +| Module | Purpose | +|--------|---------| +| `cvs/lib/inference/utils/inference_suite_lifecycle.py` | Lifecycle stage tests, `InferenceLifecycle`, pytest HTML hooks | +| `cvs/lib/inference/utils/inference_suite_results_table.py` | Configurable results table (`make_print_results_table`) | +| `cvs/lib/inference/unittests/fake_orch.py` | `FakeOrch` for Job parse unit tests | + +`inferencex_atom` imports these today; `vllm_single` may adopt them in a follow-up without duplicating code. + +## Pytest layout + +1. `test_launch_container` → `test_setup_sshd` → `test_model_fetch` +2. `test_inferencex_atom_inference` (per sweep cell; reuses server when `reuse_server_across_sweep: true`) +3. `test_cell_metrics` (one HTML row per **metric tier** per cell: throughput, ttft, tpot, health, record) +4. `test_print_results_table` → `test_teardown` + +W1 MI300X single with two concurrency cells expects **~17** pytest rows (not one row per scalar metric). + +## Before the first lab run + +- On the **launcher** host after `git checkout` / `git pull`: run **`make install` first**, then **`source .cvs_venv/bin/activate`**. Do not activate `.cvs_venv` before `make install` — the Makefile manages that venv and install can fail if it is already active. + +```bash +cd ~/cvs +git fetch origin hnimrama/ix-atom-multinode +git reset --hard origin/hnimrama/ix-atom-multinode +make install +source .cvs_venv/bin/activate +``` + +- Edit `~/input/cluster_file/inferencex_atom_cluster.json`: node IPs, `username`, `priv_key_file`. Trim `node_dict` to one host for single-node variants. +- **Launcher vs GPU node:** CVS pytest runs on the launcher; `ContainerOrchestrator` SSHes to cluster nodes and runs `sudo docker` there. Local Docker on the launcher is not used. Prerequisites split by host: + + | Item | Launcher | GPU node (cluster `mgmt_ip`) | + |------|----------|------------------------------| + | `cvs run`, venv, `~/input/`, `~/cvs_results/` | Yes | No | + | `priv_key_file`, `~/.hf_token` (read locally by pytest) | Yes | No | + | `/home/models` (when `model.remote: 0`) | No | Yes | + | `rocm/atom-dev` image, `sudo docker` | No | Yes | + | `~/LOGS/` (server/bench logs via volume mount) | No | Yes | + +- Preflight from the launcher: `ssh -i ~/.ssh/ @ 'sudo docker images | grep atom-dev; du -sh /home/models'` + +## W1 single-node (MI300X, `driver=atom`) + +Two concurrency cells (C=128, C=256), 1000 prompts. Second cell reuses the ATOM server when `reuse_server_across_sweep: true`. For a quick smoke, add `-k "w1_1k_1k-conc128"`. + +```bash +cd ~/cvs +make install # after git pull only; run before activating venv +source .cvs_venv/bin/activate + +SINGLE_DIR=~/input/config_file/inference/inferencex_atom/single +mkdir -p "$SINGLE_DIR" + +cvs copy-config inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_single.json \ + --output "$SINGLE_DIR/mi300x_inferencex-atom_deepseek-r1_fp8_single.json" +cvs copy-config inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_single_threshold.json \ + --output "$SINGLE_DIR/mi300x_inferencex-atom_deepseek-r1_fp8_single_threshold.json" + +TS=$(date +%Y%m%d_%H%M%S) +HTML=~/cvs_results/${TS}_ix-atom-w1-single_mi300x.html +LOG=~/cvs_results/${TS}_ix-atom-w1-single_mi300x.log + +cvs run inferencex_atom \ + --cluster_file ~/input/cluster_file/inferencex_atom_cluster.json \ + --config_file "$SINGLE_DIR/mi300x_inferencex-atom_deepseek-r1_fp8_single.json" \ + --html="$HTML" \ + --self-contained-html \ + --log-file="$LOG" \ + -vvv -s + +echo "HTML: $HTML" +echo "LOG: $LOG" +``` + +When `--html` is set, the **IX Run Deck** (`inferencex_atom_run_deck.html`, `.json`, +`_viewer.html`) is generated at session end and bundled into the pytest zip. +See `cvs/lib/report/README.md` for wiring other suites. Open the pytest HTML **Reports** +section for links. Render-only; does not affect gates. + +## W1 perf baseline sweep (MI300X) — DTNI matrix + +DTNI baseline matrix: **1K/1K** and **8K/1K** at **C=4, 8, 16, 32, 64, 128, 256** (14 cells). `max_model_length=10240`. Expect a long run (~several hours); server is reused within each shape. + +```bash +cd ~/cvs +make install +source .cvs_venv/bin/activate + +BASELINE_DIR=~/input/config_file/inference/inferencex_atom/baseline_sweep +mkdir -p "$BASELINE_DIR" + +cvs copy-config inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_baseline_sweep.json \ + --output "$BASELINE_DIR/mi300x_inferencex-atom_deepseek-r1_fp8_baseline_sweep.json" +cvs copy-config inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_baseline_sweep_threshold.json \ + --output "$BASELINE_DIR/mi300x_inferencex-atom_deepseek-r1_fp8_baseline_sweep_threshold.json" + +TS=$(date +%Y%m%d_%H%M%S) +HTML=~/cvs_results/${TS}_ix-atom-baseline-sweep_mi300x.html +LOG=~/cvs_results/${TS}_ix-atom-baseline-sweep_mi300x.log + +cvs run inferencex_atom \ + --cluster_file ~/input/cluster_file/inferencex_atom_cluster.json \ + --config_file "$BASELINE_DIR/mi300x_inferencex-atom_deepseek-r1_fp8_baseline_sweep.json" \ + --html="$HTML" \ + --self-contained-html \ + --log-file="$LOG" \ + -vvv -s + +echo "HTML: $HTML" +echo "LOG: $LOG" +``` + +## W1 perf baseline sweep multinode (MI300X, 2-node) + +Same **14-cell** DTNI matrix as single-node baseline sweep (1K/1K + 8K/1K × C=4–256), with `nnodes=2`, **`driver=vllm_atom`**, **`pipeline_parallel_size=2`** (true pipeline parallel via vLLM coordinator + ATOM kernels; cell keys use `PP=2`), and `scaling.efficiency_pct` gates. Set `roles.server.ib_netdev` and a vLLM+ATOM container image before lab run. Expect a long run (~4–8 hours). Use a **2-host** `inferencex_atom_cluster.json` and set `params.master_addr` to the head VPC IP after `copy-config` (replace `{head-node-ip}` placeholder). + +```bash +cd ~/cvs +make install +source .cvs_venv/bin/activate + +BASELINE_MULTI_DIR=~/input/config_file/inference/inferencex_atom/baseline_sweep_distributed +mkdir -p "$BASELINE_MULTI_DIR" + +cvs copy-config inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_baseline_sweep_distributed.json \ + --output "$BASELINE_MULTI_DIR/mi300x_inferencex-atom_deepseek-r1_fp8_baseline_sweep_distributed.json" +cvs copy-config inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_baseline_sweep_distributed_threshold.json \ + --output "$BASELINE_MULTI_DIR/mi300x_inferencex-atom_deepseek-r1_fp8_baseline_sweep_distributed_threshold.json" +cvs copy-config inferencex_atom_cluster.json --output ~/input/cluster_file/inferencex_atom_cluster.json + +# Ensure node_dict lists head + worker. Edit cluster IPs and set master_addr in the copied config. + +TS=$(date +%Y%m%d_%H%M%S) +HTML=~/cvs_results/${TS}_ix-atom-baseline-sweep-multinode_mi300x.html +LOG=~/cvs_results/${TS}_ix-atom-baseline-sweep-multinode_mi300x.log + +cvs run inferencex_atom \ + --cluster_file ~/input/cluster_file/inferencex_atom_cluster.json \ + --config_file "$BASELINE_MULTI_DIR/mi300x_inferencex-atom_deepseek-r1_fp8_baseline_sweep_distributed.json" \ + --html="$HTML" \ + --self-contained-html \ + --log-file="$LOG" \ + -vvv -s + +echo "HTML: $HTML" +echo "LOG: $LOG" +``` + +## W1 perf multinode (MI300X, 2-node, `driver=vllm_atom`) + +15-cell W1 scaling matrix with **`pipeline_parallel_size=2`**, **`driver=vllm_atom`**, and `scaling.efficiency_pct` gates. Requires vLLM+ATOM container, `ib_netdev`, and 2-node cluster. Recalibrate thresholds after the first true PP=2 lab run. + +```bash +cd ~/cvs +make install # after git pull only; run before activating venv +source .cvs_venv/bin/activate + +DISTRIBUTED_DIR=~/input/config_file/inference/inferencex_atom/distributed +mkdir -p "$DISTRIBUTED_DIR" + +cvs copy-config inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_distributed.json \ + --output "$DISTRIBUTED_DIR/mi300x_inferencex-atom_deepseek-r1_fp8_distributed.json" +cvs copy-config inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_distributed_threshold.json \ + --output "$DISTRIBUTED_DIR/mi300x_inferencex-atom_deepseek-r1_fp8_distributed_threshold.json" +cvs copy-config inferencex_atom_cluster.json --output ~/input/cluster_file/inferencex_atom_cluster.json + +# Ensure node_dict lists head + worker. Edit cluster IPs, ib_netdev, container.image, +# and set params.master_addr in the copied config. + +TS=$(date +%Y%m%d_%H%M%S) +HTML=~/cvs_results/${TS}_ix-atom-w1-perf-multi_mi300x.html +LOG=~/cvs_results/${TS}_ix-atom-w1-perf-multi_mi300x.log + +cvs run inferencex_atom \ + --cluster_file ~/input/cluster_file/inferencex_atom_cluster.json \ + --config_file "$DISTRIBUTED_DIR/mi300x_inferencex-atom_deepseek-r1_fp8_distributed.json" \ + --html="$HTML" \ + --self-contained-html \ + --log-file="$LOG" \ + -vvv -s + +echo "HTML: $HTML" +echo "LOG: $LOG" +``` + +## W1 perf multinode SGLang (MI300X, 2-node, `driver=sglang`) + +Same 15-cell sweep as vLLM-ATOM multinode, using SGLang pipeline parallel. `enforce_thresholds: false` until lab confirms — seed thresholds only. + +```bash +cd ~/cvs +make install +source .cvs_venv/bin/activate + +SGLANG_DIR=~/input/config_file/inference/inferencex_atom/sglang_distributed +mkdir -p "$SGLANG_DIR" + +cvs copy-config inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_sglang_distributed.json \ + --output "$SGLANG_DIR/mi300x_inferencex-atom_deepseek-r1_fp8_sglang_distributed.json" +cvs copy-config inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_sglang_distributed_threshold.json \ + --output "$SGLANG_DIR/mi300x_inferencex-atom_deepseek-r1_fp8_sglang_distributed_threshold.json" +cvs copy-config inferencex_atom_cluster.json --output ~/input/cluster_file/inferencex_atom_cluster.json + +# Edit cluster IPs, ib_netdev, SGLang container.image, params.master_addr. + +TS=$(date +%Y%m%d_%H%M%S) +HTML=~/cvs_results/${TS}_ix-atom-w1-perf-multi-sglang_mi300x.html +LOG=~/cvs_results/${TS}_ix-atom-w1-perf-multi-sglang_mi300x.log + +cvs run inferencex_atom \ + --cluster_file ~/input/cluster_file/inferencex_atom_cluster.json \ + --config_file "$SGLANG_DIR/mi300x_inferencex-atom_deepseek-r1_fp8_sglang_distributed.json" \ + --html="$HTML" \ + --self-contained-html \ + --log-file="$LOG" \ + -vvv -s + +echo "HTML: $HTML" +echo "LOG: $LOG" +``` + +## W1 perf multinode (MI355X, 2-node, `driver=vllm_atom`) + +Same sweep matrix as MI300X multinode (`vllm_atom`, PP=2). Thresholds are seeded from the MI355X single-node CI reference; `enforce_thresholds` stays `false` until a 2-node MI355X lab run confirms. + +```bash +cd ~/cvs +make install # after git pull only; run before activating venv +source .cvs_venv/bin/activate + +DISTRIBUTED_DIR=~/input/config_file/inference/inferencex_atom/mi355x_distributed +mkdir -p "$DISTRIBUTED_DIR" + +cvs copy-config inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_distributed.json \ + --output "$DISTRIBUTED_DIR/mi355x_inferencex-atom_deepseek-r1_fp8_distributed.json" +cvs copy-config inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_distributed_threshold.json \ + --output "$DISTRIBUTED_DIR/mi355x_inferencex-atom_deepseek-r1_fp8_distributed_threshold.json" +cvs copy-config inferencex_atom_cluster.json --output ~/input/cluster_file/inferencex_atom_cluster.json + +# Ensure node_dict lists head + worker (params.nnodes=2 in multinode variant). + +# Edit cluster + config: replace {head-node-ip} / {worker-node-ip} and set params.master_addr. + +TS=$(date +%Y%m%d_%H%M%S) +HTML=~/cvs_results/${TS}_ix-atom-w1-perf-multi_mi355x.html +LOG=~/cvs_results/${TS}_ix-atom-w1-perf-multi_mi355x.log + +cvs run inferencex_atom \ + --cluster_file ~/input/cluster_file/inferencex_atom_cluster.json \ + --config_file "$DISTRIBUTED_DIR/mi355x_inferencex-atom_deepseek-r1_fp8_distributed.json" \ + --html="$HTML" \ + --self-contained-html \ + --log-file="$LOG" \ + -vvv -s + +echo "HTML: $HTML" +echo "LOG: $LOG" +``` + +## W1 single-node (MI355X, `driver=atom`) + +Thresholds are seeded from [ROCm/ATOM run 27912164002](https://github.com/ROCm/ATOM/actions/runs/27912164002). `enforce_thresholds` stays `false` until an MI355X lab run confirms. + +```bash +cd ~/cvs +make install # after git pull only; run before activating venv +source .cvs_venv/bin/activate + +SINGLE_DIR=~/input/config_file/inference/inferencex_atom/mi355x_single +mkdir -p "$SINGLE_DIR" + +cvs copy-config inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_single.json \ + --output "$SINGLE_DIR/mi355x_inferencex-atom_deepseek-r1_fp8_single.json" +cvs copy-config inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_single_threshold.json \ + --output "$SINGLE_DIR/mi355x_inferencex-atom_deepseek-r1_fp8_single_threshold.json" +cvs copy-config inferencex_atom_cluster.json --output ~/input/cluster_file/inferencex_atom_cluster.json + +TS=$(date +%Y%m%d_%H%M%S) +HTML=~/cvs_results/${TS}_ix-atom-w1-single_mi355x.html +LOG=~/cvs_results/${TS}_ix-atom-w1-single_mi355x.log + +cvs run inferencex_atom \ + --cluster_file ~/input/cluster_file/inferencex_atom_cluster.json \ + --config_file "$SINGLE_DIR/mi355x_inferencex-atom_deepseek-r1_fp8_single.json" \ + --html="$HTML" \ + --self-contained-html \ + --log-file="$LOG" \ + -vvv -s + +echo "HTML: $HTML" +echo "LOG: $LOG" +``` diff --git a/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_baseline_sweep.json b/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_baseline_sweep.json new file mode 100644 index 000000000..55af4a01e --- /dev/null +++ b/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_baseline_sweep.json @@ -0,0 +1,143 @@ +{ + "_comment": "DTNI baseline sweep: 1K/1K + 8K/1K \u00d7 C=4-256 (14 cells). max_model_length=10240 fits 8192+1024 with headroom. Runs grouped by shape for server reuse.", + "schema_version": 1, + "framework": "inferencex_atom", + "gpu_arch": "mi300x", + "enforce_thresholds": true, + "threshold_json": "mi300x_inferencex-atom_deepseek-r1_fp8_baseline_sweep_threshold.json", + "run_card": { + "atom_image_pin": "rocm/atom-dev:latest", + "notes": "DTNI baseline matrix; portable threshold floors until lab calibration" + }, + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/home/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "deepseek-ai/DeepSeek-R1-0528", + "remote": 0, + "precision": "fp8" + }, + "container": { + "lifetime": "per_run", + "name": "inferencex_atom_mi300x", + "image": "rocm/atom-dev:latest", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/home/models:/home/models" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "atom_args": [ + "-tp", + "8", + "--kv_cache_dtype", + "fp8", + "--trust-remote-code" + ], + "env": { + "ATOM_DISABLE_MMAP": "true" + } + } + }, + "params": { + "driver": "atom", + "port_no": "8000", + "tensor_parallelism": "8", + "random_range_ratio": "0.8", + "num_prompts": "1000", + "max_model_length": "10240", + "metric_percentiles": "95,99", + "reuse_server_across_sweep": "true", + "client_poll_count": "80", + "client_poll_wait_time": "60" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "baseline_1k_1k", + "isl": "1024", + "osl": "1024" + }, + { + "name": "baseline_8k_1k", + "isl": "8192", + "osl": "1024" + } + ], + "runs": [ + { + "combo": "baseline_1k_1k", + "concurrency": 4 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 8 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 16 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 32 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 64 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 128 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 256 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 4 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 8 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 16 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 32 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 64 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 128 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 256 + } + ] + } +} diff --git a/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_baseline_sweep_distributed.json b/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_baseline_sweep_distributed.json new file mode 100644 index 000000000..52e993949 --- /dev/null +++ b/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_baseline_sweep_distributed.json @@ -0,0 +1,152 @@ +{ + "_comment": "DTNI baseline sweep multinode (2x8 GPU, PP=2): 1K/1K + 8K/1K x C=4-256 (14 cells). vLLM-ATOM pipeline parallel; align master_addr and ib_netdev with cluster head.", + "schema_version": 1, + "framework": "inferencex_atom", + "gpu_arch": "mi300x", + "enforce_thresholds": true, + "threshold_json": "mi300x_inferencex-atom_deepseek-r1_fp8_baseline_sweep_distributed_threshold.json", + "run_card": { + "atom_image_pin": "rocm/atom-dev:latest", + "notes": "MI300X 2-node DTNI baseline matrix (PP=2 vLLM-ATOM); recalibrate thresholds after true PP lab run" + }, + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/home/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "deepseek-ai/DeepSeek-R1-0528", + "remote": 0, + "precision": "fp8" + }, + "container": { + "lifetime": "per_run", + "name": "inferencex_atom_mi300x_multi", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/home/models:/home/models" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "kv-cache-dtype": "fp8", + "trust-remote-code": true, + "enforce-eager": true, + "gpu-memory-utilization": "0.95", + "block-size": 64, + "no-enable-prefix-caching": true + }, + "ib_hca_devices": "auto", + "ib_netdev": "auto", + "env": { + "VLLM_ROCM_USE_AITER": "1", + "AMDGCN_USE_BUFFER_OPS": "0" + } + } + }, + "params": { + "driver": "vllm_atom", + "port_no": "8000", + "tensor_parallelism": "8", + "random_range_ratio": "0.8", + "num_prompts": "1000", + "max_model_length": "10240", + "metric_percentiles": "95,99", + "reuse_server_across_sweep": "true", + "client_poll_count": "80", + "client_poll_wait_time": "60", + "pipeline_parallel_size": "2", + "nnodes": "2", + "master_addr": "{head-node-ip}", + "master_port": "29501", + "scaling_baseline_output_throughput": "1500" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "baseline_1k_1k", + "isl": "1024", + "osl": "1024" + }, + { + "name": "baseline_8k_1k", + "isl": "8192", + "osl": "1024" + } + ], + "runs": [ + { + "combo": "baseline_1k_1k", + "concurrency": 4 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 8 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 16 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 32 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 64 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 128 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 256 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 4 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 8 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 16 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 32 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 64 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 128 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 256 + } + ] + } +} diff --git a/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_baseline_sweep_distributed_threshold.json b/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_baseline_sweep_distributed_threshold.json new file mode 100644 index 000000000..641190785 --- /dev/null +++ b/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_baseline_sweep_distributed_threshold.json @@ -0,0 +1,1487 @@ +{ + "_comment": "MI300X 2-node baseline sweep thresholds (PP=2 vLLM-ATOM); recalibrate after true pipeline-parallel lab run.", + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=4": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 12.5 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 6 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 9.0 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=8": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 25 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 12 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 15.5 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 400 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 25 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 25.0 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=32": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 800 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 400 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 50 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 40.0 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 1600 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 800 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 100 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 40.0 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 3000 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1500 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 375 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 188 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 40.0 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=256": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 5000 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 2500 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 625 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 313 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 40.0 + } + }, + "ISL=8192,OSL=1024,TP=8,PP=2,NNODES=2,CONC=4": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 25 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 6 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 3 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 7.9 + } + }, + "ISL=8192,OSL=1024,TP=8,PP=2,NNODES=2,CONC=8": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 12.5 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 6 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 13.0 + } + }, + "ISL=8192,OSL=1024,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 25 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 12 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 19.2 + } + }, + "ISL=8192,OSL=1024,TP=8,PP=2,NNODES=2,CONC=32": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 400 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 25 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 27.6 + } + }, + "ISL=8192,OSL=1024,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 800 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 400 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 50 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 34.7 + } + }, + "ISL=8192,OSL=1024,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 1500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 187 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 94 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 40.0 + } + }, + "ISL=8192,OSL=1024,TP=8,PP=2,NNODES=2,CONC=256": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 2500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1250 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 312 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 156 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 40.0 + } + } +} diff --git a/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_baseline_sweep_threshold.json b/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_baseline_sweep_threshold.json new file mode 100644 index 000000000..63d578729 --- /dev/null +++ b/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_baseline_sweep_threshold.json @@ -0,0 +1,1431 @@ +{ + "_comment": "MI300X baseline sweep portable mins (1K/1K + 8K/1K, C=4-256). Throughput floors are conservative; latency gates loose. Health: success_rate=1, failed=0. Recalibrate after first lab run.", + "ISL=1024,OSL=1024,TP=8,CONC=4": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 12.5 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 6 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + } + }, + "ISL=1024,OSL=1024,TP=8,CONC=8": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 25 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 12 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + } + }, + "ISL=1024,OSL=1024,TP=8,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 400 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 25 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + } + }, + "ISL=1024,OSL=1024,TP=8,CONC=32": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 800 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 400 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 50 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + } + }, + "ISL=1024,OSL=1024,TP=8,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 1600 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 800 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 100 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + } + }, + "ISL=1024,OSL=1024,TP=8,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 3000 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1500 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 375 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 188 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + } + }, + "ISL=1024,OSL=1024,TP=8,CONC=256": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 5000 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 2500 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 625 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 313 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + } + }, + "ISL=8192,OSL=1024,TP=8,CONC=4": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 25 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 6 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 3 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + } + }, + "ISL=8192,OSL=1024,TP=8,CONC=8": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 12.5 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 6 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + } + }, + "ISL=8192,OSL=1024,TP=8,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 25 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 12 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + } + }, + "ISL=8192,OSL=1024,TP=8,CONC=32": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 400 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 25 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + } + }, + "ISL=8192,OSL=1024,TP=8,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 800 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 400 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 50 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + } + }, + "ISL=8192,OSL=1024,TP=8,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 1500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 187 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 94 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + } + }, + "ISL=8192,OSL=1024,TP=8,CONC=256": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 2500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1250 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 312 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 156 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + } + } +} diff --git a/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_distributed.json b/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_distributed.json new file mode 100644 index 000000000..7cc85d583 --- /dev/null +++ b/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_distributed.json @@ -0,0 +1,175 @@ +{ + "_comment": "W1 DeepSeek R1 FP8 multinode (2\u00c3\u20148 GPU, PP=2). vLLM coordinates pipeline parallel; ATOM accelerates local kernels via vllm_atom driver. max_model_length=8192 fits W1 sweep (2k+2k \u00d7 random_range_ratio 0.8). Set roles.server.ib_netdev and container.image before lab run.", + "schema_version": 1, + "framework": "inferencex_atom", + "gpu_arch": "mi300x", + "enforce_thresholds": true, + "threshold_json": "mi300x_inferencex-atom_deepseek-r1_fp8_distributed_threshold.json", + "run_card": { + "atom_image_pin": "rocm/atom-dev:latest", + "notes": "MI300X 2-node PP=2 via vLLM-ATOM (M5); align master_addr and ib_netdev with cluster" + }, + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/home/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "deepseek-ai/DeepSeek-R1-0528", + "remote": 0, + "precision": "fp8" + }, + "container": { + "lifetime": "per_run", + "name": "inferencex_atom_mi300x_multi", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/home/models:/home/models" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "kv-cache-dtype": "fp8", + "trust-remote-code": true, + "enforce-eager": true, + "gpu-memory-utilization": "0.95", + "block-size": 64, + "no-enable-prefix-caching": true + }, + "ib_hca_devices": "auto", + "ib_netdev": "auto", + "env": { + "VLLM_ROCM_USE_AITER": "1", + "AMDGCN_USE_BUFFER_OPS": "0" + } + } + }, + "params": { + "driver": "vllm_atom", + "port_no": "8000", + "tensor_parallelism": "8", + "pipeline_parallel_size": "2", + "nnodes": "2", + "master_addr": "{head-node-ip}", + "master_port": "29501", + "scaling_baseline_output_throughput": "1500", + "random_range_ratio": "0.8", + "num_prompts": "1000", + "max_model_length": "8192", + "metric_percentiles": "95,99", + "reuse_server_across_sweep": "true", + "server_precheck_wait_s": "30", + "server_warmup_wait_s": "330", + "server_poll_count": "120", + "server_poll_wait_time": "60", + "client_poll_count": "150", + "client_poll_wait_time": "60" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w1_512_512", + "isl": "512", + "osl": "512" + }, + { + "name": "w1_1k_1k", + "isl": "1024", + "osl": "1024" + }, + { + "name": "w1_2k_1k", + "isl": "2048", + "osl": "1024" + }, + { + "name": "w1_1k_2k", + "isl": "1024", + "osl": "2048" + }, + { + "name": "w1_2k_2k", + "isl": "2048", + "osl": "2048" + } + ], + "runs": [ + { + "combo": "w1_512_512", + "concurrency": 16 + }, + { + "combo": "w1_512_512", + "concurrency": 64 + }, + { + "combo": "w1_512_512", + "concurrency": 128 + }, + { + "combo": "w1_1k_1k", + "concurrency": 16 + }, + { + "combo": "w1_1k_1k", + "concurrency": 64 + }, + { + "combo": "w1_1k_1k", + "concurrency": 128 + }, + { + "combo": "w1_2k_1k", + "concurrency": 16 + }, + { + "combo": "w1_2k_1k", + "concurrency": 64 + }, + { + "combo": "w1_2k_1k", + "concurrency": 128 + }, + { + "combo": "w1_1k_2k", + "concurrency": 16 + }, + { + "combo": "w1_1k_2k", + "concurrency": 64 + }, + { + "combo": "w1_1k_2k", + "concurrency": 128 + }, + { + "combo": "w1_2k_2k", + "concurrency": 16 + }, + { + "combo": "w1_2k_2k", + "concurrency": 64 + }, + { + "combo": "w1_2k_2k", + "concurrency": 128 + } + ] + } +} diff --git a/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_distributed_threshold.json b/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_distributed_threshold.json new file mode 100644 index 000000000..283351a88 --- /dev/null +++ b/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_distributed_threshold.json @@ -0,0 +1,1593 @@ +{ + "_comment": "MI300X W1 multinode thresholds calibrated from lab runs 20260723-20260724 (10.32.80.112/113, vLLM-ATOM PP=2). ISL=512/OSL=512 CONC=16: throughput/scaling mins at 80% of measured; ttft/tpot gated maxes at 110%. Other cells retain prior seeds until recalibrated.", + "ISL=512,OSL=512,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 671 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 334 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 84 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 42 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 402 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 15441 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 42 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 43 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 11 + } + }, + "ISL=512,OSL=512,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 3500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 438 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 219 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=512,OSL=512,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 5000 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 2500 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 625 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 313 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 1500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 188 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 94 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 23 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 3500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 438 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 219 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 5000 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 2500 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 625 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 313 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=2048,OSL=1024,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 1500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 188 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 94 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 22 + } + }, + "ISL=2048,OSL=1024,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 3500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 438 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 219 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=2048,OSL=1024,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 5000 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 2500 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 625 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 313 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=1024,OSL=2048,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 1100 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 735 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 138 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 92 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 24 + } + }, + "ISL=1024,OSL=2048,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 3500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 438 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 219 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=1024,OSL=2048,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 5000 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 2500 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 625 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 313 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=2048,OSL=2048,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 1500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 188 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 94 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 24 + } + }, + "ISL=2048,OSL=2048,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 3500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 438 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 219 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=2048,OSL=2048,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 5000 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 2500 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 625 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 313 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + } +} diff --git a/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_mtp3.json b/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_mtp3.json new file mode 100644 index 000000000..5c229d927 --- /dev/null +++ b/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_mtp3.json @@ -0,0 +1,94 @@ +{ + "_comment": "W1 DeepSeek R1 FP8+MTP3 on 8x MI300X. Recipe: dsr1-fp8-mi300x-atom-mtp3. Thresholds: plan Section 4.2.", + "schema_version": 1, + "framework": "inferencex_atom", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_inferencex-atom_deepseek-r1_fp8_mtp3_threshold.json", + "run_card": { + "atom_image_pin": "rocm/atom-dev:latest", + "notes": "MI300X MTP3 lab reference Section 4.2" + }, + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/home/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "deepseek-ai/DeepSeek-R1-0528", + "remote": 0, + "precision": "fp8" + }, + "container": { + "lifetime": "per_run", + "name": "inferencex_atom_mi300x", + "image": "rocm/atom-dev:latest", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/home/models:/home/models" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "atom_args": [ + "-tp", + "8", + "--kv_cache_dtype", + "fp8", + "--trust-remote-code", + "--method", + "mtp", + "--num-speculative-tokens", + "3" + ], + "env": { + "ATOM_DISABLE_MMAP": "true" + } + } + }, + "params": { + "driver": "atom", + "port_no": "8000", + "tensor_parallelism": "8", + "random_range_ratio": "0.8", + "num_prompts": "1000", + "max_model_length": "4096", + "metric_percentiles": "95,99", + "bench_extra_args": "--use-chat-template", + "client_poll_count": "80", + "client_poll_wait_time": "60" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w1_1k_1k", + "isl": "1024", + "osl": "1024" + } + ], + "runs": [ + { + "combo": "w1_1k_1k", + "concurrency": 128 + }, + { + "combo": "w1_1k_1k", + "concurrency": 256 + } + ] + } +} diff --git a/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_mtp3_threshold.json b/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_mtp3_threshold.json new file mode 100644 index 000000000..7d047749a --- /dev/null +++ b/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_mtp3_threshold.json @@ -0,0 +1,57 @@ +{ + "_comment": "MI300X W1 FP8+MTP3 seeds from plan Section 4.2 (record-only).", + "ISL=1024,OSL=1024,TP=8,CONC=128": { + "client.total_token_throughput": {"kind": "min_tok_s", "value": 12470.4}, + "client.output_throughput": {"kind": "min_tok_s", "value": 6913}, + "client.per_gpu_throughput": {"kind": "min_tok_s", "value": 1558.8}, + "client.output_tput_per_gpu": {"kind": "min_tok_s", "value": 777.71}, + "client.mean_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_tpot_ms": {"kind": "max_ms", "value": 17.5}, + "client.median_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_tpot_ms": {"kind": "max_ms", "value": 20.13}, + "client.p99_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.success_rate": {"kind": "min", "value": 0}, + "client.failed": {"kind": "max", "value": 1000000000} + }, + "ISL=1024,OSL=1024,TP=8,CONC=256": { + "client.total_token_throughput": {"kind": "min_tok_s", "value": 13124.7}, + "client.output_throughput": {"kind": "min_tok_s", "value": 7284}, + "client.per_gpu_throughput": {"kind": "min_tok_s", "value": 1640.59}, + "client.output_tput_per_gpu": {"kind": "min_tok_s", "value": 819.45}, + "client.mean_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_tpot_ms": {"kind": "max_ms", "value": 33.0}, + "client.median_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_tpot_ms": {"kind": "max_ms", "value": 37.95}, + "client.p99_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.success_rate": {"kind": "min", "value": 0}, + "client.failed": {"kind": "max", "value": 1000000000} + } +} diff --git a/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_sglang_distributed.json b/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_sglang_distributed.json new file mode 100644 index 000000000..cfe61b3b0 --- /dev/null +++ b/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_sglang_distributed.json @@ -0,0 +1,172 @@ +{ + "_comment": "W1 DeepSeek R1 FP8 multinode (2x8 GPU, PP=2) via SGLang pipeline parallel. Set ib_netdev and SGLang-capable container.image before lab run.", + "schema_version": 1, + "framework": "inferencex_atom", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_inferencex-atom_deepseek-r1_fp8_sglang_distributed_threshold.json", + "run_card": { + "atom_image_pin": "rocm/atom-dev:latest", + "notes": "MI300X 2-node PP=2 via SGLang; enforce_thresholds false until lab confirm" + }, + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/home/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "deepseek-ai/DeepSeek-R1-0528", + "remote": 0, + "precision": "fp8" + }, + "container": { + "lifetime": "per_run", + "name": "inferencex_atom_mi300x_multi_sglang", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/home/models:/home/models" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "sglang_args": [ + "--kv-cache-dtype", + "fp8", + "--trust-remote-code", + "--disable-cuda-graph", + "--mem-fraction-static", + "0.9", + "--attention-backend", + "aiter" + ], + "ib_hca_devices": "auto", + "ib_netdev": "auto", + "env": { + "SGLANG_USE_AITER": "1" + } + } + }, + "params": { + "driver": "sglang", + "port_no": "8000", + "tensor_parallelism": "8", + "pipeline_parallel_size": "2", + "nnodes": "2", + "master_addr": "{head-node-ip}", + "master_port": "29501", + "scaling_baseline_output_throughput": "1500", + "random_range_ratio": "0.8", + "num_prompts": "1000", + "max_model_length": "4096", + "metric_percentiles": "95,99", + "reuse_server_across_sweep": "true", + "client_poll_count": "80", + "client_poll_wait_time": "60" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w1_512_512", + "isl": "512", + "osl": "512" + }, + { + "name": "w1_1k_1k", + "isl": "1024", + "osl": "1024" + }, + { + "name": "w1_2k_1k", + "isl": "2048", + "osl": "1024" + }, + { + "name": "w1_1k_2k", + "isl": "1024", + "osl": "2048" + }, + { + "name": "w1_2k_2k", + "isl": "2048", + "osl": "2048" + } + ], + "runs": [ + { + "combo": "w1_512_512", + "concurrency": 16 + }, + { + "combo": "w1_512_512", + "concurrency": 64 + }, + { + "combo": "w1_512_512", + "concurrency": 128 + }, + { + "combo": "w1_1k_1k", + "concurrency": 16 + }, + { + "combo": "w1_1k_1k", + "concurrency": 64 + }, + { + "combo": "w1_1k_1k", + "concurrency": 128 + }, + { + "combo": "w1_2k_1k", + "concurrency": 16 + }, + { + "combo": "w1_2k_1k", + "concurrency": 64 + }, + { + "combo": "w1_2k_1k", + "concurrency": 128 + }, + { + "combo": "w1_1k_2k", + "concurrency": 16 + }, + { + "combo": "w1_1k_2k", + "concurrency": 64 + }, + { + "combo": "w1_1k_2k", + "concurrency": 128 + }, + { + "combo": "w1_2k_2k", + "concurrency": 16 + }, + { + "combo": "w1_2k_2k", + "concurrency": 64 + }, + { + "combo": "w1_2k_2k", + "concurrency": 128 + } + ] + } +} diff --git a/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_sglang_distributed_threshold.json b/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_sglang_distributed_threshold.json new file mode 100644 index 000000000..a5de4dd92 --- /dev/null +++ b/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_sglang_distributed_threshold.json @@ -0,0 +1,1593 @@ +{ + "_comment": "MI300X W1 multinode SGLang PP=2 seed thresholds (copy of vLLM-ATOM keys); recalibrate after lab run.", + "ISL=512,OSL=512,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 1500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 188 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 94 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 22 + } + }, + "ISL=512,OSL=512,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 3500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 438 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 219 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=512,OSL=512,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 5000 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 2500 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 625 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 313 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 1500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 188 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 94 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 23 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 3500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 438 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 219 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 5000 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 2500 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 625 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 313 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=2048,OSL=1024,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 1500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 188 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 94 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 22 + } + }, + "ISL=2048,OSL=1024,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 3500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 438 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 219 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=2048,OSL=1024,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 5000 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 2500 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 625 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 313 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=1024,OSL=2048,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 1100 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 735 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 138 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 92 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 24 + } + }, + "ISL=1024,OSL=2048,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 3500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 438 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 219 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=1024,OSL=2048,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 5000 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 2500 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 625 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 313 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=2048,OSL=2048,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 1500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 188 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 94 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 24 + } + }, + "ISL=2048,OSL=2048,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 3500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 438 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 219 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=2048,OSL=2048,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 5000 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 2500 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 625 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 313 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + } +} diff --git a/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_single.json b/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_single.json new file mode 100644 index 000000000..d145325f9 --- /dev/null +++ b/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_single.json @@ -0,0 +1,90 @@ +{ + "_comment": "W1 DeepSeek R1 FP8 on 8x MI300X. Recipe: dsr1-fp8-mi300x-atom. Thresholds: portable minimum SLOs (throughput + health), not per-node calibration.", + "schema_version": 1, + "framework": "inferencex_atom", + "gpu_arch": "mi300x", + "enforce_thresholds": true, + "threshold_json": "mi300x_inferencex-atom_deepseek-r1_fp8_single_threshold.json", + "run_card": { + "atom_image_pin": "rocm/atom-dev:latest", + "notes": "MI300X lab reference Section 4.1" + }, + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/home/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "deepseek-ai/DeepSeek-R1-0528", + "remote": 0, + "precision": "fp8" + }, + "container": { + "lifetime": "per_run", + "name": "inferencex_atom_mi300x", + "image": "rocm/atom-dev:latest", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/home/models:/home/models" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "atom_args": [ + "-tp", + "8", + "--kv_cache_dtype", + "fp8", + "--trust-remote-code" + ], + "env": { + "ATOM_DISABLE_MMAP": "true" + } + } + }, + "params": { + "driver": "atom", + "port_no": "8000", + "tensor_parallelism": "8", + "random_range_ratio": "0.8", + "num_prompts": "1000", + "max_model_length": "4096", + "metric_percentiles": "95,99", + "reuse_server_across_sweep": "true", + "client_poll_count": "80", + "client_poll_wait_time": "60" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w1_1k_1k", + "isl": "1024", + "osl": "1024" + } + ], + "runs": [ + { + "combo": "w1_1k_1k", + "concurrency": 128 + }, + { + "combo": "w1_1k_1k", + "concurrency": 256 + } + ] + } +} diff --git a/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_single_threshold.json b/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_single_threshold.json new file mode 100644 index 000000000..19c1cd417 --- /dev/null +++ b/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_single_threshold.json @@ -0,0 +1,57 @@ +{ + "_comment": "MI300X W1 portable minimum SLOs (any healthy lab node). Throughput mins are conservative floors (~50% below typical W1 runs), not per-node calibration. Latency tier gates (ttft/tpot) are loose (1e6 ms) so enforcement focuses on throughput + health. Health: success_rate=1, failed=0.", + "ISL=1024,OSL=1024,TP=8,CONC=128": { + "client.total_token_throughput": {"kind": "min_tok_s", "value": 3000}, + "client.output_throughput": {"kind": "min_tok_s", "value": 1500}, + "client.per_gpu_throughput": {"kind": "min_tok_s", "value": 375}, + "client.output_tput_per_gpu": {"kind": "min_tok_s", "value": 188}, + "client.mean_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.success_rate": {"kind": "min", "value": 1}, + "client.failed": {"kind": "max", "value": 0} + }, + "ISL=1024,OSL=1024,TP=8,CONC=256": { + "client.total_token_throughput": {"kind": "min_tok_s", "value": 5000}, + "client.output_throughput": {"kind": "min_tok_s", "value": 2500}, + "client.per_gpu_throughput": {"kind": "min_tok_s", "value": 625}, + "client.output_tput_per_gpu": {"kind": "min_tok_s", "value": 313}, + "client.mean_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.success_rate": {"kind": "min", "value": 1}, + "client.failed": {"kind": "max", "value": 0} + } +} diff --git a/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_gpt-oss-120b_bf16.json b/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_gpt-oss-120b_bf16.json new file mode 100644 index 000000000..f78edaca0 --- /dev/null +++ b/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_gpt-oss-120b_bf16.json @@ -0,0 +1,91 @@ +{ + "_comment": "Single-node InferenceX ATOM (schema_version 1). Set container.image and container.name. ISL+OSL must fit params.max_model_length. Volume bind home only; ContainerOrchestrator adds /home/:/workspace.", + "schema_version": 1, + "framework": "inferencex_atom", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_inferencex-atom_gpt-oss-120b_bf16_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/home/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "openai/gpt-oss-120b", + "remote": 0, + "precision": "bf16" + }, + "container": { + "lifetime": "per_run", + "name": "", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/home/models:/home/models" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "enforce-eager": true, + "gpu-memory-utilization": "0.95", + "block-size": 64, + "no-enable-prefix-caching": true + }, + "env": { + "AMDGCN_USE_BUFFER_OPS": "0", + "VLLM_ROCM_USE_AITER": "1", + "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" + } + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8000", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.8", + "random_prefix_len": "0", + "tensor_parallelism": "8", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "99", + "num_prompts": "1000", + "max_model_length": "8192", + "client_poll_count": "50", + "client_poll_wait_time": "60", + "bench_max_failed_requests": "0" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "legacy_profile", + "isl": "7168", + "osl": "1024" + } + ], + "runs": [ + { + "combo": "legacy_profile", + "concurrency": 64 + } + ] + } +} diff --git a/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_gpt-oss-120b_bf16_threshold.json b/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_gpt-oss-120b_bf16_threshold.json new file mode 100644 index 000000000..bffa3c01d --- /dev/null +++ b/cvs/input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_gpt-oss-120b_bf16_threshold.json @@ -0,0 +1,105 @@ +{ + "_comment": "client.* thresholds for ISL=7168,OSL=1024,TP=8,CONC=64. Every GATED_METRICS member has a spec so the loader coverage check passes. config.json sets enforce_thresholds=false until calibrated; output_throughput/mean_ttft_ms/mean_tpot_ms carry legacy verify_inference_results targets.", + "ISL=7168,OSL=1024,TP=8,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 4200 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 525 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 500 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 15 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + } +} diff --git a/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_baseline_sweep.json b/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_baseline_sweep.json new file mode 100644 index 000000000..45d52fa4d --- /dev/null +++ b/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_baseline_sweep.json @@ -0,0 +1,144 @@ +{ + "_comment": "DTNI baseline sweep: 1K/1K + 8K/1K \u00d7 C=4-256 (14 cells). max_model_length=10240 fits 8192+1024 with headroom. enforce_thresholds false until MI355X lab calibration.", + "schema_version": 1, + "framework": "inferencex_atom", + "gpu_arch": "mi355x", + "enforce_thresholds": false, + "threshold_json": "mi355x_inferencex-atom_deepseek-r1_fp8_baseline_sweep_threshold.json", + "run_card": { + "upstream_run_url": "https://github.com/ROCm/ATOM/actions/runs/27912164002", + "atom_image_pin": "rocm/atom-dev:nightly_202606211542", + "notes": "DTNI baseline matrix; threshold seeds only until lab confirm" + }, + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/home/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "deepseek-ai/DeepSeek-R1-0528", + "remote": 0, + "precision": "fp8" + }, + "container": { + "lifetime": "per_run", + "name": "inferencex_atom_mi355x", + "image": "rocm/atom-dev:nightly_202606211542", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/home/models:/home/models" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "atom_args": [ + "-tp", + "8", + "--kv_cache_dtype", + "fp8", + "--trust-remote-code" + ], + "env": { + "ATOM_DISABLE_MMAP": "true" + } + } + }, + "params": { + "driver": "atom", + "port_no": "8000", + "tensor_parallelism": "8", + "random_range_ratio": "0.8", + "num_prompts": "1000", + "max_model_length": "10240", + "metric_percentiles": "95,99", + "reuse_server_across_sweep": "true", + "client_poll_count": "80", + "client_poll_wait_time": "60" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "baseline_1k_1k", + "isl": "1024", + "osl": "1024" + }, + { + "name": "baseline_8k_1k", + "isl": "8192", + "osl": "1024" + } + ], + "runs": [ + { + "combo": "baseline_1k_1k", + "concurrency": 4 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 8 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 16 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 32 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 64 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 128 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 256 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 4 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 8 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 16 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 32 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 64 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 128 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 256 + } + ] + } +} diff --git a/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_baseline_sweep_threshold.json b/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_baseline_sweep_threshold.json new file mode 100644 index 000000000..613369e16 --- /dev/null +++ b/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_baseline_sweep_threshold.json @@ -0,0 +1,1431 @@ +{ + "_comment": "MI355X baseline sweep seeds (record-only until lab). Placeholder throughput mins; flip enforce_thresholds after calibration.", + "ISL=1024,OSL=1024,TP=8,CONC=4": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 12.5 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 6 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + }, + "ISL=1024,OSL=1024,TP=8,CONC=8": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 25 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 12 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + }, + "ISL=1024,OSL=1024,TP=8,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 400 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 25 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + }, + "ISL=1024,OSL=1024,TP=8,CONC=32": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 800 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 400 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 50 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + }, + "ISL=1024,OSL=1024,TP=8,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 1600 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 800 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 100 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + }, + "ISL=1024,OSL=1024,TP=8,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 3000 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1500 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 375 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 188 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + }, + "ISL=1024,OSL=1024,TP=8,CONC=256": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 5000 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 2500 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 625 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 313 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + }, + "ISL=8192,OSL=1024,TP=8,CONC=4": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 25 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 6 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 3 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + }, + "ISL=8192,OSL=1024,TP=8,CONC=8": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 12.5 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 6 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + }, + "ISL=8192,OSL=1024,TP=8,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 25 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 12 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + }, + "ISL=8192,OSL=1024,TP=8,CONC=32": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 400 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 25 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + }, + "ISL=8192,OSL=1024,TP=8,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 800 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 400 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 50 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + }, + "ISL=8192,OSL=1024,TP=8,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 1500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 187 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 94 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + }, + "ISL=8192,OSL=1024,TP=8,CONC=256": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 2500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1250 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 312 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 156 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + } +} diff --git a/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_distributed.json b/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_distributed.json new file mode 100644 index 000000000..d3aba19d4 --- /dev/null +++ b/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_distributed.json @@ -0,0 +1,172 @@ +{ + "_comment": "W1 DeepSeek R1 FP8 multinode (2\u00c3\u20148 GPU MI355X, PP=2). vLLM-ATOM pipeline parallel; set ib_netdev and container.image before lab run.", + "schema_version": 1, + "framework": "inferencex_atom", + "gpu_arch": "mi355x", + "enforce_thresholds": false, + "threshold_json": "mi355x_inferencex-atom_deepseek-r1_fp8_distributed_threshold.json", + "run_card": { + "upstream_run_url": "https://github.com/ROCm/ATOM/actions/runs/27912164002", + "atom_image_pin": "rocm/atom-dev:nightly_202606211542", + "notes": "MI355X 2-node PP=2 via vLLM-ATOM (M5); lab re-run required before enforce_thresholds: true" + }, + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/home/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "deepseek-ai/DeepSeek-R1-0528", + "remote": 0, + "precision": "fp8" + }, + "container": { + "lifetime": "per_run", + "name": "inferencex_atom_mi355x_multi", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/home/models:/home/models" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "kv-cache-dtype": "fp8", + "trust-remote-code": true, + "enforce-eager": true, + "gpu-memory-utilization": "0.95", + "block-size": 64, + "no-enable-prefix-caching": true + }, + "ib_hca_devices": "auto", + "ib_netdev": "auto", + "env": { + "VLLM_ROCM_USE_AITER": "1", + "AMDGCN_USE_BUFFER_OPS": "0" + } + } + }, + "params": { + "driver": "vllm_atom", + "port_no": "8000", + "tensor_parallelism": "8", + "pipeline_parallel_size": "2", + "nnodes": "2", + "master_addr": "{head-node-ip}", + "master_port": "29501", + "scaling_baseline_output_throughput": "4000", + "random_range_ratio": "0.8", + "num_prompts": "1000", + "max_model_length": "4096", + "metric_percentiles": "95,99", + "reuse_server_across_sweep": "true", + "client_poll_count": "80", + "client_poll_wait_time": "60" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w1_512_512", + "isl": "512", + "osl": "512" + }, + { + "name": "w1_1k_1k", + "isl": "1024", + "osl": "1024" + }, + { + "name": "w1_2k_1k", + "isl": "2048", + "osl": "1024" + }, + { + "name": "w1_1k_2k", + "isl": "1024", + "osl": "2048" + }, + { + "name": "w1_2k_2k", + "isl": "2048", + "osl": "2048" + } + ], + "runs": [ + { + "combo": "w1_512_512", + "concurrency": 16 + }, + { + "combo": "w1_512_512", + "concurrency": 64 + }, + { + "combo": "w1_512_512", + "concurrency": 128 + }, + { + "combo": "w1_1k_1k", + "concurrency": 16 + }, + { + "combo": "w1_1k_1k", + "concurrency": 64 + }, + { + "combo": "w1_1k_1k", + "concurrency": 128 + }, + { + "combo": "w1_2k_1k", + "concurrency": 16 + }, + { + "combo": "w1_2k_1k", + "concurrency": 64 + }, + { + "combo": "w1_2k_1k", + "concurrency": 128 + }, + { + "combo": "w1_1k_2k", + "concurrency": 16 + }, + { + "combo": "w1_1k_2k", + "concurrency": 64 + }, + { + "combo": "w1_1k_2k", + "concurrency": 128 + }, + { + "combo": "w1_2k_2k", + "concurrency": 16 + }, + { + "combo": "w1_2k_2k", + "concurrency": 64 + }, + { + "combo": "w1_2k_2k", + "concurrency": 128 + } + ] + } +} diff --git a/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_distributed_threshold.json b/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_distributed_threshold.json new file mode 100644 index 000000000..071b1ba74 --- /dev/null +++ b/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_distributed_threshold.json @@ -0,0 +1,1593 @@ +{ + "_comment": "MI355X W1 multinode seeded thresholds (scaling.efficiency_pct min 50% floor). Throughput mins scaled from MI300X multinode scaffold \u00d7 single-node MI355X/MI300X ratio. Lab re-run required before enforce_thresholds: true.", + "ISL=512,OSL=512,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 8009.32 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 4004.66 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 1003.83 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 501.92 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=512,OSL=512,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 18688.41 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 9344.21 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 2338.72 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 1169.36 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=512,OSL=512,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 26697.73 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 13348.87 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 3337.22 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 1671.28 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 8009.32 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 4004.66 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 1003.83 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 501.92 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 18688.41 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 9344.21 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 2338.72 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 1169.36 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 26697.73 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 13348.87 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 3337.22 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 1671.28 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=2048,OSL=1024,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 8009.32 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 4004.66 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 1003.83 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 501.92 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=2048,OSL=1024,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 18688.41 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 9344.21 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 2338.72 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 1169.36 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=2048,OSL=1024,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 26697.73 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 13348.87 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 3337.22 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 1671.28 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=1024,OSL=2048,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 8009.32 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 4004.66 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 1003.83 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 501.92 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=1024,OSL=2048,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 18688.41 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 9344.21 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 2338.72 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 1169.36 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=1024,OSL=2048,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 26697.73 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 13348.87 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 3337.22 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 1671.28 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=2048,OSL=2048,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 8009.32 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 4004.66 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 1003.83 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 501.92 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=2048,OSL=2048,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 18688.41 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 9344.21 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 2338.72 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 1169.36 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=2048,OSL=2048,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 26697.73 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 13348.87 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 3337.22 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 1671.28 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + } +} diff --git a/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_mtp3.json b/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_mtp3.json new file mode 100644 index 000000000..5884babcf --- /dev/null +++ b/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_mtp3.json @@ -0,0 +1,95 @@ +{ + "_comment": "W1 DeepSeek R1 FP8+MTP3 on 8x MI355X. Recipe: dsr1-fp8-mi355x-atom-mtp3. Thresholds: plan Section 4.3.2.", + "schema_version": 1, + "framework": "inferencex_atom", + "gpu_arch": "mi355x", + "enforce_thresholds": false, + "threshold_json": "mi355x_inferencex-atom_deepseek-r1_fp8_mtp3_threshold.json", + "run_card": { + "upstream_run_url": "https://github.com/ROCm/ATOM/actions/runs/27912164002", + "atom_image_pin": "rocm/atom-dev:nightly_202606211542", + "notes": "MTP3 cells from ATOM CI run 27912164002" + }, + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/home/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "deepseek-ai/DeepSeek-R1-0528", + "remote": 0, + "precision": "fp8" + }, + "container": { + "lifetime": "per_run", + "name": "inferencex_atom_mi355x", + "image": "rocm/atom-dev:nightly_202606211542", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/home/models:/home/models" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "atom_args": [ + "-tp", + "8", + "--kv_cache_dtype", + "fp8", + "--trust-remote-code", + "--method", + "mtp", + "--num-speculative-tokens", + "3" + ], + "env": { + "ATOM_DISABLE_MMAP": "true" + } + } + }, + "params": { + "driver": "atom", + "port_no": "8000", + "tensor_parallelism": "8", + "random_range_ratio": "0.8", + "num_prompts": "1000", + "max_model_length": "4096", + "metric_percentiles": "95,99", + "bench_extra_args": "--use-chat-template", + "client_poll_count": "80", + "client_poll_wait_time": "60" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w1_1k_1k", + "isl": "1024", + "osl": "1024" + } + ], + "runs": [ + { + "combo": "w1_1k_1k", + "concurrency": 128 + }, + { + "combo": "w1_1k_1k", + "concurrency": 256 + } + ] + } +} diff --git a/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_mtp3_threshold.json b/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_mtp3_threshold.json new file mode 100644 index 000000000..e2f74248e --- /dev/null +++ b/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_mtp3_threshold.json @@ -0,0 +1,57 @@ +{ + "_comment": "MI355X W1 FP8+MTP3 calibrated from ROCm/ATOM run 27912164002 (Section 4.3.2). Throughput mins = ATOM CI × 0.9; latency maxes = ATOM CI × 1.1. Lab re-run required before enforce_thresholds: true.", + "ISL=1024,OSL=1024,TP=8,CONC=128": { + "client.total_token_throughput": {"kind": "min_tok_s", "value": 0}, + "client.output_throughput": {"kind": "min_tok_s", "value": 4591.79}, + "client.per_gpu_throughput": {"kind": "min_tok_s", "value": 0}, + "client.output_tput_per_gpu": {"kind": "min_tok_s", "value": 573.97}, + "client.mean_ttft_ms": {"kind": "max_ms", "value": 627.46}, + "client.median_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_tpot_ms": {"kind": "max_ms", "value": 26.15}, + "client.median_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.success_rate": {"kind": "min", "value": 0}, + "client.failed": {"kind": "max", "value": 1000000000} + }, + "ISL=1024,OSL=1024,TP=8,CONC=256": { + "client.total_token_throughput": {"kind": "min_tok_s", "value": 0}, + "client.output_throughput": {"kind": "min_tok_s", "value": 6451.59}, + "client.per_gpu_throughput": {"kind": "min_tok_s", "value": 0}, + "client.output_tput_per_gpu": {"kind": "min_tok_s", "value": 806.45}, + "client.mean_ttft_ms": {"kind": "max_ms", "value": 667.34}, + "client.median_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_tpot_ms": {"kind": "max_ms", "value": 37.64}, + "client.median_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.success_rate": {"kind": "min", "value": 0}, + "client.failed": {"kind": "max", "value": 1000000000} + } +} diff --git a/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_single.json b/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_single.json new file mode 100644 index 000000000..b50e3334f --- /dev/null +++ b/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_single.json @@ -0,0 +1,91 @@ +{ + "_comment": "W1 DeepSeek R1 FP8 on 8x MI355X. Recipe: dsr1-fp8-mi355x-atom. Thresholds: plan Section 4.3 (ATOM run 27912164002).", + "schema_version": 1, + "framework": "inferencex_atom", + "gpu_arch": "mi355x", + "enforce_thresholds": false, + "threshold_json": "mi355x_inferencex-atom_deepseek-r1_fp8_single_threshold.json", + "run_card": { + "upstream_run_url": "https://github.com/ROCm/ATOM/actions/runs/27912164002", + "atom_image_pin": "rocm/atom-dev:nightly_202606211542", + "notes": "ATOM commit ea08015; re-pin on image bumps" + }, + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/home/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "deepseek-ai/DeepSeek-R1-0528", + "remote": 0, + "precision": "fp8" + }, + "container": { + "lifetime": "per_run", + "name": "inferencex_atom_mi355x", + "image": "rocm/atom-dev:nightly_202606211542", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/home/models:/home/models" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "atom_args": [ + "-tp", + "8", + "--kv_cache_dtype", + "fp8", + "--trust-remote-code" + ], + "env": { + "ATOM_DISABLE_MMAP": "true" + } + } + }, + "params": { + "driver": "atom", + "port_no": "8000", + "tensor_parallelism": "8", + "random_range_ratio": "0.8", + "num_prompts": "1000", + "max_model_length": "4096", + "metric_percentiles": "95,99", + "reuse_server_across_sweep": "true", + "client_poll_count": "80", + "client_poll_wait_time": "60" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w1_1k_1k", + "isl": "1024", + "osl": "1024" + } + ], + "runs": [ + { + "combo": "w1_1k_1k", + "concurrency": 128 + }, + { + "combo": "w1_1k_1k", + "concurrency": 256 + } + ] + } +} diff --git a/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_single_threshold.json b/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_single_threshold.json new file mode 100644 index 000000000..3c66d23c2 --- /dev/null +++ b/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_single_threshold.json @@ -0,0 +1,57 @@ +{ + "_comment": "MI355X W1 calibrated from ROCm/ATOM run 27912164002 (Section 4.3.1). Throughput mins = ATOM CI × 0.9; latency maxes = ATOM CI × 1.1. Tail gates seeded from mean × 2.5 / × 1.15. Lab re-run required before enforce_thresholds: true.", + "ISL=1024,OSL=1024,TP=8,CONC=128": { + "client.total_token_throughput": {"kind": "min_tok_s", "value": 8018.11}, + "client.output_throughput": {"kind": "min_tok_s", "value": 4004.66}, + "client.per_gpu_throughput": {"kind": "min_tok_s", "value": 1002.26}, + "client.output_tput_per_gpu": {"kind": "min_tok_s", "value": 500.58}, + "client.mean_ttft_ms": {"kind": "max_ms", "value": 362.18}, + "client.median_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_ttft_ms": {"kind": "max_ms", "value": 905.45}, + "client.mean_tpot_ms": {"kind": "max_ms", "value": 30.40}, + "client.median_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_tpot_ms": {"kind": "max_ms", "value": 34.96}, + "client.p99_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.success_rate": {"kind": "min", "value": 0}, + "client.failed": {"kind": "max", "value": 1000000000} + }, + "ISL=1024,OSL=1024,TP=8,CONC=256": { + "client.total_token_throughput": {"kind": "min_tok_s", "value": 11244.09}, + "client.output_throughput": {"kind": "min_tok_s", "value": 5624.76}, + "client.per_gpu_throughput": {"kind": "min_tok_s", "value": 1405.51}, + "client.output_tput_per_gpu": {"kind": "min_tok_s", "value": 703.10}, + "client.mean_ttft_ms": {"kind": "max_ms", "value": 606.83}, + "client.median_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_ttft_ms": {"kind": "max_ms", "value": 1517.08}, + "client.mean_tpot_ms": {"kind": "max_ms", "value": 43.41}, + "client.median_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_tpot_ms": {"kind": "max_ms", "value": 49.92}, + "client.p99_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.success_rate": {"kind": "min", "value": 0}, + "client.failed": {"kind": "max", "value": 1000000000} + } +} diff --git a/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_gpt-oss-120b_bf16.json b/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_gpt-oss-120b_bf16.json new file mode 100644 index 000000000..b09ab4277 --- /dev/null +++ b/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_gpt-oss-120b_bf16.json @@ -0,0 +1,87 @@ +{ + "_comment": "Single-node InferenceX ATOM sample (schema_version 1). Set container.image and container.name. Verify serve_args against your InferenceX revision.", + "schema_version": 1, + "framework": "inferencex_atom", + "gpu_arch": "mi355x", + "enforce_thresholds": false, + "threshold_json": "mi355x_inferencex-atom_gpt-oss-120b_bf16_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/home/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "openai/gpt-oss-120b", + "remote": 0, + "precision": "bf16" + }, + "container": { + "lifetime": "per_run", + "name": "", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/home/models:/home/models" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "enforce-eager": true, + "gpu-memory-utilization": "0.92", + "block-size": 64, + "no-enable-prefix-caching": true + }, + "env": {} + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8000", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.8", + "random_prefix_len": "0", + "tensor_parallelism": "8", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "99", + "num_prompts": "1000", + "max_model_length": "8192", + "client_poll_count": "50", + "client_poll_wait_time": "60", + "bench_max_failed_requests": "0" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "legacy_profile", + "isl": "7168", + "osl": "1024" + } + ], + "runs": [ + { + "combo": "legacy_profile", + "concurrency": 64 + } + ] + } +} diff --git a/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_gpt-oss-120b_bf16_threshold.json b/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_gpt-oss-120b_bf16_threshold.json new file mode 100644 index 000000000..1b21327dd --- /dev/null +++ b/cvs/input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_gpt-oss-120b_bf16_threshold.json @@ -0,0 +1,105 @@ +{ + "_comment": "client.* thresholds for ISL=7168,OSL=1024,TP=8,CONC=64. Placeholder values until MI355x is calibrated; enforce_thresholds=false in config.json.", + "ISL=7168,OSL=1024,TP=8,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 4200 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 525 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 500 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 15 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + } +} diff --git a/cvs/input/config_file/inference/sglang/mi30x_sglang_deepseek_r1_0528_threshold.json b/cvs/input/config_file/inference/sglang/mi30x_sglang_deepseek_r1_0528_threshold.json new file mode 100644 index 000000000..18e8a82b8 --- /dev/null +++ b/cvs/input/config_file/inference/sglang/mi30x_sglang_deepseek_r1_0528_threshold.json @@ -0,0 +1,47 @@ +{ + "_comment": "DeepSeek-R1-0528 thresholds for MI30X SGLang disaggregated. Performance from bench_serv_random expected_results.auto; accuracy from lm_eval_* expected_results in mi30x_sglang_distributed.json.", + "ISL=1024,OSL=1024,TP=16,CONC=64": { + "output_throughput_per_sec": { + "kind": "min_tok_s", + "value": 340 + }, + "mean_ttft_ms": { + "kind": "max_ms", + "value": 60000 + }, + "mean_tpot_ms": { + "kind": "max_ms", + "value": 250 + }, + "mean_e2e_latency_ms": { + "kind": "max_ms", + "value": 120000 + }, + "goodput": { + "kind": "min", + "value": 0.99 + }, + "mfu": { + "kind": "min", + "value": 0.25 + } + }, + "BENCH=lm_eval_hellaswag": { + "acc_norm,none": { + "kind": "min", + "value": 0.23 + } + }, + "BENCH=lm_eval_gsm8k": { + "exact_match,flexible-extract": { + "kind": "min", + "value": 0.99 + } + }, + "BENCH=lm_eval_mmlu": { + "acc,none": { + "kind": "min", + "value": 0.5 + } + } + } \ No newline at end of file diff --git a/cvs/input/config_file/inference/sglang/mi30x_sglang_distributed.json b/cvs/input/config_file/inference/sglang/mi30x_sglang_distributed.json index d2f8e108a..0c102c5a6 100644 --- a/cvs/input/config_file/inference/sglang/mi30x_sglang_distributed.json +++ b/cvs/input/config_file/inference/sglang/mi30x_sglang_distributed.json @@ -185,4 +185,4 @@ } -} +} \ No newline at end of file diff --git a/cvs/input/config_file/inference/sglang/mi30x_sglang_gpt_oss_120b_threshold.json b/cvs/input/config_file/inference/sglang/mi30x_sglang_gpt_oss_120b_threshold.json new file mode 100644 index 000000000..569abd35f --- /dev/null +++ b/cvs/input/config_file/inference/sglang/mi30x_sglang_gpt_oss_120b_threshold.json @@ -0,0 +1,47 @@ +{ + "_comment": "GPT-OSS-120B thresholds for MI30X SGLang disaggregated. Performance from bench_serv_random expected_results.auto; accuracy from lm_eval_* expected_results in mi30x_sglang_distributed.json.", + "ISL=1024,OSL=1024,TP=8,CONC=25": { + "output_throughput_per_sec": { + "kind": "min_tok_s", + "value": 900 + }, + "mean_ttft_ms": { + "kind": "max_ms", + "value": 60000 + }, + "mean_tpot_ms": { + "kind": "max_ms", + "value": 150 + }, + "mean_e2e_latency_ms": { + "kind": "max_ms", + "value": 120000 + }, + "goodput": { + "kind": "min", + "value": 0.99 + }, + "mfu": { + "kind": "min", + "value": 0.25 + } + }, + "BENCH=lm_eval_hellaswag": { + "acc_norm,none": { + "kind": "min", + "value": 0.23 + } + }, + "BENCH=lm_eval_gsm8k": { + "exact_match,flexible-extract": { + "kind": "min", + "value": 0.96 + } + }, + "BENCH=lm_eval_mmlu": { + "acc,none": { + "kind": "min", + "value": 0.29 + } + } + } \ No newline at end of file diff --git a/cvs/input/config_file/inference/sglang/mi30x_sglang_llama_70b_threshold.json b/cvs/input/config_file/inference/sglang/mi30x_sglang_llama_70b_threshold.json new file mode 100644 index 000000000..85ff220d4 --- /dev/null +++ b/cvs/input/config_file/inference/sglang/mi30x_sglang_llama_70b_threshold.json @@ -0,0 +1,47 @@ +{ + "_comment": "Llama 3.1 70B thresholds for MI30X SGLang disaggregated. Performance from bench_serv_random expected_results.auto; accuracy from lm_eval_* expected_results in mi30x_sglang_distributed.json.", + "ISL=1024,OSL=1024,TP=8,CONC=25": { + "output_throughput_per_sec": { + "kind": "min_tok_s", + "value": 900 + }, + "mean_ttft_ms": { + "kind": "max_ms", + "value": 60000 + }, + "mean_tpot_ms": { + "kind": "max_ms", + "value": 150 + }, + "mean_e2e_latency_ms": { + "kind": "max_ms", + "value": 120000 + }, + "goodput": { + "kind": "min", + "value": 0.99 + }, + "mfu": { + "kind": "min", + "value": 0.02 + } + }, + "BENCH=lm_eval_hellaswag": { + "acc_norm,none": { + "kind": "min", + "value": 0.23 + } + }, + "BENCH=lm_eval_gsm8k": { + "exact_match,flexible-extract": { + "kind": "min", + "value": 0.96 + } + }, + "BENCH=lm_eval_mmlu": { + "acc,none": { + "kind": "min", + "value": 0.29 + } + } + } \ No newline at end of file diff --git a/cvs/input/config_file/inference/vllm/mi300x_vllm_llama31-70b_fp8_distributed.json b/cvs/input/config_file/inference/vllm/mi300x_vllm_llama31-70b_fp8_distributed.json new file mode 100644 index 000000000..97fc35911 --- /dev/null +++ b/cvs/input/config_file/inference/vllm/mi300x_vllm_llama31-70b_fp8_distributed.json @@ -0,0 +1,87 @@ +{ + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "", + "paths": { + "shared_fs": "/mnt/dtni/{user-id}", + "models_dir": "{shared_fs}/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.cache/huggingface/token" + }, + "model": { + "id": "amd/Llama-3.1-70B-Instruct-FP8-KV", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "w2_llama31_70b_fp8kv_dist_rocm", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/mnt/dtni:/mnt/dtni", + "{paths.models_dir}:/models" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "kv-cache-dtype": "fp8", + "enforce-eager": true + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1" + }, + "ib_hca_devices": "auto", + "ib_netdev": "" + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.8", + "random_prefix_len": "0", + "tensor_parallelism": "8", + "pipeline_parallel_size": "2", + "nnodes": "2", + "master_addr": "", + "master_port": "29501", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "3200", + "client_poll_count": "90" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w2_isl=1000_osl=1000", + "isl": "1000", + "osl": "1000", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { "combo": "w2_isl=1000_osl=1000", "concurrency": 16 } + ] + } +} diff --git a/cvs/input/config_file/inference/vllm/mi300x_vllm_llama31-70b_fp8_single.json b/cvs/input/config_file/inference/vllm/mi300x_vllm_llama31-70b_fp8_single.json new file mode 100644 index 000000000..5265dce98 --- /dev/null +++ b/cvs/input/config_file/inference/vllm/mi300x_vllm_llama31-70b_fp8_single.json @@ -0,0 +1,83 @@ +{ + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "", + "paths": { + "shared_fs": "/mnt/dtni/{user-id}", + "models_dir": "{shared_fs}/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.cache/huggingface/token" + }, + "model": { + "id": "amd/Llama-3.1-70B-Instruct-FP8-KV", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "w1_llama31_70b_fp8kv_perf_inference_rocm", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/mnt/dtni:/mnt/dtni", + "{paths.models_dir}:/models" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "kv-cache-dtype": "fp8", + "enforce-eager": true + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1" + } + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.8", + "random_prefix_len": "0", + "tensor_parallelism": "8", + "pipeline_parallel_size": "1", + "nnodes": "1", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "3200", + "client_poll_count": "90" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w1_isl=1000_osl=1000", + "isl": "1000", + "osl": "1000", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { "combo": "w1_isl=1000_osl=1000", "concurrency": 16 } + ] + } +} diff --git a/cvs/input/config_file/inference/vllm/mi355x_vllm_single.json b/cvs/input/config_file/inference/vllm/mi355x_vllm_single.json deleted file mode 100644 index 801acb6bf..000000000 --- a/cvs/input/config_file/inference/vllm/mi355x_vllm_single.json +++ /dev/null @@ -1,371 +0,0 @@ -{ - "config": { - "container_image": "rocm/7.0:rocm7.0_ubuntu_22.04_vllm_0.10.1_instinct_20250927_rc1", - "container_name": "vllm_inference_rocm", - "nnodes": "1", - "benchmark_server_script_path": "/home/{user-id}/benchmark_server_scripts/", - "benchmark_script_repo": "https://github.com/kimbochen/bench_serving.git", - "hf_token_file": "/home/{user-id}/.hf_token", - "shm_size": "16G", - "log_dir": "/home/{user-id}/LOGS", - "data_cache_dir": "/it-share/models/", - "container_config": { - "device_list": [ - "/dev/dri", - "/dev/kfd", - "/dev/mem" - ], - "volume_dict": { - "/home/{user-id}": "/home/{user-id}", - "/it-share/models/": "/models" - }, - "env_dict": { - "HF_HUB_CACHE": "/models/huggingface-cache" - } - } - }, - "benchmark_params": { - "gpt-oss-120b": { - "container_image": "rocm/7.0:rocm7.0_ubuntu_22.04_vllm_0.10.1_instinct_20250927_rc1", - "backend": "vllm", - "base_url": "http://0.0.0.0", - "port_no": "8888", - "_example_dataset_name": "sharegpt|hf|random|sonnet|burstgpt", - "dataset_name": "random", - "concurrency_levels": [ - 16, - 32, - 64 - ], - "model": "openai/gpt-oss-120b", - "num_prompts": "3200", - "sequence_combinations": [ - { - "isl": "1024", - "osl": "1024", - "name": "balanced" - }, - { - "isl": "1024", - "osl": "8192", - "name": "long_generation" - }, - { - "isl": "8192", - "osl": "1024", - "name": "long_context" - } - ], - "burstiness": "1.0", - "seed": "0", - "request_rate": "inf", - "max_model_length": "9216", - "random_range_ratio": "0.8", - "random_prefix_len": "0", - "tensor_parallelism": "1", - "_example_tokenizer_mode": "auto|slow|mistral|custom", - "tokenizer_mode": "auto", - "percentile_metrics": "ttft,tpot,itl,e2el", - "metric_percentiles": "99", - "server_script": "gpt-oss-120b_fp4_mi355x_vllm_docker.sh", - "bench_serv_script": "benchmark_serving.py", - "result_dict": { - "ISL=1024,OSL=1024,TP=1,CONC=16": { - "total_throughput_per_sec": "4651", - "mean_ttft_ms": "70", - "mean_tpot_ms": "8" - }, - "ISL=1024,OSL=1024,TP=1,CONC=32": { - "total_throughput_per_sec": "7043", - "mean_ttft_ms": "180", - "mean_tpot_ms": "9" - }, - "ISL=1024,OSL=1024,TP=1,CONC=64": { - "total_throughput_per_sec": "10677", - "mean_ttft_ms": "76", - "mean_tpot_ms": "13" - }, - "ISL=1024,OSL=8192,TP=1,CONC=16": { - "total_throughput_per_sec": "2735", - "mean_ttft_ms": "57", - "mean_tpot_ms": "7" - }, - "ISL=1024,OSL=8192,TP=1,CONC=32": { - "total_throughput_per_sec": "4038", - "mean_ttft_ms": "67", - "mean_tpot_ms": "10" - }, - "ISL=1024,OSL=8192,TP=1,CONC=64": { - "total_throughput_per_sec": "6140", - "mean_ttft_ms": "93", - "mean_tpot_ms": "13" - }, - "ISL=8192,OSL=1024,TP=1,CONC=16": { - "total_throughput_per_sec": "16509", - "mean_ttft_ms": "335", - "mean_tpot_ms": "24" - }, - "ISL=8192,OSL=1024,TP=1,CONC=32": { - "total_throughput_per_sec": "22072", - "mean_ttft_ms": "320", - "mean_tpot_ms": "19" - }, - "ISL=8192,OSL=1024,TP=1,CONC=64": { - "total_throughput_per_sec": "28863", - "mean_ttft_ms": "280", - "mean_tpot_ms": "22" - } - } - }, - "qwen3-235b": { - "container_image": "amdsiloai/vllm:2025111-0.11.1rc2-qwen3", - "backend": "vllm", - "base_url": "http://0.0.0.0", - "port_no": "8888", - "dataset_name": "random", - "concurrency_levels": [ - 16, - 32, - 64 - ], - "model": "Qwen/Qwen3-235B-A22B-Instruct-2507", - "num_prompts": "3200", - "sequence_combinations": [ - { - "isl": "1024", - "osl": "1024", - "name": "balanced" - }, - { - "isl": "1024", - "osl": "8192", - "name": "long_generation" - }, - { - "isl": "8192", - "osl": "1024", - "name": "long_context" - } - ], - "burstiness": "1.0", - "seed": "0", - "request_rate": "inf", - "max_model_length": "9216", - "random_range_ratio": "0.8", - "random_prefix_len": "0", - "tensor_parallelism": "8", - "tokenizer_mode": "auto", - "percentile_metrics": "ttft,tpot,itl,e2el", - "metric_percentiles": "99", - "server_script": "qwen3-235b-bf16_mi355x_vllm_docker.sh", - "bench_serv_script": "benchmark_serving.py", - "result_dict": { - "ISL=1024,OSL=1024,TP=8,CONC=16": { - "total_throughput_per_sec": "2000", - "mean_ttft_ms": "850", - "mean_tpot_ms": "18" - }, - "ISL=1024,OSL=1024,TP=8,CONC=32": { - "total_throughput_per_sec": "3435", - "mean_ttft_ms": "80", - "mean_tpot_ms": "10" - }, - "ISL=1024,OSL=1024,TP=8,CONC=64": { - "total_throughput_per_sec": "5840", - "mean_ttft_ms": "260", - "mean_tpot_ms": "10" - }, - "ISL=1024,OSL=8192,TP=8,CONC=16": { - "total_throughput_per_sec": "1119", - "mean_ttft_ms": "415", - "mean_tpot_ms": "25" - }, - "ISL=1024,OSL=8192,TP=8,CONC=32": { - "total_throughput_per_sec": "1876", - "mean_ttft_ms": "70", - "mean_tpot_ms": "10" - }, - "ISL=1024,OSL=8192,TP=8,CONC=64": { - "total_throughput_per_sec": "3139", - "mean_ttft_ms": "310", - "mean_tpot_ms": "14" - }, - "ISL=8192,OSL=1024,TP=8,CONC=16": { - "total_throughput_per_sec": "7476", - "mean_ttft_ms": "300", - "mean_tpot_ms": "21" - }, - "ISL=8192,OSL=1024,TP=8,CONC=32": { - "total_throughput_per_sec": "11312", - "mean_ttft_ms": "355", - "mean_tpot_ms": "27" - }, - "ISL=8192,OSL=1024,TP=8,CONC=64": { - "total_throughput_per_sec": "16082", - "mean_ttft_ms": "450", - "mean_tpot_ms": "39" - } - } - }, - "qwen3-80b": { - "container_image": "rocm/vllm-dev:nightly", - "backend": "vllm", - "base_url": "http://0.0.0.0", - "port_no": "8888", - "dataset_name": "random", - "concurrency_levels": [ - 16, - 32, - 64 - ], - "model": "Qwen/Qwen3-Next-80B-A3B-Instruct", - "num_prompts": "3200", - "sequence_combinations": [ - { - "isl": "1024", - "osl": "1024", - "name": "balanced" - }, - { - "isl": "1024", - "osl": "8192", - "name": "long_generation" - }, - { - "isl": "8192", - "osl": "1024", - "name": "long_context" - } - ], - "burstiness": "1.0", - "seed": "0", - "request_rate": "inf", - "max_model_length": "9216", - "random_range_ratio": "0.8", - "random_prefix_len": "0", - "tensor_parallelism": "1", - "tokenizer_mode": "auto", - "percentile_metrics": "ttft,tpot,itl,e2el", - "metric_percentiles": "99", - "server_script": "qwen3-80b-bf16_mi355x_vllm_docker.sh", - "bench_serv_script": "benchmark_serving.py", - "result_dict": { - "ISL=1024,OSL=1024,TP=1,CONC=16": { - "total_throughput_per_sec": "2003", - "mean_ttft_ms": "69", - "mean_tpot_ms": "9" - }, - "ISL=1024,OSL=1024,TP=1,CONC=32": { - "total_throughput_per_sec": "3155", - "mean_ttft_ms": "69", - "mean_tpot_ms": "12" - }, - "ISL=1024,OSL=1024,TP=1,CONC=64": { - "total_throughput_per_sec": "4570", - "mean_ttft_ms": "375", - "mean_tpot_ms": "23" - }, - "ISL=1024,OSL=8192,TP=1,CONC=16": { - "total_throughput_per_sec": "1200", - "mean_ttft_ms": "84", - "mean_tpot_ms": "12" - }, - "ISL=1024,OSL=8192,TP=1,CONC=32": { - "total_throughput_per_sec": "1800", - "mean_ttft_ms": "200", - "mean_tpot_ms": "12" - }, - "ISL=1024,OSL=8192,TP=1,CONC=64": { - "total_throughput_per_sec": "2600", - "mean_ttft_ms": "768", - "mean_tpot_ms": "21" - }, - "ISL=8192,OSL=1024,TP=1,CONC=16": { - "total_throughput_per_sec": "7500", - "mean_ttft_ms": "495", - "mean_tpot_ms": "16" - }, - "ISL=8192,OSL=1024,TP=1,CONC=32": { - "total_throughput_per_sec": "11300", - "mean_ttft_ms": "280", - "mean_tpot_ms": "17" - }, - "ISL=8192,OSL=1024,TP=1,CONC=64": { - "total_throughput_per_sec": "16000", - "mean_ttft_ms": "91", - "mean_tpot_ms": "18" - } - } - }, - "deepseek-v31": { - "container_image": "rocm/7.x-preview:rocm7.2_preview_ubuntu_22.04_vlm_0.10.1_instinct_20251029", - "backend": "vllm", - "base_url": "http://0.0.0.0", - "port_no": "8888", - "dataset_name": "random", - "concurrency_levels": [ - 16, - 32, - 64 - ], - "model": "deepseek-ai/DeepSeek-V3.1", - "num_prompts": "3200", - "sequence_combinations": [ - { - "isl": "1024", - "osl": "1024", - "name": "balanced" - }, - { - "isl": "1024", - "osl": "8192", - "name": "long_generation" - } - ], - "burstiness": "1.0", - "seed": "0", - "request_rate": "inf", - "max_model_length": "9216", - "random_range_ratio": "0.8", - "random_prefix_len": "0", - "tensor_parallelism": "8", - "tokenizer_mode": "auto", - "percentile_metrics": "ttft,tpot,itl,e2el", - "metric_percentiles": "99", - "server_script": "dsr1_fp8_mi355x_vllm_docker.sh", - "bench_serv_script": "benchmark_serving.py", - "result_dict": { - "ISL=1024,OSL=1024,TP=8,CONC=16": { - "total_throughput_per_sec": "1944", - "mean_ttft_ms": "84", - "mean_tpot_ms": "12" - }, - "ISL=1024,OSL=1024,TP=8,CONC=32": { - "total_throughput_per_sec": "2939", - "mean_ttft_ms": "302", - "mean_tpot_ms": "21" - }, - "ISL=1024,OSL=1024,TP=8,CONC=64": { - "total_throughput_per_sec": "4834", - "mean_ttft_ms": "250", - "mean_tpot_ms": "11" - }, - "ISL=1024,OSL=8192,TP=8,CONC=16": { - "total_throughput_per_sec": "1109", - "mean_ttft_ms": "253", - "mean_tpot_ms": "19" - }, - "ISL=1024,OSL=8192,TP=8,CONC=32": { - "total_throughput_per_sec": "1676", - "mean_ttft_ms": "305", - "mean_tpot_ms": "21" - }, - "ISL=1024,OSL=8192,TP=8,CONC=64": { - "total_throughput_per_sec": "2716", - "mean_ttft_ms": "280", - "mean_tpot_ms": "13" - } - } - } - } -} diff --git a/cvs/input/config_file/preflight/README_preflight_config.md b/cvs/input/config_file/preflight/README_preflight_config.md index dbd714070..f9ef32bc7 100644 --- a/cvs/input/config_file/preflight/README_preflight_config.md +++ b/cvs/input/config_file/preflight/README_preflight_config.md @@ -12,6 +12,7 @@ The preflight checks system validates essential cluster health before running pe 4. **Interface Name Consistency** - Validates RDMA interface naming patterns 5. **IFoE L2 Connectivity (AIMVT-180; opt-in)** - Runs `afmctl test ping` on each node and enforces per-port and Summary pass/fail accounting +6. **Primus Node Smoke (opt-in)** - Per-node host / GPU / RDMA roll-call via `primus-cli direct -- node_smoke` ## Configuration File Structure @@ -37,6 +38,13 @@ The preflight configuration file follows this structure: "inter_full_mesh_group_pairs_per_wave": "auto" } }, + "node_smoke": { + "connectivity_mode": "skip", + "auto_setup": true, + "primus_dir": "/home/{user-id}/INSTALL/Primus", + "venv_activate": "/home/{user-id}/envs/preflight/.venv/bin/activate", + "gpus_per_node": 8 + }, "reporting": { "generate_html_report": "true", "artifacts_root_dir": "/tmp/{user-id}/preflight", @@ -59,6 +67,7 @@ preflight/ ├── connectivity_check/ # Inter-node connectivity tests │ ├── rdma/ # RDMA-specific parameters (including nodes_per_full_mesh_group) │ └── ifoe/ # IFoE L2 ping parameters (AIMVT-180; opt-in) +├── node_smoke/ # Primus node_smoke per-node health screening (opt-in) └── reporting/ # Output and report generation ``` @@ -199,6 +208,52 @@ pass/fail table plus the aggregate `Summary:` block in afmctl's output. - **`ssh_timeout`** (default: `180`) - Per-invocation SSH timeout (seconds); raise for high `pings_per_port` +#### Node Smoke Settings (`node_smoke`) — opt-in (Primus Tier 1) + +Runs Primus `node_smoke` on each reachable node via `primus-cli direct --single -- node_smoke` +over parallel SSH (no Slurm required). Reference: Primus `docs/node-smoke-test-instruction.md` +on branch `dev/preflight-direct-test`. + +- **`connectivity_mode`** (default: `"skip"`) + - `"run"` — execute node_smoke on every reachable node + - `"skip"` — preflight records a SKIPPED result and does not invoke Primus +- **`auto_setup`** (default: `true`) + - Clone/update Primus and create the venv with minimal deps (ROCm PyTorch) before node_smoke +- **`setup_timeout`** (default: `600`) + - SSH timeout (seconds) for the per-node Primus auto_setup step +- **`force_reclone`** (default: `false`) + - Remove `primus_dir` and clone fresh on every run (destructive) +- **`shared_install`** (default: `true`) + - Leader node clones/installs on shared NFS home; other nodes wait (recommended for shared home) +- **`pip_install_mode`** (default: `"minimal"`) + - `"minimal"` — ROCm PyTorch only; `"requirements"` — `pip install -r requirements.txt`; `"skip"` — venv only +- **`torch_pip_index_url`** (default: `"https://download.pytorch.org/whl/rocm6.2"`) + - PyTorch wheel index for minimal install; match your ROCm version +- **`primus_git_url`** (default: `"https://github.com/AMD-AIG-AIMA/Primus.git"`) +- **`primus_git_branch`** (default: `"dev/preflight-direct-test"`) +- **`primus_git_recurse_submodules`** (default: `false`) +- **`primus_dir`** (default: `"/home/{user-id}/INSTALL/Primus"`) + - Required when `connectivity_mode` is `"run"`; `{user-id}` is resolved at runtime +- **`venv_activate`** (default: `"/home/{user-id}/envs/preflight/.venv/bin/activate"`) + - Required when `connectivity_mode` is `"run"` +- **`gpus_per_node`** (default: `8`) +- **`master_port`** (default: `1234`) +- **`dump_path`** (default: `""`) + - Per-node smoke JSON output; empty uses `/node_smoke` +- **`expected_rdma_nics`** (default: `null`) + - Defaults to `len(node_check.rdma_interfaces)` when null +- **`ulimit_l_min_gb`** (default: `32`) — FAIL below this memlock limit; `0` disables +- **`shm_min_gb`** (default: `8`) — FAIL below this `/dev/shm` size; `0` disables +- **`skip_dmesg`** (default: `false`) +- **`allow_foreign_procs`** (default: `false`) +- **`allowed_procs`** (default: `"gpuagent,rocm-smi-daemon,amd-smi,dcgm-exporter"`) +- **`require_tools`** (default: `""`) — empty = warn only +- **`nccl_socket_ifname`** / **`gloo_socket_ifname`** (default: `""`) +- **`nccl_ib_hca`** (default: `""`) — defaults to comma-joined `node_check.rdma_interfaces` +- **`nccl_ib_gid_index`** (default: `null`) — defaults to `node_check.gid_index` +- **`ssh_timeout`** (default: `300`) +- **`extra_args`** (default: `[]`) — additional flags forwarded to primus-cli + ### Reporting Settings (`reporting`) - **`generate_html_report`** (default: "true") @@ -277,6 +332,28 @@ pass/fail table plus the aggregate `Summary:` block in afmctl's output. } ``` +### Enable Primus Node Smoke + +```json +{ + "preflight": { + "node_check": { + "gid_index": "3", + "expected_rocm_version": "6.4.2", + "rdma_interfaces": ["rdma0", "rdma1", "rdma2", "rdma3", "rdma4", "rdma5", "rdma6", "rdma7"] + }, + "node_smoke": { + "connectivity_mode": "run", + "auto_setup": true, + "shared_install": true, + "primus_dir": "/home/{user-id}/INSTALL/Primus", + "venv_activate": "/home/{user-id}/envs/preflight/.venv/bin/activate", + "gpus_per_node": 8 + } + } +} +``` + ### Advanced Configuration with Debug and Tuning ```json @@ -317,6 +394,11 @@ pass/fail table plus the aggregate `Summary:` block in afmctl's output. # Basic usage with default config cvs run preflight_checks --cluster_file cluster.json --config_file preflight_config.json +# Run only the node_smoke check +cvs run preflight_checks test_node_smoke \ + --cluster_file cluster.json \ + --config_file preflight_config.json + # With custom HTML output cvs run preflight_checks \ --cluster_file cluster.json \ @@ -351,6 +433,13 @@ cvs run preflight_checks \ - Update rdma_interfaces list to match your cluster setup - Ensure all expected interfaces are present on each node +5. **Node Smoke Failures** + - Set `node_smoke.connectivity_mode` to `"run"` (default is `"skip"`) + - Verify `primus_dir` and `venv_activate`, or enable `auto_setup: true` + - On shared NFS home, use `shared_install: true` to avoid parallel clone races + - Match `torch_pip_index_url` to your ROCm version + - Review per-node fail reasons in the preflight HTML report + ### Performance Considerations **RDMA Connectivity Testing Times:** @@ -358,6 +447,10 @@ cvs run preflight_checks \ - **Full mesh mode**: ~5-10 minutes for 8 nodes - **Skip mode**: fastest path when validating only node-local checks +**Node Smoke Testing Times:** +- **First run with auto_setup**: several minutes per node (clone + ROCm PyTorch install) +- **Subsequent runs**: ~30–60 seconds per node + **Parallel Processing Impact:** - **Small nodes_per_full_mesh_group (16-32)**: More rounds, less resource usage per node, better for resource-constrained environments - **Large nodes_per_full_mesh_group (128+)**: Fewer rounds, more resource usage per node, faster overall completion @@ -396,4 +489,4 @@ cvs run ib_perf_bw_test --cluster_file cluster.json --config_file ib_config.json cvs run rccl_multinode_default_cvs --cluster_file cluster.json --config_file rccl_config.json ``` -This ensures your cluster is healthy before running resource-intensive performance tests. \ No newline at end of file +This ensures your cluster is healthy before running resource-intensive performance tests. diff --git a/cvs/input/config_file/preflight/preflight_config.json b/cvs/input/config_file/preflight/preflight_config.json index e46c8a9ca..3d8b0e279 100644 --- a/cvs/input/config_file/preflight/preflight_config.json +++ b/cvs/input/config_file/preflight/preflight_config.json @@ -91,6 +91,100 @@ "_comment_ssh_timeout": "Overall SSH timeout (seconds) for each afmctl invocation. Increase for large port counts or high pings_per_port values." } }, + + "node_smoke": { + "_comment": "Primus node_smoke checks via primus-cli direct (opt-in; default skip). See Primus docs/node-smoke-test-instruction.md", + + "_setup_comment": "Primus clone/venv setup runs automatically when auto_setup is true (default). Manual equivalent:", + "_setup_step_1": "git clone --recurse-submodules https://github.com/AMD-AIG-AIMA/Primus.git /home/{user-id}/INSTALL/Primus", + "_setup_step_2": "cd /home/{user-id}/INSTALL/Primus && git checkout dev/preflight-direct-test", + "_setup_step_3": "python3 -m venv /home/{user-id}/envs/preflight/.venv && pip install torch --index-url https://download.pytorch.org/whl/rocm6.2", + "_setup_note": "Paths use {user-id} resolved at runtime. Set auto_setup to false to skip automatic install.", + + "auto_setup": true, + "_comment_auto_setup": "When true, clone/update Primus and create venv with minimal deps (torch) on each node before node_smoke.", + + "setup_timeout": 600, + "_comment_setup_timeout": "SSH timeout (seconds) for the per-node Primus auto_setup step (clone + pip install).", + + "force_reclone": false, + "_comment_force_reclone": "When true, rm -rf primus_dir and clone fresh on every run (destructive). With shared_install (default), only the leader node reclones.", + + "shared_install": true, + "_comment_shared_install": "When true (default), only the first node clones/updates Primus and installs the venv on shared NFS home; other nodes wait. Set false only if primus_dir and venv_activate are local per node.", + + "pip_install_mode": "minimal", + "_comment_pip_install_mode": "Venv deps after clone: 'minimal' (torch only), 'requirements' (pip install -r requirements.txt), or 'skip' (venv only).", + + "torch_pip_index_url": "https://download.pytorch.org/whl/rocm6.2", + "_comment_torch_pip_index_url": "PyTorch wheel index for minimal install. Match your ROCm version (e.g. rocm6.2, rocm7.1).", + + "primus_git_url": "https://github.com/AMD-AIG-AIMA/Primus.git", + "_comment_primus_git_url": "Primus repository URL for one-time clone.", + + "primus_git_branch": "dev/preflight-direct-test", + "_comment_primus_git_branch": "Git branch to checkout after clone. node_smoke and primus-cli direct preflight live on this branch.", + + "primus_git_recurse_submodules": false, + "_comment_primus_git_recurse_submodules": "Clone submodules (Megatron, etc.). false is recommended for node_smoke — submodules are not required and slow setup.", + + "primus_dir": "/home/{user-id}/INSTALL/Primus", + "_comment_primus_dir": "Path where Primus is cloned on each cluster node. Must match the clone target in setup step 1. Required when connectivity_mode is 'run'.", + + "venv_activate": "/home/{user-id}/envs/preflight/.venv/bin/activate", + "_comment_venv_activate": "Path to the Python virtualenv activate script used by primus-cli direct. Required when connectivity_mode is 'run'.", + + "connectivity_mode": "skip", + "_comment_connectivity_mode": "Options: 'run' (host/GPU/RDMA roll-call via node_smoke) or 'skip' (default).", + + "gpus_per_node": 8, + "_comment_gpus_per_node": "Expected GPU count per node (exported as GPUS_PER_NODE and passed to node_smoke --expected-gpus).", + + "master_port": 1234, + "_comment_master_port": "MASTER_PORT for the distributed env primus-cli sets up across SSH-launched ranks.", + + "dump_path": "", + "_comment_dump_path": "Directory for per-node smoke/*.json output. Leave empty to use /node_smoke.", + + "expected_rdma_nics": null, + "_comment_expected_rdma_nics": "Hard-fail when training RDMA NIC count differs. null defaults to len(node_check.rdma_interfaces). Example: 8.", + + "ulimit_l_min_gb": 32, + "_comment_ulimit_l_min_gb": "FAIL when RLIMIT_MEMLOCK is below this many GiB. 0 disables.", + + "shm_min_gb": 8, + "_comment_shm_min_gb": "FAIL when /dev/shm is below this many GiB. 0 disables.", + + "skip_dmesg": false, + "_comment_skip_dmesg": "Skip the dmesg recent-error scan (use inside unprivileged containers).", + + "allow_foreign_procs": false, + "_comment_allow_foreign_procs": "Do not FAIL on foreign GPU processes. Recommended inside containers where proc names resolve to N/A.", + + "allowed_procs": "gpuagent,rocm-smi-daemon,amd-smi,dcgm-exporter", + "_comment_allowed_procs": "Comma-separated process names allowed to hold GPUs without failing the node.", + + "require_tools": "", + "_comment_require_tools": "Comma-separated tools that must exist in PATH for PASS (amd-smi, rocm-smi, lsof). Empty = warn only.", + + "nccl_socket_ifname": "", + "_comment_nccl_socket_ifname": "Optional NCCL_SOCKET_IFNAME / GLOO_SOCKET_IFNAME override for node_smoke.", + + "gloo_socket_ifname": "", + "_comment_gloo_socket_ifname": "Optional GLOO_SOCKET_IFNAME override (defaults to nccl_socket_ifname when empty).", + + "nccl_ib_hca": "", + "_comment_nccl_ib_hca": "Optional NCCL_IB_HCA override. Defaults to comma-joined node_check.rdma_interfaces.", + + "nccl_ib_gid_index": null, + "_comment_nccl_ib_gid_index": "Optional NCCL_IB_GID_INDEX override. Defaults to node_check.gid_index.", + + "ssh_timeout": 300, + "_comment_ssh_timeout": "SSH timeout in seconds for each node's node_smoke invocation (~30s; increase for slow nodes).", + + "extra_args": [], + "_comment_extra_args": "Additional node_smoke CLI flags forwarded to primus-cli. Example: [\"--no-clean-dump-path\"]." + }, "reporting": { "_comment": "Post-test reporting and output", @@ -112,4 +206,4 @@ "_comment_scriptlet": "Enable ScriptLet debug mode: preserve generated scripts/logs on remote nodes. For RDMA connectivity, wraps each ibv_rc_pingpong server in strace with per-test traces under /tmp/preflight/strace_server__.log. WARNING: Can be expensive at scale due to strace overhead." } } -} \ No newline at end of file +} diff --git a/cvs/lib/inference/ADDING_A_SUITE.md b/cvs/lib/inference/ADDING_A_SUITE.md new file mode 100644 index 000000000..94e90a35c --- /dev/null +++ b/cvs/lib/inference/ADDING_A_SUITE.md @@ -0,0 +1,660 @@ +# Adding a new DTNI suite + +Step-by-step guide for adding a new Distributed Training aNd Inference suite. +The `vllm_single` suite is the reference implementation throughout. +Follow this top to bottom; each step links to the authoritative contract at the +moment you need it. + +--- + +## The layer map (read this first) + +Every concern has exactly one home. Before writing any code, locate your work on +this table: + +| Layer | Directory | What belongs here | +|---|---|---| +| Framework-agnostic | `cvs/lib/utils/` | `BaseVariantConfig`, `substitute_config`, `Paths`, `ContainerSpec`, `evaluate_all` | +| Serving-generic | `cvs/lib/inference/utils/` | `Sweep`, `SeqCombo`, `GoodputSlo`, `Roles`, `validate_sweep_selector` | +| Framework-specific | `cvs/lib//utils/` | `VariantConfig` subclass, `Params`, `load_variant`, metric vocabulary | +| Test suite | `cvs/tests/inference//` | `conftest.py`, test module(s) | +| Input configs | `cvs/input/config_file/inference//` | `_config.json`, `_threshold.json` | + +Decision rule at every layer boundary: + +- "Does any other suite (now or plausibly soon) need this?" → move it up one layer. +- "Is this specific to my framework's CLI flags or artifact format?" → keep it here. + +When in doubt, push up. Code stranded too low gets copy-pasted into the next +suite; code pushed too high creates invisible coupling. Neither is free. + +--- + +## Step 1: Decide what is generic vs framework-specific + +Before writing a single class, answer these questions: + +**Is this a serving/inference suite?** + +Serving suites sweep sequence lengths (`isl`/`osl`) at concurrency levels. +The sweep machinery — `Sweep`, `SeqCombo`, `GoodputSlo`, `Roles`, +`validate_sweep_selector` — already lives in `cvs/lib/inference/utils/` and is +reusable unchanged. The only framework-specific piece is `Params`: the CLI flags +your benchmark tool accepts. + +See [The serving-generic / vllm-specific seam](utils/AGENTS.md#the-serving-generic--vllm-specific-seam) +for the second-framework checklist. + +**Is this a training suite?** + +Training suites typically sweep different dimensions: `batch_size`, `seq_len`, +`num_gpus`, or similar. Write your own sweep schema. Still subclass +`BaseVariantConfig` (the framework-agnostic skeleton is always your base). +Your `cell_key` format is your choice — it just has to match your +`threshold.json` top-level keys exactly. + +--- + +## Step 2: Subclass `BaseVariantConfig` + +Create `cvs/lib//utils/_config_loader.py`. + +**Minimal skeleton for a serving suite:** + +```python +from pydantic import model_validator +from typing_extensions import Literal + +from cvs.lib.utils.config_loader import BaseVariantConfig, _Forbid, substitute_config +from cvs.lib.inference.utils.inferencing_config_loader import ( + GoodputSlo, Roles, Run, Sweep, SeqCombo, validate_sweep_selector, +) +from cvs.lib..utils._parsing import GATED_METRICS + + +class Params(_Forbid): + # Your framework's CLI flags. All fields str (passed as CLI arguments). + tensor_parallelism: str = "1" + port_no: str = "8888" + # ... add your flags here + + +class VariantConfig(BaseVariantConfig): + framework: Literal["your_framework"] + gpu_arch: str + roles: Roles = Roles() + params: Params + sweep: Sweep + + def cell_key(self, isl, osl, concurrency) -> str: + """Single source of truth for the threshold key for one sweep cell.""" + return f"ISL={isl},OSL={osl},TP={self.params.tensor_parallelism},CONC={concurrency}" + + def expected_cells(self) -> list: + """Every cell key the sweep's runs selector picks.""" + by_name = {c.name: c for c in self.sweep.sequence_combinations} + return [ + self.cell_key(by_name[r.combo].isl, by_name[r.combo].osl, r.concurrency) + for r in self.sweep.runs + ] + + @model_validator(mode="after") + def _check_thresholds_cover_sweep(self): + # Copy the two-axis check from inferencing_config_loader.py: + # Axis 1: every sweep cell has a threshold entry; no key names a phantom cell. + # Axis 2: every present cell has a spec for every GATED_METRICS member. + # When enforce_thresholds=False: warn instead of raise. + ... + return self + + +def load_variant(config_path, cluster_dict) -> VariantConfig: + raw, thresholds = substitute_config(config_path, cluster_dict) + raw["thresholds"] = thresholds + return VariantConfig(**raw) +``` + +See [Subclassing BaseVariantConfig](../utils/AGENTS.md#subclassing-basevariantconfig) +for the full contract: what fields you must add, what methods you must implement, +and the validator ordering rules. + +Key points: + +- `cell_key` is the **single source of truth**. The loader's coverage check + (`_check_thresholds_cover_sweep`) calls it to build expected keys; `test_metric` + calls it to look up the threshold spec. Change the format in one place and both + paths move together. A space, field-order change, or separator difference silently + drops the cell (no threshold match, no verdict). +- `_check_thresholds_cover_sweep` must check **both axes**. Without axis 2, a gated + metric with no threshold spec falls through the record-only branch of `test_metric` + and reports a green PASS with zero assertions even when `enforce_thresholds=True`. +- `load_variant` must call `substitute_config` — never reimplement file-read or + placeholder substitution. See [substitute_config contract](../utils/AGENTS.md#config_loaderpy) + for what it returns and what it does not do (it does not validate or type-coerce). + +--- + +## Step 2b: Write the metric vocabulary module + +Create `cvs/lib//utils/_parsing.py`. + +Reference: `cvs/lib/inference/utils/vllm_parsing.py`. + +This module is a pure-transform layer with no I/O. It contains: + +1. **A pure transform function** that maps a raw benchmark artifact dict to a + namespaced `{"client.": value}` dict. This function accepts only + data structures (no `orch`, no file paths) so it can be unit-tested without + a running container. + +2. **`YOUR_METRICS: list[tuple[str, str]]`** — the display surface: a list of + `(short_name, unit)` pairs for every metric the suite surfaces. This list is + iterated by `pytest_generate_tests` to emit one `test_metric` row per metric + per cell. + +3. **`GATED_METRICS: set[str]`** — the asserted subset: the short names whose + threshold specs are required in `threshold.json` for every sweep cell when + `enforce_thresholds=True`. This set is imported by `VariantConfig`'s + `_check_thresholds_cover_sweep` to run the axis-2 coverage check at load time. + +4. **The gated-vs-record-only decision rule:** gate a metric (add it to + `GATED_METRICS`) when you have a calibrated baseline and a regression means a + real performance failure. Keep a metric record-only (in `YOUR_METRICS` but not + in `GATED_METRICS`) for diagnostic or informational metrics (e.g. percentiles + useful for debugging but not yet part of the SLO contract) or for metrics + whose baselines are not yet calibrated. Record-only metrics still appear in + the HTML results table; they simply do not trigger a FAIL. + +--- + +## Step 3: Write the job class + +Create `cvs/lib/inference/_job.py` (or a similarly named module). + +Reference: `cvs/lib/inference/vllm_single.py` (`VllmJob`). + +The job class owns the benchmark lifecycle for a single cell. It is deliberately +I/O-agnostic above the `orch.exec` boundary — all container/SSH plumbing belongs +to `orch`, which is injected. + +**Constructor:** accept `orch`, `variant` (your `VariantConfig`), and every +per-cell parameter (`isl`, `osl`, `concurrency`, etc.) as explicit arguments. +Pull all config values from `variant.params` and `variant.paths` here, not in +the methods, so the methods stay stateless and testable. + +**Required methods:** + +```python +class YourJob: + def build_server_cmd(self): + """Write the env script and create per-cell output directories inside the container.""" + + def start_server(self): + """Launch the server in the background inside the container.""" + + def is_ready(self) -> bool: + """Check readiness by scanning the server log (not a fixed tail).""" + + def wait_ready(self): + """Poll until is_ready() or raise on timeout.""" + + def stop_server(self): + """Kill the server process.""" + + def run_client(self): + """Launch the benchmark client in the background inside the container.""" + + def wait_client_complete(self): + """Poll the client log until completion, crash, or timeout.""" + + def parse_results(self) -> dict: + """Fetches the results artifact (the only method that reads output data); the metric + transform is delegated to the pure function in _parsing.py.""" +``` + +**Key patterns from `VllmJob` to carry forward:** + +- **Scan the whole server log for readiness**, not `tail -N`. The startup banner + scrolls out of a fixed tail once the server gets chatty. +- **Accumulate completion and failure states independently in each poll iteration, + then raise on failure before returning on completion.** The benchmark tool + always prints an explicit completion marker (`COMPLETION_RE`) — key off that + positive signal rather than the absence of an error line. +- **Per-cell output directories** keyed by `isl/osl/concurrency`. A multi-cell + sweep must not overwrite an earlier cell's artifact; without this, + `parse_results` may silently read stale data from a prior cell. +- **`parse_results` raises on empty/missing/unparseable artifacts.** The test + wraps it in `try/except ... raise`, so a hard failure here is the correct + behavior — it breaks the cell cleanly rather than recording a silently-green row. +- **`parse_results` delegates the transform** to the pure function in + `_parsing.py`. The fetch (I/O) lives in the job class because + artifact layout is job-specific; the metric math lives in `_parsing.py` so + other suite variants can reuse it. + +--- + +## Step 4: Write the conftest fixtures + +Create `cvs/tests/inference//conftest.py`. + +Reference: `cvs/tests/inference/vllm/conftest.py`. + +See [conftest fixtures](utils/AGENTS.md#conftest-fixtures) for the fixture +ownership table. + +**All fixtures must be `scope="module"`** so they are shared across the entire +parametrized test run. A `scope="function"` fixture would re-launch the container +for every single metric row. + +**Required fixtures:** + +```python +@pytest.fixture(scope="module") +def cluster_dict(pytestconfig): + cluster_file = pytestconfig.getoption("cluster_file") + if not cluster_file: + pytest.fail("--cluster_file is required") + with open(cluster_file) as fp: + d = json.load(fp) + return resolve_cluster_config_placeholders(d) + + +@pytest.fixture(scope="module") +def variant_config(pytestconfig, cluster_dict): + config_file = pytestconfig.getoption("config_file") + if not config_file: + pytest.fail("--config_file is required") + return load_variant(config_file, cluster_dict) + + +@pytest.fixture(scope="module") +def lifecycle(): + return _Lifecycle() # see _Lifecycle class below + + +@pytest.fixture(scope="module") +def orch(cluster_dict, variant_config, lifecycle): + container_block = _deep_merge( + cluster_dict.get("container", {}), + variant_config.container.model_dump(), + ) + testsuite_config = {"orchestrator": "container", "container": container_block} + cfg = OrchestratorConfig.from_configs(cluster_dict, testsuite_config) + o = OrchestratorFactory.create_orchestrator(log, cfg) + yield o + if not lifecycle.torn_down: + log.info("orch fixture leak-guard: tearing down container") + o.teardown_containers() + + +@pytest.fixture(scope="module") +def hf_token(variant_config): + path = variant_config.paths.hf_token_file + if not os.path.isfile(path): + pytest.skip(f"hf_token file missing: {path}") + with open(path) as fp: + return fp.read().strip() + + +@pytest.fixture(scope="module") +def inf_res_dict(): # or train_res_dict for a training suite + return {} +``` + +The `hf_token` fixture reads `variant_config.paths.hf_token_file` and calls +`pytest.skip` (not `pytest.fail`) when the file is missing. Using `skip` rather +than `fail` means suites that do not require an HF token can simply omit this +fixture from their conftest without breaking collection; the inference test that +accepts it as an argument will be skipped rather than erroring at fixture setup. + +**`_Lifecycle`** — cross-test state for the lifecycle-as-tests model. Copy from +`cvs/tests/inference/vllm/conftest.py`. It carries three fields: +- `failed: bool` — set when any stage fails; causes remaining stages to skip +- `torn_down: bool` — set when `test_teardown` succeeds; suppresses the + `orch` leak-guard finalizer so teardown never runs twice +- `report: dict` — maps `nodeid → [(label, value, unit)]`; populated by + `lifecycle.record(...)` and rendered by `pytest_runtest_makereport` + +**The `_deep_merge` pattern:** + +`OrchestratorConfig.from_configs` does a top-level `dict.update`, so a bare +variant container block wipes all cluster-set container settings. Deep-merge the +variant ONTO the cluster block so cluster-set keys (e.g. `shm_size`, env maps) +survive, with the variant winning on conflicts. Copy `_deep_merge` verbatim from +the vllm conftest. If your suite is the second to need it, extract it to +`cvs/lib/utils/` instead of copy-pasting again. + +**Required pytest hooks:** + +```python +def pytest_collection_modifyitems(items): + """Pin lifecycle order explicitly — never rely on definition order.""" + rank = { + "test_launch_container": 0, + "test_setup_sshd": 1, + "test_model_fetch": 2, + "test_": 3, + "test_metric": 4, + "test_print_results_table": 5, + "test_teardown": 6, + } + items.sort(key=lambda it: rank.get(it.originalname or it.name.split("[")[0], 99)) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + """Attach this test's recorded rows to its HTML detail panel.""" + ... # copy from vllm conftest + + +def pytest_html_results_table_header(cells): + """Add Value + Unit columns.""" + cells.insert(-1, "Value") + cells.insert(-1, "Unit") + + +def pytest_html_results_table_row(report, cells): + """Populate Value + Unit from metric_value / metric_unit user properties.""" + ... # copy from vllm conftest +``` + +`pytest_collection_modifyitems` is not optional. `test_print_results_table` is +typically an imported function whose source line points into a shared module, so +default pytest ordering collects it first — which logs an empty table before any +cell ran. Explicit ranking fixes this. + +--- + +## Step 5: Wire up `pytest_generate_tests` + +Add `pytest_generate_tests` to your **test module** (not conftest), immediately +above the test functions. + +Reference: `cvs/tests/inference/vllm/vllm_single.py`. + +See [pytest_generate_tests mirror rule](utils/AGENTS.md#pytest_generate_tests-mirror-rule) +for why this must call `validate_sweep_selector`. + +```python +def pytest_generate_tests(metafunc): + """Parametrize the workload test and test_metric from the raw config sweep. + + Runs at collection time before fixtures exist — reads raw JSON directly. + """ + config_file = metafunc.config.getoption("config_file") + if not config_file or not os.path.isfile(config_file): + return + with open(config_file) as fp: + raw = json.load(fp) + sweep = raw.get("sweep", {}) + combos = sweep.get("sequence_combinations", []) + runs = sweep.get("runs", []) + + # Validate GoodputSlo dicts through the _Forbid model so a typo'd SLO key + # fails collection, not silently drops the gate on hardware. + for combo in combos: + if combo.get("goodput_slo") is not None: + GoodputSlo(**combo["goodput_slo"]) + + # Mirror the typed Sweep validator via the shared rule. + # If you add a check to Sweep, add it to validate_sweep_selector so both paths enforce it. + validate_sweep_selector([c["name"] for c in combos], [r["combo"] for r in runs]) + + by_name = {c["name"]: c for c in combos} + cases = [(by_name[r["combo"]], r["concurrency"]) for r in runs] + ids = [r["combo"] + "-conc" + str(r["concurrency"]) for r in runs] + + if "metric" in metafunc.fixturenames: + # test_metric: one case per (cell, metric) + metric_cases = [] + metric_ids = [] + for (combo, c), cid in zip(cases, ids): + for short, _unit in YOUR_METRICS: + metric_cases.append((combo, c, short)) + metric_ids.append(cid + "-" + short) + metafunc.parametrize("seq_combo,concurrency,metric", metric_cases, ids=metric_ids) + elif "seq_combo" in metafunc.fixturenames and "concurrency" in metafunc.fixturenames and cases: + # workload test: one case per cell + metafunc.parametrize("seq_combo,concurrency", cases, ids=ids) +``` + +**Why `validate_sweep_selector` is mandatory here:** +`pytest_generate_tests` reads raw JSON at collection time, before `load_variant` +and the typed `Sweep` validator have run. Without calling `validate_sweep_selector` +here, a duplicate combo name or a `run.combo` typo is a silently-dropped cell at +collection time — the sweep runs a different matrix than the config reads. + +--- + +## Step 6: Write the test module + +Create `cvs/tests/inference//.py`. + +Reference: `cvs/tests/inference/vllm/vllm_single.py`. + +**The lifecycle-as-tests model:** each stage is an independent pytest test, not +fixture body code. Each appears as a timed, independently pass/fail HTML row. + +**Standard lifecycle order** (must match the rank dict in `pytest_collection_modifyitems`): + +1. `test_launch_container` — calls `orch.setup_containers()`; asserts container is running +2. `test_setup_sshd` — calls `orch.setup_sshd()`; probes `:2224` for multinode +3. `test_model_fetch` — ensures model bytes present; polls/downloads if remote +4. `test_` — benchmark loop; stores results in `res_dict` +5. `test_metric` — one test per metric per cell; reads `res_dict`; asserts verdict +6. `test_print_results_table` — summary log; must run after all cells +7. `test_teardown` — calls `orch.teardown_containers()`; sets `lifecycle.torn_down` + +**Invariants — every suite must enforce these:** + +- Every test except `test_launch_container`, `test_print_results_table`, and + `test_teardown` checks `lifecycle.failed` and calls `pytest.skip(...)` if + `True`. This prevents cascading failures where a broken launch causes every + subsequent cell to re-fail instead of skipping cleanly. + `test_print_results_table` does not guard on `lifecycle.failed`; instead it + checks whether `inf_res_dict` is empty and logs whatever results were + recorded (even a partial sweep produces a useful table). This behavior is + implemented in `cvs/tests/inference/vllm/_shared.py`; verify the check there + if you are adapting the pattern for a new suite. +- `test_` wraps its entire body in `try/except`: on any exception, set + `lifecycle.failed = True` then re-raise. +- `test_teardown` **never skips** — it must run even when `lifecycle.failed` is + `True`. The container must be torn down regardless of what happened in the sweep. +- `test_teardown` sets `lifecycle.torn_down = True` after a successful teardown. + This suppresses the `orch` fixture's leak-guard finalizer so the container is + not torn down twice. + +**`test_metric` pattern:** + +```python +def test_metric(seq_combo, concurrency, metric, inf_res_dict, variant_config, lifecycle, request): + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + + # Build the lookup key (must match what test_ stored) + isl, osl = seq_combo["isl"], seq_combo["osl"] + key = (variant_config.model.id, variant_config.gpu_arch, isl, osl, + seq_combo.get("name", "default"), concurrency) + if key not in inf_res_dict: + pytest.skip(f"no recorded results for cell {key!r}") + + host_dict = inf_res_dict[key] + _host, actuals = next(iter(host_dict.items())) + full = "client." + metric + value = actuals.get(full) + unit = YOUR_METRIC_UNITS.get(metric, "-") + + # Attach for HTML rendering (Value/Unit columns) + request.node.user_properties.append(("metric_value", value)) + request.node.user_properties.append(("metric_unit", unit)) + + if not variant_config.enforce_thresholds: + return # record-only + + cell = variant_config.cell_key(isl, osl, concurrency) + spec = (variant_config.thresholds.get(cell) or {}).get(full) + if spec is None: + return # record-only (no spec for this metric) + + # Pass the FULL per-cell actuals dict, not just this one metric's value. + # evaluate_all needs the full dict so a min_ratio spec can resolve its + # reference metric from the same actuals. + evaluate_all(actuals, {full: spec}) +``` + +See [evaluate_all contract](../utils/AGENTS.md#verdictpy) for why full cell +actuals are passed (needed for `min_ratio` reference resolution), and for the +behavior on `None` values and missing metrics. + +--- + +## Step 7: Write the config and threshold files + +**Config JSON** (`cvs/input/config_file/inference//_config.json`): + +```json +{ + "schema_version": 1, + "framework": "your_framework", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "/absolute/path/to/your_threshold.json", + "paths": { + "shared_fs": "/mnt/data/{user-id}", + "models_dir": "{shared_fs}/models", + "log_dir": "{shared_fs}/logs", + "hf_token_file": "/home/{user-id}/.hf_token" + }, + "model": {"id": "meta-llama/Llama-3.1-70B-Instruct", "remote": 0}, + "container": { + "lifetime": "per_run", + "name": "your_suite_container", + "image": "your.registry/image:tag", + "runtime": {"name": "docker", "args": {}} + }, + "params": {"tensor_parallelism": "8"}, + "sweep": { + "sequence_combinations": [ + {"name": "isl1000_osl1000", "isl": "1000", "osl": "1000"} + ], + "runs": [ + {"combo": "isl1000_osl1000", "concurrency": 16} + ] + } +} +``` + +Start with `"enforce_thresholds": false` until you have calibrated baselines. +Flip to `true` once threshold values are established. + +`threshold_json` is a **literal absolute path** — not relative to the config +file, not a glob. No placeholder substitution of any kind is applied to +`threshold_json` — not cluster placeholders, not `{paths.*}`. It is read +verbatim before any substitution pass runs. If the threshold path must vary by +user, it must be pre-resolved before being written into the config file. See +[placeholder-substitution.md](../utils/docs/placeholder-substitution.md) for a +worked example. + +**Threshold JSON** (`_threshold.json`): + +```json +{ + "_comment": "keys starting with _ are stripped before the coverage check", + "ISL=1000,OSL=1000,TP=8,CONC=16": { + "client.total_token_throughput": {"kind": "min_tok_s", "value": 12000}, + "client.output_throughput": {"kind": "min_tok_s", "value": 1500}, + "client.mean_ttft_ms": {"kind": "max_ms", "value": 200}, + "client.success_rate": {"kind": "min", "value": 0.99}, + "client.failed": {"kind": "max", "value": 0} + } +} +``` + +Every top-level key must match `cell_key(...)` output exactly — same field +order, same separators, no spaces. See +[cell-key-format.md](utils/docs/cell-key-format.md) for the exact format spec +and common mistake patterns. + +Every `GATED_METRICS` member must have a spec for every present cell. Missing +specs are caught at load time by `_check_thresholds_cover_sweep` (axis 2) when +`enforce_thresholds=True`. + +See [threshold-kinds.md](../utils/docs/threshold-kinds.md) for the full kind +reference (`min`, `max`, `max_ms`, `within`, `min_tok_s`, `min_ratio`). + +--- + +## Step 8: Suite reports (optional, `--html`) + +**Full guide:** `cvs/lib/report/README.md` + +Add one preset at `cvs/lib/report/presets/.py`. Root `cvs/conftest.py` +auto-loads it when the stem matches `cvs run …` and writes HTML/JSON/viewer at session +end when `--html` is set. Reports are render-only and do not change pass/fail. + +Do **not** wire reports in suite `conftest.py`. **IX-atom reference:** +`presets/inferencex_atom.py` + `presets/inferencex_atom.py`. + +--- + +## Pre-PR checklist + +Walk this list against your suite before opening a PR. Each item is verifiable +in the existing code. + +**Config machinery** + +- [ ] `load_variant` calls `substitute_config` — does not reimplement file-read or substitution +- [ ] `VariantConfig` subclasses `BaseVariantConfig` +- [ ] `VariantConfig` declares `framework: Literal["your_framework"]`, `params`, and `sweep` +- [ ] `cell_key` is implemented and is the single source of truth used by both + `_check_thresholds_cover_sweep` and `test_metric` +- [ ] `expected_cells` is implemented and returns the full list of cell keys +- [ ] `_check_thresholds_cover_sweep` is present as a `@model_validator(mode="after")` + and checks both axes (cell coverage + gated-metric coverage) + +**Test fixtures** + +- [ ] All fixtures are `scope="module"` +- [ ] `orch` fixture has a leak-guard finalizer that calls `teardown_containers()` when + `lifecycle.torn_down` is `False` +- [ ] `_deep_merge` is used when building the container block (not bare `dict.update`) + +**Test lifecycle** + +- [ ] Lifecycle order is pinned explicitly in `pytest_collection_modifyitems` +- [ ] Every test except `test_launch_container`, `test_print_results_table`, and `test_teardown` checks `lifecycle.failed` and skips if `True` (`test_print_results_table` instead checks whether `inf_res_dict` is empty — see `_shared.py`) +- [ ] `test_` catches all exceptions, sets `lifecycle.failed = True`, re-raises +- [ ] `test_teardown` does NOT skip on `lifecycle.failed` +- [ ] `test_teardown` sets `lifecycle.torn_down = True` after successful teardown + +**`pytest_generate_tests`** + +- [ ] Calls `validate_sweep_selector` to mirror the typed `Sweep` validator — collection-time + and load-time paths enforce the same rules +- [ ] Validates `GoodputSlo` dicts through the `_Forbid` model if your sweep uses goodput SLOs + +**`test_metric`** + +- [ ] Passes the full per-cell actuals dict to `evaluate_all`, not just the single metric value +- [ ] Returns (record-only) when `enforce_thresholds` is `False` +- [ ] Returns (record-only) when `spec is None` for this cell+metric + +**Layer placement** + +- [ ] Things only this suite needs → `cvs/lib//utils/` (not pushed up) +- [ ] Things any serving suite needs → `cvs/lib/inference/utils/` (not kept local) +- [ ] Things any CVS suite needs → `cvs/lib/utils/` (not kept at the inference layer) + +**Config files** + +- [ ] `threshold_json` is a literal absolute path (not relative, not a glob) +- [ ] Every threshold key matches `cell_key(...)` output exactly +- [ ] Every `GATED_METRICS` member has a spec for every present cell +- [ ] `enforce_thresholds` starts as `false` until baselines are calibrated +- [ ] New gated metric → every existing `threshold.json` that covers cells where the + metric is defined has a spec for it + +**Suite reports (optional)** + +- [ ] Added `cvs/lib/report/presets/.py` when using `--html` (see [Step 8](#step-8-suite-reports-optional-html)) diff --git a/cvs/lib/inference/base.py b/cvs/lib/inference/base.py index d735f1658..7c2bf0c22 100644 --- a/cvs/lib/inference/base.py +++ b/cvs/lib/inference/base.py @@ -7,9 +7,14 @@ import os import re +import shlex import time from cvs.lib import globals +from cvs.lib.inference.utils.vllm_benchmark_scripts import ( + bash_export_bench_script_from_vllm_install, + clamped_bench_random_range_ratio_str, +) from cvs.lib.utils_lib import * from cvs.lib.verify_lib import * from cvs.lib import linux_utils @@ -45,7 +50,9 @@ def __init__( hf_token, gpu_type='mi300', distributed_inference=False, - server_launch_poll_count=20, + # 60 * 60s polls after warmup matches VllmJob: large HF model cache + weight load + # on MI300 can exceed 20min with little log churn before Uvicorn prints ready. + server_launch_poll_count=60, ): # Client instance phdl self.c_phdl = c_phdl @@ -75,6 +82,7 @@ def __init__( self.rdma_stats_dict_after = {} self.inference_start_time = s_phdl.exec('date +"%a %b %e %H:%M"') self.inference_end_time = None + self.inference_results_dict = {} self.home_dir = os.path.expanduser("~") self.if_dict.setdefault('container_image', 'rocm/7.0:rocm7.0_ubuntu_22.04_vllm_0.10.1_instinct_20250927_rc1') @@ -90,7 +98,6 @@ def __init__( self.if_dict.setdefault('nccl_debug', 'ERROR') self.if_dict.setdefault('data_cache_dir', f'{self.home_dir}/cache') self.if_dict.setdefault('log_dir', f'{self.home_dir}/LOG_DIR') - self.if_dict.setdefault('benchmark_script_repo', 'https://github.com/kimbochen/bench_serving.git') log.info('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%') log.info(f'inference_dict = {self.if_dict}') @@ -138,8 +145,6 @@ def __init__( 'failed to start|no such file or directory|command not found|cannot access', re.I ) self.default_client_wait_time = 120 - self.default_client_poll_count = 20 - self.default_client_poll_wait_time = 60 # Regex/parse defaults that derived classes may override self.readiness_pattern = re.compile('Application startup complete|Uvicorn running|Started server', re.I) @@ -157,13 +162,32 @@ def __init__( self.bp_dict.setdefault('seed', '0') self.bp_dict.setdefault('request_rate', 'inf') self.bp_dict.setdefault('max_model_length', '9216') - self.bp_dict.setdefault('random_range_ration', '1.0') + self.bp_dict.setdefault('random_range_ratio', '1.0') self.bp_dict.setdefault('random_prefix_len', '0') self.bp_dict.setdefault('tensor_parallelism', '1') self.bp_dict.setdefault('port_no', '8000') self.bp_dict.setdefault('tokenizer_mode', 'auto') self.bp_dict.setdefault('percentile_metrics', 'ttft,tpot,itl,e2el') self.bp_dict.setdefault('metric_percentiles', '99') + # Bench client can exceed 20min for large num_prompts × long ISL/OSL; budget is + # default_client_wait_time + client_poll_count * client_poll_wait_time. + self.bp_dict.setdefault('client_poll_count', '50') + self.bp_dict.setdefault('client_poll_wait_time', '60') + self.bp_dict.setdefault('bench_max_failed_requests', '0') + try: + self.default_client_poll_count = max(1, int(float(str(self.bp_dict['client_poll_count']).strip()))) + except (TypeError, ValueError): + self.default_client_poll_count = 50 + try: + self.default_client_poll_wait_time = max(1, int(float(str(self.bp_dict['client_poll_wait_time']).strip()))) + except (TypeError, ValueError): + self.default_client_poll_wait_time = 60 + try: + self.bench_max_failed_requests_cap = max( + 0, int(float(str(self.bp_dict['bench_max_failed_requests']).strip())) + ) + except (TypeError, ValueError): + self.bench_max_failed_requests_cap = 0 # Set server and client scripts self.server_script = self.bp_dict['server_script'] @@ -247,6 +271,9 @@ def exec_nic_setup_scripts( def build_server_inference_job_cmd( self, ): + eager_line = ( + "\n export VLLM_ENFORCE_EAGER=1" if self.if_dict.get("vllm_enforce_eager") else "" + ) s_cmd = f'''docker exec {self.container_name} /bin/bash -c "echo ' export MODEL={self.bp_dict['model']} export ISL={self.bp_dict['input_sequence_length']} @@ -258,7 +285,7 @@ def build_server_inference_job_cmd( export HF_TOKEN={self.hf_token} export VLLM_USE_AITER_UNIFIED_ATTENTION=1 export VLLM_ROCM_USE_AITER_MHA=0 - export VLLM_ROCM_USE_AITER_FUSED_MOE_A16W4=1 + export VLLM_ROCM_USE_AITER_FUSED_MOE_A16W4=1{eager_line} export RESULT_FILENAME=results export PORT={self.bp_dict['port_no']}' > /tmp/server_env_script.sh" ''' @@ -298,14 +325,10 @@ def build_server_inference_job_cmd( self.s_phdl.exec_cmd_list(cmd_list) def clone_bench_serving_repo(self, clone_dir): - """Clone bench_serving repository for client benchmarks.""" - cmd = f'''docker exec {self.container_name} /bin/bash -c "cd {clone_dir}; git clone {self.if_dict['benchmark_script_repo']}" ''' - out_dict = self.c_phdl.exec(cmd) - for node in out_dict.keys(): - # Ignore "already exists" error - repo was cloned in previous test - if re.search('error|fail', out_dict[node], re.I) and not re.search('already exists', out_dict[node], re.I): - fail_test('Errors or failures seen in pulling bench_serving repo from Github, pls check') - time.sleep(3) + """No-op: client benchmarks use the installed ``vllm`` package ``benchmarks/``.""" + log.info( + "clone_bench_serving_repo skipped; using vLLM-shipped benchmarks/ (no third-party bench_serving clone)" + ) def launch_server(self): """Launch inference server.""" @@ -352,10 +375,26 @@ def is_server_ready(self, out_dict, readiness_pattern): node_ready = {node: bool(readiness_pattern.search(output or '')) for node, output in out_dict.items()} return bool(node_ready) and all(node_ready.values()) + def _readiness_grep_cmd_list(self, log_file: str) -> list[str]: + """Remote bash lines: print CVS_SERVER_READY if the full server log matches readiness. + + Uses grep on the whole file (not tail) so the marker is not lost once vLLM logs scroll. + """ + pat = self.readiness_pattern.pattern + cmd_list: list[str] = [] + for i in range(0, int(self.nnodes)): + path = f'{self.log_dir}/{self.get_log_subdir()}/out-node{i}/{log_file}'.replace('\\', '/') + inner = f'grep -qiE {shlex.quote(pat)} {shlex.quote(path)} && echo CVS_SERVER_READY || true' + cmd_list.append(f'bash -c {shlex.quote(inner)}') + return cmd_list + + @staticmethod + def _grep_readiness_outputs_ok(out_dict: dict) -> bool: + return bool(out_dict) and all('CVS_SERVER_READY' in (output or '') for output in out_dict.values()) + def poll_server_startup(self): """Poll for server startup completion.""" log_file = f'{self.server_script}_server.log' - readiness_pattern = self.readiness_pattern # Do an early check for fast failures before the long wait log.info(f'Waiting {self.default_server_precheck_wait_time} secs for server to start writing logs...') @@ -381,11 +420,10 @@ def poll_server_startup(self): for j in range(0, self.default_server_poll_count): log.info(f'Polling for application startup complete on all nodes, iteration {j}') - cmd_list = [] + tail_cmds = [] for i in range(0, int(self.nnodes)): - cmd = f'tail -30 {self.log_dir}/{self.get_log_subdir()}/out-node{i}/{log_file}' - cmd_list.append(cmd) - out_dict = self.s_phdl.exec_cmd_list(cmd_list) + tail_cmds.append(f'tail -30 {self.log_dir}/{self.get_log_subdir()}/out-node{i}/{log_file}') + out_dict = self.s_phdl.exec_cmd_list(tail_cmds) for node in out_dict.keys(): if self.default_server_error_pattern_poll.search(out_dict[node] or ''): @@ -393,7 +431,8 @@ def poll_server_startup(self): fail_test(error_msg) raise Exception(error_msg) - if self.is_server_ready(out_dict, readiness_pattern): + grep_out = self.s_phdl.exec_cmd_list(self._readiness_grep_cmd_list(log_file)) + if self._grep_readiness_outputs_ok(grep_out): log.info('Server startup confirmed on all nodes') return @@ -410,11 +449,30 @@ def launch_client(self): backend = self.bp_dict['backend'] result_filename = self.get_result_filename() + export_bench = bash_export_bench_script_from_vllm_install(self.bench_serv_script) + + rr_str, rr_clamped = clamped_bench_random_range_ratio_str( + self.bp_dict["random_range_ratio"], + self.bp_dict["input_sequence_length"], + self.bp_dict["output_sequence_length"], + self.bp_dict["max_model_length"], + ) + if rr_clamped: + log.info( + "CVS: clamped --random-range-ratio from %s to %s so peak random (ISL+OSL)*(1+r) " + "fits max_model_length=%s (ISL=%s OSL=%s)", + self.bp_dict["random_range_ratio"], + rr_str, + self.bp_dict["max_model_length"], + self.bp_dict["input_sequence_length"], + self.bp_dict["output_sequence_length"], + ) + # Launch client benchmark cmd_list = [] for i in range(0, int(self.nnodes)): - client_cmd = f'''source /tmp/server_env_script.sh; cd {clone_dir}; \ - python3 bench_serving/{self.bench_serv_script} \ + client_cmd = f'''source /tmp/server_env_script.sh; {export_bench}; cd {clone_dir}; \ + _cvs_run_bench \ --model {self.bp_dict['model']} \ --backend {backend} \ --base-url {self.bp_dict['base_url']}:{self.bp_dict['port_no']} \ @@ -427,15 +485,17 @@ def launch_client(self): --burstiness {self.bp_dict['burstiness']} \ --tokenizer-mode {self.bp_dict['tokenizer_mode']} \ --seed {self.bp_dict['seed']} \ - --random-range-ratio {self.bp_dict['random_range_ratio']} \ + --random-range-ratio {rr_str} \ --random-prefix-len {self.bp_dict['random_prefix_len']} \ --percentile-metrics {self.bp_dict['percentile_metrics']} \ + --metric-percentiles {self.bp_dict['metric_percentiles']} \ + --temperature 0 \ --ignore-eos \ --save-result \ --result-dir {self.log_dir}/{self.get_log_subdir()}/out-node{i} \ --result-filename {result_filename} \ > {self.log_dir}/{self.get_log_subdir()}/out-node{i}/bench_serv_script.log 2>&1 &''' - cmd = f'''docker exec {self.container_name} /bin/bash -c "{client_cmd}" ''' + cmd = f"docker exec {shlex.quote(str(self.container_name))} /bin/bash -c {shlex.quote(client_cmd)}" cmd_list.append(cmd) self.c_phdl.exec_cmd_list(cmd_list) @@ -450,13 +510,35 @@ def poll_client_completion(self): cmd = f'tail -30 {self.log_dir}/{self.get_log_subdir()}/out-node{i}/bench_serv_script.log' cmd_list.append(cmd) out_dict = self.c_phdl.exec_cmd_list(cmd_list) + done = [] for node in out_dict.keys(): - if re.search('Failed', out_dict[node], re.I): + log_tail = out_dict[node] or '' + if re.search(r"can't open file|No such file or directory", log_tail, re.I): + fail_test( + f'Benchmark script missing or unreadable on node {node} ' + f'(see bench_serv_script.log); install vllm[bench] or use an image with benchmarks/.' + ) + return + if re.search('Failed', log_tail, re.I): fail_test(f'Failed to run benchmark script on node {node}') return - if not re.search('End-to-end Latency', out_dict[node], re.I): - log.info(f'Waiting {self.default_client_poll_wait_time} secs for next poll') - time.sleep(self.default_client_poll_wait_time) + done.append( + bool( + re.search( + r'Serving Benchmark Result|End-to-end Latency', + log_tail, + re.I, + ) + ) + ) + if done and all(done): + log.info('Benchmark client complete on all nodes (iter=%d)', j) + return + log.info(f'Waiting {self.default_client_poll_wait_time} secs for next poll') + time.sleep(self.default_client_poll_wait_time) + msg = 'client did not complete before poll cap' + fail_test(msg) + raise Exception(msg) def start_inference_server_job( self, @@ -471,7 +553,7 @@ def start_inference_client_job( ): log.info('Start Client side benchmark script on all Nodes') - # Clone bench_serving repo to /app + # Resolve benchmark driver from the installed vllm package (see inference.utils.vllm_benchmark_scripts) self.clone_bench_serving_repo('/app') if self.distributed_inference: @@ -498,7 +580,7 @@ def get_inference_results_dict(self, out_dict): self.inference_results_dict[node]['total_input_tokens'] = match.group(1) if re.search('Total generated tokens:', out_dict[node], re.I): match = re.search('Total generated tokens:\s+([0-9\.]+)', out_dict[node], re.I) - self.inference_results_dict[node]['Total generated tokens:'] = match.group(1) + self.inference_results_dict[node]['total_generated_tokens'] = match.group(1) if re.search('Request throughput \(req/s\):', out_dict[node], re.I): match = re.search('Request throughput \(req/s\):\s+([0-9\.]+)', out_dict[node], re.I) self.inference_results_dict[node]['request_throughput_per_sec'] = match.group(1) @@ -511,20 +593,20 @@ def get_inference_results_dict(self, out_dict): if re.search('Mean TTFT \(ms\):', out_dict[node], re.I): match = re.search('Mean TTFT \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) self.inference_results_dict[node]['mean_ttft_ms'] = match.group(1) - if re.search('Median TTFT (ms):', out_dict[node], re.I): - match = re.search('Median TTFT \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) + if re.search(r'Median TTFT \(ms\):', out_dict[node], re.I): + match = re.search(r'Median TTFT \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) self.inference_results_dict[node]['median_ttft_ms'] = match.group(1) - if re.search('P99 TTFT (ms):', out_dict[node], re.I): - match = re.search('P99 TTFT \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) + if re.search(r'P99 TTFT \(ms\):', out_dict[node], re.I): + match = re.search(r'P99 TTFT \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) self.inference_results_dict[node]['p99_ttft_ms'] = match.group(1) if re.search('Mean TPOT \(ms\)', out_dict[node], re.I): match = re.search('Mean TPOT \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) self.inference_results_dict[node]['mean_tpot_ms'] = match.group(1) if re.search('Median TPOT \(ms\):', out_dict[node], re.I): - match = re.search('Median TPOT \(ms\):\s+([0-9]+)', out_dict[node], re.I) + match = re.search('Median TPOT \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) self.inference_results_dict[node]['median_tpot_ms'] = match.group(1) - if re.search('P99 TPOT (ms):', out_dict[node], re.I): - match = re.search('P99 TPOT \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) + if re.search(r'P99 TPOT \(ms\):', out_dict[node], re.I): + match = re.search(r'P99 TPOT \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) self.inference_results_dict[node]['p99_tpot_ms'] = match.group(1) if re.search('Mean ITL \(ms\):', out_dict[node], re.I): match = re.search('Mean ITL \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) diff --git a/cvs/lib/inference/inference_max.py b/cvs/lib/inference/inference_max.py deleted file mode 100644 index 61f91844d..000000000 --- a/cvs/lib/inference/inference_max.py +++ /dev/null @@ -1,56 +0,0 @@ -''' -Copyright 2025 Advanced Micro Devices, Inc. -All rights reserved. This notice is intended as a precaution against inadvertent publication and does not imply publication or any waiver of confidentiality. -The year included in the foregoing notice is the year of creation of the work. -All code contained here is Property of Advanced Micro Devices, Inc. -''' - -import re -import time - -from cvs.lib.inference.base import InferenceBaseJob -from cvs.lib.verify_lib import fail_test - - -class InferenceMaxJob(InferenceBaseJob): - """InferenceMAX-specific implementation.""" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.if_dict.setdefault('inferencemax_repo', 'https://github.com/InferenceMAX/InferenceMAX.git') - - def get_server_script_directory(self): - """InferenceMAX scripts are in the cloned repo.""" - return '/app/InferenceX' - - def get_server_script_path(self): - """InferenceMAX scripts are in the cloned repo.""" - base = "single_node" if int(self.nnodes) == 1 else "multi_node" - return f'benchmarks/{base}/{self.server_script}' - - def get_result_filename(self): - """InferenceMAX result filename.""" - return 'inferencemax_test_result.json' - - def get_completion_pattern(self): - """InferenceMAX completion pattern.""" - return re.compile('Serving Benchmark Result', re.I) - - def get_log_subdir(self): - """InferenceMAX uses 'inference-max' log subdirectory.""" - return 'inference-max' - - def clone_inferencemax_repo(self): - """Clone InferenceMAX repository.""" - cmd = f'''docker exec {self.container_name} /bin/bash -c "git clone {self.if_dict['inferencemax_repo']}" ''' - out_dict = self.s_phdl.exec(cmd) - for node in out_dict.keys(): - if re.search('error|fail', out_dict[node], re.I): - fail_test('Errors or failures seen in pulling InferenceMAX repo from Github, pls check') - time.sleep(3) - self.s_phdl.exec(f'''docker exec {self.container_name} /bin/bash -c "ls -ld /app/InferenceX" ''') - - def start_inference_server_job(self): - """Start InferenceMAX server - clone repo, then call base implementation.""" - self.clone_inferencemax_repo() - super().start_inference_server_job() diff --git a/cvs/lib/inference/inferencex_atom/__init__.py b/cvs/lib/inference/inferencex_atom/__init__.py new file mode 100644 index 000000000..5ee11fc11 --- /dev/null +++ b/cvs/lib/inference/inferencex_atom/__init__.py @@ -0,0 +1 @@ +'''InferenceX ATOM suite library (orchestrator, config loader, parsing).''' diff --git a/cvs/lib/inference/inferencex_atom/inferencex_atom_config_loader.py b/cvs/lib/inference/inferencex_atom/inferencex_atom_config_loader.py new file mode 100644 index 000000000..1d481b75b --- /dev/null +++ b/cvs/lib/inference/inferencex_atom/inferencex_atom_config_loader.py @@ -0,0 +1,368 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +InferenceX ATOM suite config schema (``inferencex_atom``). + +Generic paths/model/container/threshold plumbing lives in +:mod:`cvs.lib.utils.config_loader`. Sweep selector types are shared with +:mod:`cvs.lib.inference.utils.inferencing_config_loader`. +''' + +from __future__ import annotations + +import re +from typing import Any, Dict, List, Optional, Union + +from pydantic import field_validator, model_validator +from typing_extensions import Literal + +INFERENCEX_ATOM_DRIVERS = ("atom", "vllm", "vllm_atom", "sglang") +INFERENCEX_ATOM_PP_DRIVERS = ("vllm", "vllm_atom", "sglang") + +from cvs.lib.inference.utils.inferencing_config_loader import ( + RoleServer, + Sweep, + validate_sweep_selector, + validate_thresholds_cover_sweep, +) +from cvs.lib.inference.inferencex_atom.inferencex_atom_parsing import GATED_METRICS +from cvs.lib.utils.config_loader import BaseVariantConfig, _Forbid, substitute_config +from cvs.lib import globals + +log = globals.log + +# Written by test_discover_topology / resolve_multinode_fabric — not user env. +_ORCH_MANAGED_NETWORK_ENV = frozenset( + {"NCCL_SOCKET_IFNAME", "GLOO_SOCKET_IFNAME", "TP_SOCKET_IFNAME", "NCCL_IB_HCA"} +) +_IB_HCA_NETDEV_RE = re.compile(r"^mlx5_\d+$", re.I) + + +class InferenceXAtomRoleServer(RoleServer): + # Extra CLI tokens for ``python -m atom.entrypoints.openai_server`` after + # ``--model`` / ``--server-port`` (e.g. ``-tp``, ``--kv_cache_dtype``). + atom_args: List[str] = [] + # Extra CLI tokens appended to ``python3 -m sglang.launch_server`` (driver=sglang). + sglang_args: List[str] = [] + # IB HCA devices for NCCL_IB_HCA (multinode only). + # absent or "auto" -> use whatever ibv_devinfo -l reports (test_discover_topology). + # explicit list -> validated at preflight against ibv_devinfo output. + ib_hca_devices: Union[Literal["auto"], List[str], None] = None + # Linux netdev for NCCL_SOCKET_IFNAME / GLOO_SOCKET_IFNAME on multinode PP runs. + # absent or "auto" -> resolved at runtime by test_discover_topology from cluster IPs. + ib_netdev: Union[Literal["auto"], str, None] = None + + @field_validator("ib_netdev", mode="after") + @classmethod + def _normalize_ib_netdev(cls, v): + raw = (v or "").strip() + if raw and raw.lower() != "auto" and _IB_HCA_NETDEV_RE.match(raw): + log.warning( + "roles.server.ib_netdev=%r looks like an IB HCA name; coercing to 'auto' " + "(socket netdev is discovered from cluster IPs at runtime)", + raw, + ) + return "auto" + return v + + @model_validator(mode="after") + def _strip_orchestrator_managed_network_env(self): + if not self.env: + return self + dropped = sorted(k for k in self.env if k in _ORCH_MANAGED_NETWORK_ENV) + if not dropped: + return self + log.warning( + "roles.server.env drops orchestrator-managed keys %s " + "(set by test_discover_topology / build_server_cmd instead)", + dropped, + ) + self.env = {k: v for k, v in self.env.items() if k not in _ORCH_MANAGED_NETWORK_ENV} + return self + + +class InferenceXAtomRoles(_Forbid): + server: InferenceXAtomRoleServer = InferenceXAtomRoleServer() + + +class InferenceXAtomParams(_Forbid): + # ``atom`` = standalone ATOM openai_server + benchmark_serving. + # ``vllm_atom`` = vLLM coordinator + ATOM local kernels (true multinode PP). + # ``vllm`` = interim ROCm vLLM uplift (vllm serve + vllm bench serve). + # ``sglang`` = SGLang coordinator (launch_server + bench_serving) for PP runs. + driver: Literal["atom", "vllm", "vllm_atom", "sglang"] = "vllm" + backend: str = "vllm" + base_url: str = "http://0.0.0.0" + port_no: str = "8000" + dataset_name: str = "random" + burstiness: str = "1.0" + seed: str = "0" + request_rate: str = "inf" + random_range_ratio: str = "0.8" + random_prefix_len: str = "0" + tensor_parallelism: str = "8" + tokenizer_mode: str = "auto" + percentile_metrics: str = "ttft,tpot,itl,e2el" + metric_percentiles: str = "95,99" + num_prompts: str = "1000" + max_model_length: str = "8192" + client_poll_count: str = "50" + client_poll_wait_time: str = "60" + client_initial_wait_s: str = "120" + server_precheck_wait_s: str = "30" + server_warmup_wait_s: str = "330" + server_poll_count: str = "60" + server_poll_wait_time: str = "60" + reuse_server_across_sweep: str = "false" + bench_max_failed_requests: str = "0" + bench_extra_args: str = "" + result_filename: str = "results" + # Multinode (M5): omit or set nnodes=1 for single-node runs. When nnodes>1, + # cluster node_dict must list the same number of hosts and test_setup_sshd runs. + nnodes: str = "1" + pipeline_parallel_size: str = "1" + master_addr: str = "" + master_port: str = "29501" + # Optional single-node reference output_throughput for scaling.efficiency_pct. + scaling_baseline_output_throughput: str = "" + + +class InferenceXAtomRunCard(_Forbid): + upstream_run_url: str = "" + atom_image_pin: str = "" + notes: str = "" + + +INFERENCEX_ATOM_FRAMEWORKS = ("inferencex_atom",) + + +class InferenceXAtomVariantConfig(BaseVariantConfig): + framework: Literal["inferencex_atom"] + + @field_validator("framework", mode="before") + @classmethod + def _normalize_deprecated_framework(cls, value): + if value == "inferencex_atom_single": + return "inferencex_atom" + return value + + gpu_arch: str + run_card: InferenceXAtomRunCard = InferenceXAtomRunCard() + roles: InferenceXAtomRoles = InferenceXAtomRoles() + params: InferenceXAtomParams + sweep: Sweep + + def cell_key(self, isl, osl, concurrency): + p = self.params + key = f"ISL={isl},OSL={osl},TP={p.tensor_parallelism}" + nnodes = int(p.nnodes) + pp = int(p.pipeline_parallel_size) + if p.driver == "atom": + if nnodes > 1: + key += f",DP={nnodes},NNODES={nnodes}" + elif p.driver in INFERENCEX_ATOM_PP_DRIVERS: + if pp > 1 or nnodes > 1: + key += f",PP={p.pipeline_parallel_size}" + if nnodes > 1: + key += f",NNODES={p.nnodes}" + return f"{key},CONC={concurrency}" + + def expected_cells(self) -> List[str]: + by_name = {c.name: c for c in self.sweep.sequence_combinations} + return [self.cell_key(by_name[r.combo].isl, by_name[r.combo].osl, r.concurrency) for r in self.sweep.runs] + + @model_validator(mode="after") + def _check_thresholds_cover_sweep(self): + validate_thresholds_cover_sweep( + expected_cells=self.expected_cells(), + thresholds=self.thresholds, + enforce_thresholds=self.enforce_thresholds, + gated_metrics=GATED_METRICS, + ) + if int(self.params.nnodes) > 1 and (self.params.scaling_baseline_output_throughput or "").strip(): + missing = [] + for cell in self.expected_cells(): + specs = self.thresholds.get(cell) or {} + if "scaling.efficiency_pct" not in specs: + missing.append(cell) + if missing: + msg = ( + "multinode variant with scaling_baseline_output_throughput requires " + f"scaling.efficiency_pct in every cell; missing: {missing}" + ) + if self.enforce_thresholds: + raise ValueError(msg) + import warnings + + warnings.warn(f"{msg} (enforce_thresholds=false -> record-only)", stacklevel=2) + return self + + @model_validator(mode="after") + def _atom_multinode_uses_dp_not_pp(self): + if self.params.driver == "atom" and int(self.params.nnodes) > 1: + if int(self.params.pipeline_parallel_size) > 1: + raise ValueError( + "params.driver='atom' with nnodes>1 uses ATOM SPMD data parallel (-dp); " + "standalone ATOM cannot execute pipeline parallel. For true PP>1 use " + "params.driver='vllm_atom' or 'sglang'." + ) + return self + + @model_validator(mode="after") + def _pp_driver_distributed_consistency(self): + driver = self.params.driver + if driver not in INFERENCEX_ATOM_PP_DRIVERS: + return self + nn = int(self.params.nnodes) + pp = int(self.params.pipeline_parallel_size) + is_ray = self.roles.server.serve_args.get("distributed-executor-backend") == "ray" + if nn > 1 and pp == 1 and not is_ray: + raise ValueError( + f"params.driver={driver!r} with nnodes={nn} requires pipeline_parallel_size>1 " + f"(got pp={pp}) for multinode pipeline parallel" + ) + if pp > 1 and nn == 1: + raise ValueError( + f"pipeline_parallel_size={pp} > 1 requires nnodes > 1 (got nnodes={nn}) " + f"for params.driver={driver!r}" + ) + return self + + @model_validator(mode="after") + def _atom_driver_requires_inline_server_args(self): + if self.params.driver == "atom" and not self.roles.server.atom_args: + raise ValueError( + "params.driver='atom' requires roles.server.atom_args " + "(inline ATOM openai_server CLI tokens, vLLM-style)" + ) + return self + + +def expand_sweep(sweep): + """Expand a sweep into ``(cases, ids)`` for pytest parametrization.""" + if hasattr(sweep, "sequence_combinations"): + combos = [c.model_dump() for c in sweep.sequence_combinations] + runs = [r.model_dump() for r in sweep.runs] + else: + combos = sweep.get("sequence_combinations", []) + runs = sweep.get("runs", []) + validate_sweep_selector([c["name"] for c in combos], [r["combo"] for r in runs]) + by_name = {c["name"]: c for c in combos} + cases = [] + ids = [] + for run in runs: + combo = by_name[run["combo"]] + conc = run["concurrency"] + cases.append((combo, conc)) + ids.append(f"{run['combo']}-conc{conc}") + return cases, ids + + +def reuse_server_flag(params) -> bool: + """Return True when ``params.reuse_server_across_sweep`` is a truthy string.""" + raw = str(getattr(params, "reuse_server_across_sweep", "false")).strip().lower() + return raw in ("true", "1", "yes") + + +def server_session_key(variant_config, isl, osl): + """Stable key for server reuse across sweep cells with identical model/shape.""" + p = variant_config.params + roles = variant_config.roles.server + if p.driver == "atom": + server_tokens = tuple(roles.atom_args) + elif p.driver == "sglang": + server_tokens = tuple(roles.sglang_args) + else: + server_tokens = tuple(sorted(roles.serve_args.items())) + return ( + variant_config.model.id, + p.driver, + str(isl), + str(osl), + server_tokens, + p.tensor_parallelism, + p.nnodes, + p.pipeline_parallel_size, + p.master_addr, + p.master_port, + ) + + +def expand_sweep_parametrize(sweep, fixturenames): + """Build pytest parametrize args for inference or metric-tier collection.""" + from cvs.lib.inference.inferencex_atom.inferencex_atom_parsing import METRIC_TIER_ORDER + + cases, ids = expand_sweep(sweep) + if "metric_tier" in fixturenames: + if not cases: + return None + tier_cases = [] + tier_ids = [] + for (combo, c), cid in zip(cases, ids): + for tier in METRIC_TIER_ORDER: + tier_cases.append((combo, c, tier)) + tier_ids.append(f"{cid}-{tier}") + return ("seq_combo,concurrency,metric_tier", tier_cases, tier_ids) + if "seq_combo" in fixturenames and "concurrency" in fixturenames and cases: + return ("seq_combo,concurrency", cases, ids) + return None + + +def load_variant(config_path, cluster_dict) -> InferenceXAtomVariantConfig: + raw, thresholds = substitute_config(config_path, cluster_dict) + raw["thresholds"] = thresholds + return InferenceXAtomVariantConfig(**raw) + + +def placeholder_gated_threshold_cell( + *, + output_throughput_min: float = 0, + total_token_throughput_min: float = 0, + per_gpu_throughput_min: float = 0, + output_tput_per_gpu_min: float = 0, + mean_ttft_max_ms: float = 1_000_000, + p99_ttft_max_ms: float = 1_000_000, + mean_tpot_max_ms: float = 1_000_000, + p95_tpot_max_ms: float = 1_000_000, + failed_max: int = 1_000_000_000, + success_rate_min: float = 0, +) -> Dict[str, Any]: + """Return one sweep cell's ``client.*`` specs covering every ``GATED_METRICS`` member.""" + loose_ms = {"kind": "max_ms", "value": 1_000_000} + return { + "client.total_token_throughput": {"kind": "min_tok_s", "value": total_token_throughput_min}, + "client.output_throughput": {"kind": "min_tok_s", "value": output_throughput_min}, + "client.per_gpu_throughput": {"kind": "min_tok_s", "value": per_gpu_throughput_min}, + "client.output_tput_per_gpu": {"kind": "min_tok_s", "value": output_tput_per_gpu_min}, + "client.mean_ttft_ms": {"kind": "max_ms", "value": mean_ttft_max_ms}, + "client.median_ttft_ms": loose_ms, + "client.p90_ttft_ms": loose_ms, + "client.p95_ttft_ms": loose_ms, + "client.p99_ttft_ms": {"kind": "max_ms", "value": p99_ttft_max_ms}, + "client.mean_tpot_ms": {"kind": "max_ms", "value": mean_tpot_max_ms}, + "client.median_tpot_ms": loose_ms, + "client.p90_tpot_ms": loose_ms, + "client.p95_tpot_ms": {"kind": "max_ms", "value": p95_tpot_max_ms}, + "client.p99_tpot_ms": loose_ms, + "client.mean_itl_ms": loose_ms, + "client.median_itl_ms": loose_ms, + "client.p95_itl_ms": loose_ms, + "client.p99_itl_ms": loose_ms, + "client.mean_e2el_ms": loose_ms, + "client.median_e2el_ms": loose_ms, + "client.p90_e2el_ms": loose_ms, + "client.p95_e2el_ms": loose_ms, + "client.p99_e2el_ms": loose_ms, + "client.success_rate": {"kind": "min", "value": success_rate_min}, + "client.failed": {"kind": "max", "value": failed_max}, + } + + +def orchestrator_container_from_variant(variant: InferenceXAtomVariantConfig) -> Dict[str, Any]: + """``container`` block for :class:`OrchestratorConfig` (includes server env).""" + block = variant.container.model_dump() + server_env = variant.roles.server.env + if server_env: + block = {**block, "env": dict(server_env)} + return block diff --git a/cvs/lib/inference/inferencex_atom/inferencex_atom_orch.py b/cvs/lib/inference/inferencex_atom/inferencex_atom_orch.py new file mode 100644 index 000000000..4e01ca801 --- /dev/null +++ b/cvs/lib/inference/inferencex_atom/inferencex_atom_orch.py @@ -0,0 +1,882 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +InferenceX ATOM job driven by a ContainerOrchestrator (single- or multi-node). + +``params.driver=atom`` (target): ``atom.entrypoints.openai_server`` + +``atom.benchmarks.benchmark_serving`` with ATOM JSON artifacts. Standalone ATOM +has no native pipeline parallel; multinode ``atom`` uses SPMD data parallel +(``-dp`` + ``ATOM_DP_*``) when scale-out is needed. + +``params.driver=vllm_atom``: ``vllm serve`` + ``vllm bench serve`` with vLLM as +the multinode coordinator (``--pipeline-parallel-size``, ``--node-rank``, …) +while ATOM accelerates local kernels via ROCm vLLM env flags. + +``params.driver=sglang``: ``sglang.launch_server`` + ``sglang.bench_serving`` with +SGLang PP flags (``--pp-size``, ``--nnodes``, ``--dist-init-addr``). + +``params.driver=vllm`` (interim uplift): same coordinator path as ``vllm_atom`` +without the ATOM-specific ROCm env block. + +Does NOT subclass :class:`cvs.lib.inference.base.InferenceBaseJob`. +''' + +from __future__ import annotations + +import json +import re +import shlex +import time + +from cvs.lib import globals +from cvs.lib.inference.inferencex_atom.inferencex_atom_parsing import to_client_metrics + +log = globals.log + + +class InferenceXAtomJob: + """InferenceX ATOM benchmark job driven by an injected ContainerOrchestrator.""" + + READINESS_RE = re.compile(r"Application startup complete|Uvicorn running|Started server", re.I) + COMPLETION_RE = re.compile(r"Serving Benchmark Result", re.I) + FAILED_REQUESTS_RE = re.compile(r"Failed requests:\s+([0-9]+)", re.I) + CLIENT_CRASH_RE = re.compile(r"Traceback \(most recent call last\)", re.I) + CLIENT_LAUNCH_FAIL_RE = re.compile( + r"unrecognized arguments|invalid choice|error: argument |command not found|: No such file or directory", + re.I, + ) + EARLY_FAILURE_RE = re.compile( + r"no such file or directory|command not found|cannot access|failed to start" + r"|unrecognized arguments|invalid choice|error: argument " + r"|Free memory on device.*less than desired" + r"|Engine core initialization failed" + r"|WorkerProc failed to start", + re.I, + ) + FATAL_LOG_RE = re.compile( + r"Free memory on device.{0,80}less than desired" + r"|Engine core initialization failed" + r"|RuntimeError:.*[Ee]ngine", + re.I, + ) + + _DEFAULT_SERVE_ARGS = { + "block-size": 64, + "no-enable-prefix-caching": True, + } + + # vLLM multinode flags; ATOM openai_server rejects these (use ATOM_DP_* / -dp instead). + _VLLM_DISTRIBUTED_FLAGS = frozenset( + { + "--node-rank", + "--master-addr", + "--master-port", + "--nnodes", + "--pipeline-parallel-size", + "--distributed-executor-backend", + } + ) + + def __init__( + self, + orch, + variant, + hf_token, + isl, + osl, + concurrency, + num_prompts, + log_subdir="inferencex-atom", + server_precheck_wait_s=30, + server_warmup_wait_s=330, + server_poll_count=60, + server_poll_wait_s=60, + client_initial_wait_s=120, + client_poll_count=50, + client_poll_wait_s=60, + bench_max_failed_requests=0, + ib_hcas=None, + ib_netdev=None, + ): + self.orch = orch + self.variant = variant + self.hf_token = hf_token + self.isl = str(isl) + self.osl = str(osl) + self.concurrency = str(concurrency) + self.num_prompts = str(num_prompts) + self.log_subdir = log_subdir + + p = variant.params + self.driver = str(p.driver or "atom").strip().lower() + self.tp = p.tensor_parallelism + self.pp = p.pipeline_parallel_size + self.nnodes = int(p.nnodes) + self.distributed = self.nnodes > 1 + raw_master = (p.master_addr or "").strip() + self.master_addr = raw_master or (orch.hosts[0] if getattr(orch, "hosts", None) else "localhost") + self.master_port = p.master_port + self.port_no = p.port_no + self.random_range_ratio = p.random_range_ratio + self.random_prefix_len = p.random_prefix_len + self.burstiness = p.burstiness + self.seed = p.seed + self.request_rate = p.request_rate + self.tokenizer_mode = p.tokenizer_mode + self.percentile_metrics = p.percentile_metrics + self.metric_percentiles = p.metric_percentiles + self.base_url = p.base_url + self.dataset_name = p.dataset_name + self.backend = p.backend + self.max_model_length = str(p.max_model_length) + self.bench_extra_args = (p.bench_extra_args or "").strip() + self.result_stem = (p.result_filename or "results").removesuffix(".json") + raw_baseline = (p.scaling_baseline_output_throughput or "").strip() + self._scaling_baseline = float(raw_baseline) if raw_baseline else None + + self.model_id = variant.model.id + self.log_dir = variant.paths.log_dir + self.models_dir = variant.paths.models_dir + self.serve_args = self._merged_serve_args(variant) + self.atom_server_args = list(variant.roles.server.atom_args) + self.sglang_server_args = list(variant.roles.server.sglang_args) + self.server_env = dict(variant.roles.server.env) + configured_netdev = (getattr(variant.roles.server, "ib_netdev", None) or "").strip() + if ib_netdev: + self.ib_netdev = str(ib_netdev).strip() + elif configured_netdev and configured_netdev.lower() != "auto": + self.ib_netdev = configured_netdev + else: + self.ib_netdev = "" + # Discovered HCA names for NCCL_IB_HCA (multinode only). Prefilled by + # test_discover_topology; build_server_cmd can resolve lazily if omitted. + self.ib_hcas = ib_hcas or [] + + self.out_dir = self._node_out_dir(0) + self.server_log = self._rank_server_log(0) + self.client_log = f"{self.out_dir}/client.log" + self._result_artifact = ( + f"{self.out_dir}/{self.result_stem}.json" if self.driver == "atom" else f"{self.out_dir}/{self.result_stem}" + ) + + self._precheck_wait = server_precheck_wait_s + self._warmup_wait = server_warmup_wait_s + self._server_poll_count = server_poll_count + self._server_poll_wait = server_poll_wait_s + self._client_initial_wait = client_initial_wait_s + self._client_poll_count = client_poll_count + self._client_poll_wait = client_poll_wait_s + self._bench_max_failed_requests = int(bench_max_failed_requests) + + @classmethod + def from_variant(cls, orch, variant, hf_token, isl, osl, concurrency, **overrides): + """Construct a job with server/client timing from ``variant.params``.""" + p = variant.params + + def _int_attr(name, default): + raw = getattr(p, name, None) + if raw is None or str(raw).strip() == "": + return default + try: + return int(raw) + except (TypeError, ValueError): + return default + + kw = dict( + orch=orch, + variant=variant, + hf_token=hf_token, + isl=isl, + osl=osl, + concurrency=concurrency, + num_prompts=p.num_prompts, + server_precheck_wait_s=_int_attr("server_precheck_wait_s", 30), + server_warmup_wait_s=_int_attr("server_warmup_wait_s", 330), + server_poll_count=_int_attr("server_poll_count", 60), + server_poll_wait_s=_int_attr("server_poll_wait_time", 60), + client_initial_wait_s=_int_attr("client_initial_wait_s", 120), + client_poll_count=_int_attr("client_poll_count", 50), + client_poll_wait_s=_int_attr("client_poll_wait_time", 60), + bench_max_failed_requests=_int_attr("bench_max_failed_requests", 0), + ) + kw.update(overrides) + return cls(**kw) + + def _node_out_dir(self, rank): + return f"{self.log_dir}/{self.log_subdir}/out-node{rank}/isl{self.isl}_osl{self.osl}_conc{self.concurrency}" + + def _uses_vllm_serve(self): + return self.driver in ("vllm", "vllm_atom") + + def _uses_sglang_serve(self): + return self.driver == "sglang" + + def _framework_coordinator_label(self): + if self.driver == "atom": + return "atom" + if self._uses_sglang_serve(): + return "sglang" + return "vllm" + + def _rank_server_log_name(self): + if self.driver == "atom": + return "atom_server.log" + if self._uses_sglang_serve(): + return "sglang_server.log" + return "vllm_serve_server.log" + + def _rank_server_log(self, rank): + base = self._node_out_dir(rank) + return f"{base}/{self._rank_server_log_name()}" + + def _exec_all(self, cmd, **kwargs): + return self.orch.exec(cmd, **kwargs) + + def _exec_head(self, cmd, **kwargs): + if self.distributed: + return self.orch.exec_on_head(cmd, **kwargs) + return self.orch.exec(cmd, **kwargs) + + def prepare_cell_out_dir(self): + """Create per-cell output directory without touching server env or cache.""" + if self.distributed: + for rank in range(self.nnodes): + self._exec_all(f"mkdir -p {shlex.quote(self._node_out_dir(rank))}") + else: + self._exec_all(f"mkdir -p {shlex.quote(self.out_dir)}") + + @classmethod + def _merged_serve_args(cls, variant): + merged = dict(cls._DEFAULT_SERVE_ARGS) + merged.update(variant.roles.server.serve_args) + env = variant.roles.server.env + gpu_mem = env.get("CVS_GPU_MEMORY_UTIL") or env.get("VLLM_GPU_MEMORY_UTIL") + if gpu_mem is not None and "gpu-memory-utilization" not in merged: + merged["gpu-memory-utilization"] = str(gpu_mem) + if "enforce-eager" not in merged: + merged["enforce-eager"] = True + return merged + + @staticmethod + def _flatten_serve_args(mapping): + argv = [] + for flag, value in mapping.items(): + opt = f"--{flag}" + if value is True: + argv.append(opt) + elif isinstance(value, (list, tuple)): + for v in value: + argv.extend([opt, str(v)]) + else: + argv.extend([opt, str(value)]) + return argv + + @staticmethod + def _argv_has_flag(argv, *names): + for tok in argv: + if tok in names: + return True + return False + + def _without_vllm_distributed_flags(self, argv): + """Strip vLLM multinode tokens from ``roles.server.atom_args`` if present.""" + out = [] + skip_next = False + for tok in argv: + if skip_next: + skip_next = False + continue + if tok in self._VLLM_DISTRIBUTED_FLAGS: + skip_next = True + continue + out.append(tok) + return out + + def _atom_spmd_dp_enabled(self): + """True when CVS should wire multinode ATOM SPMD data parallel (``-dp`` + ``ATOM_DP_*``).""" + if self.driver != "atom" or not self.distributed: + return False + atom_argv = self._without_vllm_distributed_flags(self.atom_server_args) + if self._argv_has_flag(atom_argv, "-dp", "--data-parallel-size"): + return False + return True + + def _atom_spmd_dp_cli(self): + """Return ``-dp nnodes`` for coupled multinode ATOM replicas (one DP rank per host).""" + if not self._atom_spmd_dp_enabled(): + return [] + tp = int(self.tp) + if tp > 8: + raise RuntimeError( + f"params.tensor_parallelism={tp} exceeds ATOM local TP limit (8); " + "multinode SPMD runs one TP group per node" + ) + return ["-dp", str(self.nnodes)] + + def _atom_multinode_argv(self): + """ATOM-only multinode CLI tokens (never vLLM ``--node-rank`` / ``--pipeline-parallel-size``).""" + return self._atom_spmd_dp_cli() + + def _atom_spmd_env_exports(self, rank): + if not self._atom_spmd_dp_enabled(): + return [] + return [ + f"export ATOM_DP_RANK={rank}", + f"export ATOM_DP_SIZE={self.nnodes}", + "export ATOM_DP_RANK_LOCAL=0", + f"export ATOM_DP_MASTER_IP={shlex.quote(self.master_addr)}", + f"export ATOM_DP_MASTER_PORT={self.master_port}", + ] + + def _vllm_distributed_argv(self, rank): + if not self.distributed: + return [] + argv = [ + "--node-rank", + str(rank), + "--master-addr", + str(self.master_addr), + "--master-port", + str(self.master_port), + "--nnodes", + str(self.nnodes), + "--pipeline-parallel-size", + str(self.pp), + "--distributed-executor-backend", + "mp", + ] + if rank > 0: + argv.append("--headless") + return argv + + def _ensure_multinode_topology(self): + if not self.distributed: + return + if self.ib_hcas and self.ib_netdev: + return + from cvs.lib.utils.ib_discovery import resolve_multinode_fabric + + roles = self.variant.roles.server + hcas, netdev = resolve_multinode_fabric( + self.orch, + ib_hca_devices=getattr(roles, "ib_hca_devices", None), + ib_netdev=self.ib_netdev or getattr(roles, "ib_netdev", None), + master_addr=self.master_addr, + ) + if not self.ib_hcas: + self.ib_hcas = hcas + if not self.ib_netdev: + self.ib_netdev = netdev + log.info( + "multinode fabric resolved: netdev=%s HCAs=%s", + self.ib_netdev, + self.ib_hcas, + ) + + def build_server_cmd(self, *, clear_atom_cache=True): + self._ensure_multinode_topology() + env_lines = [ + f"export HF_TOKEN={shlex.quote(self.hf_token)}", + f"export HF_HUB_CACHE={shlex.quote(self.models_dir)}", + ] + if self._uses_vllm_serve(): + env_lines.extend( + [ + "export VLLM_USE_AITER_UNIFIED_ATTENTION=1", + "export VLLM_ROCM_USE_AITER_MHA=0", + "export VLLM_ROCM_USE_AITER_FUSED_MOE_A16W4=1", + ] + ) + elif self._uses_sglang_serve(): + env_lines.append("export SGLANG_USE_AITER=1") + if self.ib_hcas: + env_lines.append(f"export NCCL_IB_HCA={shlex.quote(','.join(self.ib_hcas))}") + if self.distributed and not self.ib_netdev: + raise RuntimeError( + "multinode run has no socket netdev after topology resolution " + "(set roles.server.ib_netdev or fix cluster IP discovery)" + ) + if self.distributed and self.ib_netdev: + env_lines.append(f"export NCCL_SOCKET_IFNAME={shlex.quote(self.ib_netdev)}") + env_lines.append(f"export GLOO_SOCKET_IFNAME={shlex.quote(self.ib_netdev)}") + env_lines.append(f"export TP_SOCKET_IFNAME={shlex.quote(self.ib_netdev)}") + for k, v in self.server_env.items(): + if k in ( + "CVS_GPU_MEMORY_UTIL", + "VLLM_GPU_MEMORY_UTIL", + "VLLM_ENFORCE_EAGER", + "NCCL_SOCKET_IFNAME", + "GLOO_SOCKET_IFNAME", + "TP_SOCKET_IFNAME", + "NCCL_IB_HCA", + ): + continue + env_lines.append(f"export {k}={shlex.quote(str(v))}") + env_script = "\n".join(env_lines) + "\n" + self._exec_all("bash -c " + shlex.quote(f"printf '%s' {shlex.quote(env_script)} > /tmp/server_env_script.sh")) + if self.distributed: + for rank in range(self.nnodes): + self._exec_all(f"mkdir -p {shlex.quote(self._node_out_dir(rank))}") + else: + self._exec_all(f"mkdir -p {shlex.quote(self.out_dir)}") + if self.driver == "atom" and clear_atom_cache: + self._exec_all("bash -c 'rm -rf ~/.cache/atom/* 2>/dev/null || true'") + + def _server_argv(self, rank=0): + argv = [ + "vllm", + "serve", + self.model_id, + "--host", + "0.0.0.0", + "--tensor-parallel-size", + str(self.tp), + "--max-model-len", + self.max_model_length, + "--port", + str(self.port_no), + ] + argv.extend(self._vllm_distributed_argv(rank)) + argv.extend(self._flatten_serve_args(self.serve_args)) + return argv + + def _sglang_server_argv(self, rank=0): + argv = [ + "python3", + "-m", + "sglang.launch_server", + "--model-path", + self.model_id, + "--host", + "0.0.0.0", + "--port", + str(self.port_no), + "--tp", + str(self.tp), + ] + if self.distributed: + dist_init = f"{self.master_addr}:{self.master_port}" + argv.extend( + [ + "--pp-size", + str(self.pp), + "--nnodes", + str(self.nnodes), + "--node-rank", + str(rank), + "--dist-init-addr", + dist_init, + ] + ) + argv.extend(self.sglang_server_args) + return argv + + def _server_argv_for_driver(self, rank=0): + if self.driver == "atom": + return self._atom_server_argv(rank) + if self._uses_vllm_serve(): + return self._server_argv(rank) + if self._uses_sglang_serve(): + return self._sglang_server_argv(rank) + raise RuntimeError( + f"unsupported params.driver={self.driver!r}; " + "expected 'atom', 'vllm', 'vllm_atom', or 'sglang'" + ) + + def _atom_server_argv(self, rank=0): + argv = [ + "python", + "-m", + "atom.entrypoints.openai_server", + "--model", + self.model_id, + "--server-port", + str(self.port_no), + ] + argv.extend(self._without_vllm_distributed_flags(self.atom_server_args)) + argv.extend(self._atom_multinode_argv()) + return argv + + def start_server(self): + hosts = list(getattr(self.orch, "hosts", []) or ["node0"]) + if self.distributed and len(hosts) != self.nnodes: + raise RuntimeError( + f"params.nnodes={self.nnodes} but cluster has {len(hosts)} host(s); " + "align cluster node_dict with params.nnodes" + ) + label = self._framework_coordinator_label() + launch_hosts = enumerate(hosts) if self.distributed else [(0, hosts[0])] + for rank, host in launch_hosts: + argv = self._server_argv_for_driver(rank) + serve_cmd = " ".join(shlex.quote(str(a)) for a in argv) + rank_log = self._rank_server_log(rank) + rank_env = " && ".join(self._atom_spmd_env_exports(rank)) + env_prefix = f"{rank_env} && " if rank_env else "" + inner = ( + f"source /tmp/server_env_script.sh && {env_prefix}nohup {serve_cmd} > {shlex.quote(rank_log)} 2>&1 &" + ) + if self.distributed: + out = self._exec_all("bash -c " + shlex.quote(inner), hosts=[host]) + else: + out = self._exec_all("bash -c " + shlex.quote(inner)) + for h, output in out.items(): + if self.EARLY_FAILURE_RE.search(output or ""): + raise RuntimeError(f"{label} server failed to launch on {h} (rank {rank}): {output[-500:]}") + + def _atom_health_ok(self): + url = f"http://localhost:{self.port_no}/health" + probe = f"curl -sf {shlex.quote(url)} -o /dev/null && echo OK || echo NO" + if self.distributed: + out = self._exec_all("bash -c " + shlex.quote(probe)) + else: + out = self._exec_head("bash -c " + shlex.quote(probe)) + return bool(out) and all("OK" in (v or "") for v in out.values()) + + def _atom_warmup_ok(self): + payload = json.dumps( + {"model": self.model_id, "prompt": "hi", "max_tokens": 1}, + separators=(",", ":"), + ) + url = f"http://localhost:{self.port_no}/v1/completions" + inner = ( + f"curl -sf {shlex.quote(url)} -H 'Content-Type: application/json' " + f"-d {shlex.quote(payload)} -o /dev/null --max-time 120 && echo OK || echo NO" + ) + out = self._exec_head("bash -c " + shlex.quote(inner)) + return bool(out) and all("OK" in (v or "") for v in out.values()) + + def is_ready(self): + if self.driver == "atom": + return self._atom_health_ok() + pattern = self.READINESS_RE.pattern + for rank, host in enumerate(self.orch.hosts): + # Headless workers (rank > 0) never log Uvicorn startup; only the head + # API server does. Match vllm_job.is_ready() multinode behaviour. + if rank > 0 and self.nnodes > 1: + continue + rank_log = self._rank_server_log(rank) if self.distributed else self.server_log + out = self.orch.exec( + f"grep -qiE {shlex.quote(pattern)} {shlex.quote(rank_log)}", + detailed=True, + hosts=[host], + ) + if not out or not all(r["exit_code"] == 0 for r in out.values()): + return False + return True + + def _check_coordinator_early_failure(self, emit_tail: bool = False): + """Tail/grep per-rank server logs on each host for fatal startup errors.""" + label = self._framework_coordinator_label() + for rank, host in enumerate(self.orch.hosts): + rank_log = self._rank_server_log(rank) if self.distributed else self.server_log + out = self.orch.exec(f"tail -30 {shlex.quote(rank_log)}", hosts=[host]) + for h, output in (out or {}).items(): + if emit_tail: + for line in (output or "").splitlines(): + log.info("[%s rank%d server.log] %s", h, rank, line) + if self.EARLY_FAILURE_RE.search(output or ""): + raise RuntimeError( + f"{label} server early failure on {h} (rank {rank}): {(output or '')[-500:]}" + ) + out = self.orch.exec( + f"grep -m1 -iE {shlex.quote(self.FATAL_LOG_RE.pattern)} {shlex.quote(rank_log)}", + detailed=True, + hosts=[host], + ) + for h, r in (out or {}).items(): + if r.get("exit_code") == 0 and r.get("output", "").strip(): + raise RuntimeError( + f"{label} server fatal error on {h} (rank {rank}): {r['output'].strip()[-500:]}" + ) + + def _tail_server_logs(self, lines=30): + if self.distributed: + out = {} + for rank in range(self.nnodes): + chunk = self._exec_all(f"tail -{lines} {shlex.quote(self._rank_server_log(rank))}") + out.update(chunk or {}) + return out + return self._exec_all(f"tail -{lines} {shlex.quote(self.server_log)}") + + def wait_ready(self): + log.info("waiting %ds for server log to materialise", self._precheck_wait) + time.sleep(self._precheck_wait) + + if self.driver == "atom": + out = self._tail_server_logs(30) + for host, output in out.items(): + if self.EARLY_FAILURE_RE.search(output or ""): + raise RuntimeError(f"atom server early failure on {host}: {output[-500:]}") + else: + self._check_coordinator_early_failure(emit_tail=True) + + log.info("warmup wait %ds", self._warmup_wait) + time.sleep(self._warmup_wait) + + if self.driver == "atom": + out = self._tail_server_logs(30) + for host, output in out.items(): + if self.EARLY_FAILURE_RE.search(output or ""): + raise RuntimeError(f"atom server early failure on {host}: {output[-500:]}") + else: + self._check_coordinator_early_failure(emit_tail=True) + + for it in range(self._server_poll_count): + log.info("readiness poll iter=%d/%d", it, self._server_poll_count - 1) + if self.is_ready(): + log.info("server health ready (iter=%d)", it) + break + if self.driver == "atom": + poll_out = self._tail_server_logs(30) + for host, output in poll_out.items(): + if self.EARLY_FAILURE_RE.search(output or ""): + raise RuntimeError(f"atom server early failure on {host}: {output[-500:]}") + else: + self._check_coordinator_early_failure() + time.sleep(self._server_poll_wait) + else: + raise RuntimeError("server did not become ready before timeout") + + if self.driver == "atom": + for it in range(10): + if self._atom_warmup_ok(): + log.info("server warmup complete (iter=%d)", it) + return + time.sleep(30) + raise RuntimeError("atom server warmup did not complete before timeout") + + def stop_server(self): + if self.driver == "atom": + log.info("stopping atom server") + self._exec_all( + "bash -c " + + shlex.quote("pkill -f 'atom.entrypoints.openai_server' || pkill -f 'openai_server' || true") + ) + else: + log.info("stopping vllm server") + self._exec_all("bash -c 'pkill -f \"vllm serve\" || true'") + time.sleep(5) + + def _atom_client_argv(self): + warmups = int(self.concurrency) * 2 + argv = [ + "python", + "-m", + "atom.benchmarks.benchmark_serving", + "--model", + self.model_id, + "--backend", + "vllm", + "--base-url", + f"http://localhost:{self.port_no}", + "--dataset-name", + self.dataset_name, + "--random-input-len", + self.isl, + "--random-output-len", + self.osl, + "--random-range-ratio", + self.random_range_ratio, + "--max-concurrency", + self.concurrency, + "--num-prompts", + self.num_prompts, + "--trust-remote-code", + "--num-warmups", + str(warmups), + "--request-rate", + self.request_rate, + "--ignore-eos", + "--save-result", + "--percentile-metrics", + self.percentile_metrics, + "--result-dir", + self.out_dir, + "--result-filename", + f"{self.result_stem}.json", + ] + if self.bench_extra_args: + argv.extend(shlex.split(self.bench_extra_args)) + return argv + + def _sglang_client_argv(self): + return [ + "python3", + "-m", + "sglang.bench_serving", + "--backend", + "sglang", + "--host", + "0.0.0.0", + "--port", + str(self.port_no), + "--dataset-name", + self.dataset_name, + "--num-prompts", + self.num_prompts, + "--random-input", + self.isl, + "--random-output", + self.osl, + "--random-range-ratio", + self.random_range_ratio, + "--max-concurrency", + self.concurrency, + "--request-rate", + self.request_rate, + ] + + def _client_argv(self): + if self.driver == "atom": + return self._atom_client_argv() + if self._uses_sglang_serve(): + return self._sglang_client_argv() + return self._vllm_client_argv() + + def _vllm_client_argv(self): + return [ + "vllm", + "bench", + "serve", + "--model", + self.model_id, + "--backend", + self.backend, + "--base-url", + f"{self.base_url}:{self.port_no}", + "--dataset-name", + self.dataset_name, + "--num-prompts", + self.num_prompts, + "--random-input-len", + self.isl, + "--random-output-len", + self.osl, + "--max-concurrency", + self.concurrency, + "--request-rate", + self.request_rate, + "--burstiness", + self.burstiness, + "--tokenizer-mode", + self.tokenizer_mode, + "--seed", + self.seed, + "--random-range-ratio", + self.random_range_ratio, + "--random-prefix-len", + self.random_prefix_len, + "--percentile-metrics", + self.percentile_metrics, + "--metric-percentiles", + self.metric_percentiles, + "--ignore-eos", + "--save-result", + "--result-dir", + self.out_dir, + "--result-filename", + self.result_stem, + ] + + def _clear_stale_result_artifact(self): + """Remove a prior run's result file so poll logic cannot treat it as complete.""" + artifact = shlex.quote(self._result_artifact) + self._exec_head(f"rm -f {artifact}") + + def run_client(self): + self._clear_stale_result_artifact() + args = self._client_argv() + bench_cmd = " ".join(shlex.quote(str(a)) for a in args) + client_cmd = f"source /tmp/server_env_script.sh && {bench_cmd} > {shlex.quote(self.client_log)} 2>&1 &" + self._exec_head("bash -c " + shlex.quote(client_cmd)) + + def _atom_result_ready(self): + out = self._exec_head(f"test -s {shlex.quote(self._result_artifact)} && echo OK || echo NO") + return bool(out) and all("OK" in (v or "") for v in out.values()) + + def _client_log_failures(self, tail_lines=2000): + out = self._exec_head(f"tail -{tail_lines} {shlex.quote(self.client_log)}") + failed = [] + for host, output in out.items(): + txt = output or "" + if self.CLIENT_CRASH_RE.search(txt) or self.CLIENT_LAUNCH_FAIL_RE.search(txt): + failed.append((host, txt[-500:])) + continue + fm = self.FAILED_REQUESTS_RE.search(txt) + if fm: + fc = int(fm.group(1)) + cap = self._bench_max_failed_requests + if fc > cap: + failed.append((host, f"Failed requests: {fc} (cap {cap}) -- {txt[-500:]}")) + elif fc > 0: + log.warning( + "client on %s completed with %d failed requests (allowed up to %d)", + host, + fc, + cap, + ) + return failed + + def wait_client_complete(self): + if self.driver == "atom": + log.info( + "client initial wait (atom: polling for result artifact, up to %ds)", + self._client_initial_wait, + ) + deadline = time.monotonic() + self._client_initial_wait + poll_s = 15 + while time.monotonic() < deadline: + failed = self._client_log_failures(tail_lines=500) + if failed: + raise RuntimeError("client failed: " + "; ".join(f"{h}: {m}" for h, m in failed)) + if self._atom_result_ready(): + log.info("client result artifact ready during initial wait") + return + remaining = deadline - time.monotonic() + if remaining <= 0: + break + time.sleep(min(poll_s, remaining)) + else: + log.info("client initial wait %ds", self._client_initial_wait) + time.sleep(self._client_initial_wait) + + for it in range(self._client_poll_count): + failed = self._client_log_failures() + if failed: + raise RuntimeError("client failed: " + "; ".join(f"{h}: {m}" for h, m in failed)) + if self.driver == "atom": + if self._atom_result_ready(): + log.info("client complete (iter=%d)", it) + return + else: + out = self._exec_head(f"tail -2000 {shlex.quote(self.client_log)}") + done = [bool(self.COMPLETION_RE.search(txt or "")) for txt in out.values()] + if done and all(done): + log.info("client complete (iter=%d)", it) + return + time.sleep(self._client_poll_wait) + raise RuntimeError("client did not complete before poll cap") + + def parse_results(self): + out = self._exec_head(f"cat {shlex.quote(self._result_artifact)}") + results = {} + for host, text in out.items(): + text = (text or "").strip() + if not text: + raise RuntimeError(f"empty/missing results artifact on {host}: {self._result_artifact}") + try: + raw = json.loads(text) + except (json.JSONDecodeError, ValueError) as e: + raise RuntimeError(f"unparseable results artifact on {host}: {self._result_artifact}: {e}") from e + if self.driver == "atom": + raw.setdefault("random_input_len", int(self.isl)) + raw.setdefault("random_output_len", int(self.osl)) + results[host] = to_client_metrics( + raw, + tp=self.tp, + isl=self.isl, + scaling_baseline_output_throughput=self._scaling_baseline, + nnodes=self.nnodes, + ) + return results diff --git a/cvs/lib/inference/inferencex_atom/inferencex_atom_parsing.py b/cvs/lib/inference/inferencex_atom/inferencex_atom_parsing.py new file mode 100644 index 000000000..f797b5cf4 --- /dev/null +++ b/cvs/lib/inference/inferencex_atom/inferencex_atom_parsing.py @@ -0,0 +1,117 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +InferenceX ATOM metric vocabulary and parsers. + +ATOM ``benchmark_serving`` emits the same JSON scalar keys as stock vLLM bench, +so base parsing reuses :func:`vllm_parsing.to_client_metrics`. W1 IX gates +(``per_gpu_throughput``, ``output_tput_per_gpu``, tail percentiles) live here — +not in the vLLM single-node ``GATED_METRICS`` set (vLLM parity is a separate track). +''' + +from __future__ import annotations + +from cvs.lib.inference.utils.vllm_parsing import ( + CLIENT_METRICS as _VLLM_CLIENT_METRICS, + GATED_METRICS as _VLLM_GATED_METRICS, + _safe_div, + to_client_metrics as _vllm_to_client_metrics, +) + +_METRIC_INSERT = ("output_tput_per_gpu", "tok/s") +_idx = next( + (i for i, (n, _) in enumerate(_VLLM_CLIENT_METRICS) if n == "per_gpu_throughput"), + None, +) +CLIENT_METRICS = list(_VLLM_CLIENT_METRICS) +CLIENT_METRICS.insert( + (_idx + 1) if _idx is not None else len(CLIENT_METRICS), + _METRIC_INSERT, +) +CLIENT_METRIC_UNITS = dict(CLIENT_METRICS) + +# IX W1 perf gates: vLLM baseline set plus per-GPU throughput derivations. +GATED_METRICS = frozenset(_VLLM_GATED_METRICS) | { + "per_gpu_throughput", + "output_tput_per_gpu", +} + +# W1 metric tiers for ``test_cell_metrics`` (one parent row per cell × tier). +METRIC_TIERS: dict[str, tuple[str, ...]] = { + "throughput": ( + "total_token_throughput", + "output_throughput", + "per_gpu_throughput", + "output_tput_per_gpu", + ), + "ttft": ( + "mean_ttft_ms", + "p99_ttft_ms", + ), + "tpot": ( + "mean_tpot_ms", + "p99_tpot_ms", + ), + "health": ( + "success_rate", + "failed", + ), + "scaling": ("efficiency_pct",), +} + +SCALING_METRICS: tuple[str, ...] = METRIC_TIERS["scaling"] +SCALING_METRIC_UNITS: dict[str, str] = {"efficiency_pct": "%"} + +METRIC_TIER_ORDER: tuple[str, ...] = tuple(METRIC_TIERS.keys()) + ("record",) + +_tiered = {m for names in METRIC_TIERS.values() for m in names} +RECORD_METRICS: tuple[str, ...] = tuple(short for short, _unit in CLIENT_METRICS if short not in _tiered) + +ENFORCED_METRICS = frozenset(_tiered) + + +def scaling_efficiency_pct(actual_output_throughput, *, baseline_single_node, nnodes): + """Linear scaling efficiency: actual / (single-node baseline × nnodes).""" + denom = _safe_div(baseline_single_node, 1) + if denom is None or int(nnodes) < 1: + return None + ideal = denom * int(nnodes) + if ideal <= 0: + return None + return _safe_div(actual_output_throughput, ideal) + + +def to_client_metrics(raw, *, tp, isl, scaling_baseline_output_throughput=None, nnodes=1): + """Map an ATOM ``results.json`` dict to the ``client.*`` namespace for IX.""" + m = _vllm_to_client_metrics(raw, tp=tp, isl=isl) + m["client.output_tput_per_gpu"] = _safe_div(raw.get("output_throughput"), tp) + if scaling_baseline_output_throughput is not None: + eff = scaling_efficiency_pct( + raw.get("output_throughput"), + baseline_single_node=scaling_baseline_output_throughput, + nnodes=nnodes, + ) + if eff is not None: + m["scaling.efficiency_pct"] = eff * 100.0 + return m + + +def tier_metric_specs(thresholds_cell: dict, tier: str) -> dict[str, dict]: + """Return threshold specs for one tier in a sweep cell.""" + if tier == "record": + names = RECORD_METRICS + prefix = "client." + elif tier == "scaling": + names = SCALING_METRICS + prefix = "scaling." + else: + names = METRIC_TIERS.get(tier, ()) + prefix = "client." + specs = {} + for short in names: + full = f"{prefix}{short}" + spec = thresholds_cell.get(full) + if spec is not None: + specs[full] = spec + return specs diff --git a/cvs/lib/sglang_disagg_lib.py b/cvs/lib/inference/sglang_disagg_lib.py similarity index 82% rename from cvs/lib/sglang_disagg_lib.py rename to cvs/lib/inference/sglang_disagg_lib.py index 5748a787d..6f7db4ac6 100644 --- a/cvs/lib/sglang_disagg_lib.py +++ b/cvs/lib/inference/sglang_disagg_lib.py @@ -5,11 +5,16 @@ All code contained here is Property of Advanced Micro Devices, Inc. ''' +import base64 +import json import os import re +import shlex import time +from typing import Any, Optional from cvs.lib import globals +from cvs.lib.utils.model_query_lib import LmEvalBenchmark, OpenAIProbe from cvs.lib.utils_lib import * from cvs.lib.verify_lib import * @@ -33,6 +38,46 @@ def textwrap_for_yml(msg_string): return '\n'.join([m.lstrip() for m in msg_string.split('\n')]) +def _as_node_list(value): + """ + Normalize cluster JSON node field to a list of host strings. + + Config may use a single hostname/IP string or a list. ``list(str)`` + would split into characters (e.g. ``'10.0.0.1'`` -> ``'1'``), breaking + SSH and HTTP clients. + """ + if isinstance(value, str): + return [value] + return list(value) + + +def _first_float(pattern, text): + m = re.search(pattern, text, re.I) + return m.group(1) if m else None + + +LM_EVAL_SPECS = { + "lm_eval_hellaswag": { + "display": "HellaSwag", + "default_metric": "acc_norm", + "default_metric_key": "acc_norm,none", + "default_num_concurrent": "1", + }, + "lm_eval_gsm8k": { + "display": "GSM8K", + "default_metric": "exact_match", + "default_metric_key": "exact_match,flexible-extract", + "default_num_concurrent": "4", + }, + "lm_eval_mmlu": { + "display": "MMLU", + "default_metric": "acc", + "default_metric_key": "acc,none", + "default_num_concurrent": "1", + }, +} + + class SglangDisaggPD: def __init__( self, @@ -97,13 +142,18 @@ def __init__( # Proxy node : Routes requests between prefill/decode # Benchmark node : Generates inference load # ------------------------------------------------------------------ + self.mount_vol = self.inf_dict.get( + 'mount_vol', + '/usr/lib/x86_64-linux-gnu/libibverbs/libbnxt_re-rdmav34.so', + ) + self.prefill_node_list = self.inf_dict['prefill_node_list'] self.decode_node_list = self.inf_dict['decode_node_list'] self.prefill_nnodes = len(self.prefill_node_list) self.decode_nnodes = len(self.decode_node_list) - self.proxy_node = list(self.inf_dict['proxy_router_node']) - self.benchmark_serv_node = list(self.inf_dict['benchmark_serv_node']) + self.proxy_node = _as_node_list(self.inf_dict['proxy_router_node']) + self.benchmark_serv_node = _as_node_list(self.inf_dict['benchmark_serv_node']) # ------------------------------------------------------------------ # SSH handlers for each node group @@ -156,6 +206,7 @@ def __init__( self.inf_dict.setdefault('nic_type', 'ainic') self.inf_dict.setdefault('nccl_ib_hca_list', 'rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7') self.inf_dict.setdefault('nccl_ib_hca', 'rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7') + self.inf_dict.setdefault('hca_id_prefix', 'bnxt_') self.inf_dict.setdefault('nccl_socket_ifname', 'eno0') self.inf_dict.setdefault('gloo_socket_ifname', 'eno0') self.inf_dict.setdefault('nccl_ib_gid_index', '1') @@ -182,6 +233,7 @@ def __init__( self.nccl_debug = self.inf_dict['nccl_debug'] self.data_cache_dir = self.inf_dict['data_cache_dir'] self.log_dir = self.inf_dict['log_dir'] + self.hca_id_prefix = str(self.inf_dict['hca_id_prefix']).strip() # set defaults for benchmark param dict if not passed via JSON file self.bp_dict.setdefault('backend', 'sglang') @@ -209,6 +261,7 @@ def __init__( log.info(f'benchmark_params_dict = {self.bp_dict}') log.info('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%') + # Helper function for installing container packages def install_container_packages( self, ): @@ -242,6 +295,7 @@ def install_container_packages( self.d_phdl.exec(cmd) self.r_phdl.exec(cmd) + # Helper function for executing NIC setup scripts def exec_nic_setup_scripts( self, ): @@ -265,21 +319,24 @@ def exec_nic_setup_scripts( if re.search('broadcom|thor', self.nic_type, re.I): # override the gid_index to 3 for broadcom self.nccl_ib_gid_index = 3 - cmd = f'docker exec {self.container_name} /bin/bash -c "sudo \ - cp /usr/lib/x86_64-linux-gnu/libibverbs/libbnxt_re-rdmav34.so.host \ - /usr/lib/x86_64-linux-gnu/libibverbs/libbnxt_re-rdmav34.so; \ - sleep 2; ibv_devinfo; sleep 2;" ' + cmd = ( + f'docker exec {self.container_name} /bin/bash -c "sudo ' + f'cp {self.mount_vol}.host {self.mount_vol}; ' + f'sleep 2; ibv_devinfo; sleep 2;" ' + ) pout_dict = self.p_phdl.exec(cmd) dout_dict = self.d_phdl.exec(cmd) + hca_id_regex = rf'hca_id:\s+{re.escape(self.hca_id_prefix)}' for node in pout_dict.keys(): - if not re.search('hca_id:\s+(bnxt_|rocep)', pout_dict[node], re.I): + if not re.search(hca_id_regex, pout_dict[node], re.I): log.info("%s", pout_dict[node]) fail_test(f'Broadcom libbnxt rdma driver is not properly copied on node {node}') for node in dout_dict.keys(): - if not re.search('hca_id:\s+(bnxt_|rocep)', dout_dict[node], re.I): + if not re.search(hca_id_regex, dout_dict[node], re.I): log.info("%s", dout_dict[node]) fail_test(f'Broadcom libbnxt rdma driver is not properly copied on node {node}') + # Helper function for checking IBV devices def check_ibv_devices( self, ): @@ -309,6 +366,7 @@ def check_ibv_devices( if re.search('No IB devices found', out_dict[node], re.I): fail_test(f'IB devices not seen inside the container for node {node}') + # Helper function for setting up prefill container environment def setup_prefill_container_env( self, ): @@ -340,6 +398,7 @@ def setup_prefill_container_env( chmod 755 /tmp/prefill_env_script.sh; /tmp/prefill_env_script.sh" ''' self.p_phdl.exec(cmd) + # Helper function for setting up decode container environment def setup_decode_container_env( self, ): @@ -371,6 +430,7 @@ def setup_decode_container_env( chmod 755 /tmp/decode_env_script.sh; /tmp/decode_env_script.sh" ''' self.d_phdl.exec(cmd) + # Helper function for setting up proxy router container environment def setup_proxy_router_container_env( self, ): @@ -397,6 +457,7 @@ def setup_proxy_router_container_env( chmod 755 /tmp/router_env_script.sh; /tmp/router_env_script.sh" ''' self.r_phdl.exec(cmd) + # Helper function for setting up benchmark server container environment def setup_benchmark_serv_container_env( self, ): @@ -423,6 +484,7 @@ def setup_benchmark_serv_container_env( self.b_phdl.exec(cmd) time.sleep(5) + # Helper function for running RMSNorm test def run_test_rmsnorm(self, max_jobs=192): """ Run RMSNorm 2D operator tests inside the SGLang container across @@ -545,6 +607,7 @@ def launch_prefill_servers(self, dtype='auto', kv_cache_dtype='auto'): self.p_phdl.exec_cmd_list(cmd_list) time.sleep(5) + # Helper function for launching Decode servers def launch_decode_servers(self, dtype='auto', kv_cache_dtype='auto'): """ Generate and deploy Decode server launch scripts on all Decode nodes @@ -616,6 +679,7 @@ def launch_decode_servers(self, dtype='auto', kv_cache_dtype='auto'): cmd_list.append(formatted_cmd) self.d_phdl.exec_cmd_list(cmd_list) + # Helper function for polling for server readiness def poll_and_check_server_ready( self, ): @@ -645,6 +709,7 @@ def poll_and_check_server_ready( self.poll_for_server_ready(0, 'prefill') self.poll_for_server_ready(0, 'decode') + # Helper function for launching Proxy Router def launch_proxy_router( self, ): @@ -730,66 +795,7 @@ def launch_proxy_router( log.info('Waiting 120 secs after launching proxy router script') time.sleep(120) - def run_gsm8k_benchmark_test(self, d_type='auto'): - """ - Run the GSM8K inference benchmark against the SGLang disaggregated - Prefill/Decode deployment and validate throughput. - - Purpose: - -------- - This method executes a real-world inference workload (GSM8K question - answering) to: - - Validate end-to-end correctness of the inference pipeline - - Measure sustained output token throughput - - Ensure performance meets expected SLA thresholds - - The benchmark traffic is sent to the Proxy Router, which: - - Routes requests to Prefill servers - - Coordinates Decode servers for token generation - """ - log.info('#================ * * * =========================#') - log.info('Create Benchmark script') - log.info('#================ * * * =========================#') - - i_dict = self.bp_dict['inference_tests']['gsm8k'] - # ------------------------------------------------------------------ - # Construct command to run GSM8K benchmark inside the container - # - # Key steps: - # - Create a directory to store benchmark logs - # - Navigate to the GSM8K benchmark directory - # - Source environment variables required for benchmark execution - # - Launch the benchmark using nohup to allow async execution - # - # Benchmark parameters: - # --num-questions : Total GSM8K questions to run - # --parallel : Maximum concurrent inference requests - # --host / --port : Proxy Router endpoint for inference - # ------------------------------------------------------------------ - cmd = f'''docker exec {self.container_name} /bin/bash -c " - mkdir -p {self.log_dir}/benchmark_node; \ - cd /sgl-workspace/sglang/benchmark/gsm8k; \ - source /tmp/benchmark_env_script.sh && \ - nohup python3 ./bench_sglang.py \ - --num-questions {i_dict['num_questions']} \ - --parallel {i_dict['max_concurrency']} \ - --host http://0.0.0.0 --port {self.inf_dict['proxy_router_serv_port']}" ''' - formatted_cmd = textwrap_for_yml(cmd) - out_dict = self.b_phdl.exec(formatted_cmd, timeout=800) - time.sleep(5) - for node in out_dict.keys(): - if not re.search('Output throughput', out_dict[node], re.I): - fail_test(f'Benchmark test did not complete properly on node {node}, no throughput pattern seen') - else: - match = re.search('Output throughput:\s+([0-9\.]+)\s+token', out_dict[node], re.I) - actual_tps = match.group(1) - if float(actual_tps) < float(i_dict['expected_results'][d_type]['tokens_per_sec']): - fail_test( - f"Test FAILED due to low performance, \ - expected tokens per sec = {i_dict['expected_results'][d_type]['tokens_per_sec']}, \ - actual tokens per sec = {actual_tps}" - ) - + # Helper function for running SGLang serving benchmark with random dataset def benchserv_test_random(self, d_type='auto'): """ Run SGLang serving benchmark using a synthetic random dataset and @@ -814,6 +820,7 @@ def benchserv_test_random(self, d_type='auto'): log.info('Benchmark Random Dataset') log.info('#================ * * * =========================#') i_dict = self.bp_dict['inference_tests']['bench_serv_random'] + self._bench_num_prompts = int(i_dict['num_prompts']) # ------------------------------------------------------------------ # Construct command to run sglang.bench_serving with random dataset # @@ -830,6 +837,7 @@ def benchserv_test_random(self, d_type='auto'): cmd = f'''docker exec {self.container_name} /bin/bash -c " mkdir -p {self.log_dir}/benchmark_node; \ source /tmp/benchmark_env_script.sh && \ + pip install --upgrade --no-deps sglang[all] && \ python3 -m sglang.bench_serving --backend {i_dict['backend']} \ --dataset-name random \ --num-prompts {i_dict['num_prompts']} \ @@ -842,8 +850,42 @@ def benchserv_test_random(self, d_type='auto'): self.b_phdl.exec(formatted_cmd, timeout=500) time.sleep(5) self.poll_for_inference_completion(iterations=10, waittime_between_iters=60) + + # MFU (derived from same bench metrics as TTFT/TPOT) + peak_tflops = float(i_dict.get("peak_gpu_tflops", 1300)) + num_params = float(i_dict.get("model_num_params", 70e9)) + tp = int(self.bp_dict.get("tensor_parallelism", 1)) + num_gpus = (int(self.prefill_nnodes) + int(self.decode_nnodes)) * tp + for node, m in (self.inference_results_dict or {}).items(): + duration = float(m.get("benchmark_duration") or 0) + in_tok = float(m.get("total_input_tokens") or 0) + out_tok = float(m.get("total_generated_tokens") or m.get("Total generated tokens:") or 0) + if duration > 0 and num_gpus > 0: + achieved = 6.0 * num_params * (in_tok + out_tok) + peak = peak_tflops * 1e12 * num_gpus * duration + m["mfu"] = f"{achieved / peak:.6f}" + + log_path = f"{self.log_dir}/benchmark_node/benchmark_results.log" + for node, m in (self.inference_results_dict or {}).items(): + gp = m.get("goodput", "n/a") + tpg = m.get("output_throughput_per_gpu_per_sec", "n/a") + tr = m.get("total_requests", "n/a") + sr = m.get("successful_requests", "n/a") + mfu = m.get("mfu", "n/a") + inner = ( + f"echo '' >> {log_path} && " + f"echo '============ Derived Benchmark Results ============' >> {log_path} && " + f"echo 'Goodput (successful / total): {sr} / {tr} => {gp}' >> {log_path} && " + f"echo 'Output token throughput per GPU (tok/s/GPU): {tpg}' >> {log_path} && " + f"echo 'MFU (estimated): {mfu}' >> {log_path} && " + f"echo '=====================================================================' >> {log_path}" + ) + cmd = f"docker exec {self.container_name} /bin/bash -c {shlex.quote(inner)}" + self.b_phdl.exec(cmd) + self.verify_inference_results('bench_serv', i_dict['expected_results'][d_type]) + # Helper function for polling for server readiness def poll_for_server_ready(self, node_no, sglang_function, no_of_iterations=16): """ Poll SGLang Prefill or Decode server logs to determine when the server @@ -914,6 +956,7 @@ def poll_for_server_ready(self, node_no, sglang_function, no_of_iterations=16): log.warning(f'Decode node {node_no} did not get to ready to serve 200 OK state in {j} iterations') fail_test(f'Decode node {node_no} did not get to ready to serve 200 OK state in {j} iterations') + # Helper function for getting inference results dictionary def get_inference_results_dict(self, out_dict): """ Parse inference benchmark output logs and extract key performance metrics @@ -939,6 +982,7 @@ def get_inference_results_dict(self, out_dict): self.inference_results_dict = {} log.info('Inside get_inference_results_dict') log.info("%s", out_dict) + for node in out_dict.keys(): self.inference_results_dict[node] = {} if re.search('Successful requests:', out_dict[node], re.I): @@ -952,7 +996,7 @@ def get_inference_results_dict(self, out_dict): self.inference_results_dict[node]['total_input_tokens'] = match.group(1) if re.search('Total generated tokens:', out_dict[node], re.I): match = re.search('Total generated tokens:\s+([0-9\.]+)', out_dict[node], re.I) - self.inference_results_dict[node]['Total generated tokens:'] = match.group(1) + self.inference_results_dict[node]['total_generated_tokens'] = match.group(1) if re.search('Request throughput \(req/s\):', out_dict[node], re.I): match = re.search('Request throughput \(req/s\):\s+([0-9\.]+)', out_dict[node], re.I) self.inference_results_dict[node]['request_throughput_per_sec'] = match.group(1) @@ -986,19 +1030,45 @@ def get_inference_results_dict(self, out_dict): if re.search('P99 ITL \(ms\):', out_dict[node], re.I): match = re.search('P99 ITL \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) self.inference_results_dict[node]['p99_itl_ms'] = match.group(1) - if re.search('Mean E2EL \(ms\):', out_dict[node], re.I): - match = re.search('Mean E2EL \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) - self.inference_results_dict[node]['mean_e2el_ms'] = match.group(1) - if re.search('Median E2EL \(ms\):', out_dict[node], re.I): - match = re.search('Median E2EL \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) - self.inference_results_dict[node]['median_e2el_ms'] = match.group(1) - if re.search('P99 E2EL \(ms\):', out_dict[node], re.I): - match = re.search('P99 E2EL \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) - self.inference_results_dict[node]['p99_e2el_ms'] = match.group(1) + # --- SGLang "E2E Latency" wording ----- + m = _first_float(r'Mean E2E Latency \(ms\):\s+([0-9\.]+)', out_dict[node]) + if m: + self.inference_results_dict[node]['mean_e2e_latency_ms'] = m + m = _first_float(r'Median E2E Latency \(ms\):\s+([0-9\.]+)', out_dict[node]) + if m: + self.inference_results_dict[node]['median_e2e_latency_ms'] = m + for p in (90, 95, 99): + m = _first_float(rf'P{p} E2E Latency \(ms\):\s+([0-9\.]+)', out_dict[node]) + if m: + self.inference_results_dict[node][f'p{p}_e2e_latency_ms'] = m + + # Goodput: log totals, or successful+failed, or benchmark num_prompts + total_req = _first_float(r"Total requests:\s+([0-9]+)", out_dict[node]) + failed_req = _first_float(r"Failed requests:\s+([0-9]+)", out_dict[node]) + succ = self.inference_results_dict[node].get("successful_requests") + if total_req: + self.inference_results_dict[node]["total_requests"] = total_req + elif succ is not None and failed_req is not None: + self.inference_results_dict[node]["total_requests"] = str(int(succ) + int(failed_req)) + elif succ is not None and getattr(self, "_bench_num_prompts", None) is not None: + self.inference_results_dict[node]["total_requests"] = str(int(self._bench_num_prompts)) + if succ and self.inference_results_dict[node].get("total_requests"): + s, t = int(succ), int(self.inference_results_dict[node]["total_requests"]) + self.inference_results_dict[node]["goodput"] = f"{(s / t):.6f}" if t else None + + # Per-GPU throughput (derived): define denominator to match *your* accounting policy + out_tps = self.inference_results_dict[node].get("output_throughput_per_sec") + if out_tps: + ng = int(self.bp_dict.get("tensor_parallelism", "1")) + if ng > 0: + self.inference_results_dict[node]["output_throughput_per_gpu_per_sec"] = ( + f"{float(out_tps) / ng:.6f}" + ) log.info("%s", self.inference_results_dict) return self.inference_results_dict + # Helper function for scanning for inference errors def scan_for_inference_errors( self, ): @@ -1056,6 +1126,7 @@ def scan_for_inference_errors( return inference_pass + # Helper function for polling for inference completion def poll_for_inference_completion( self, iterations=10, waittime_between_iters=60, total_timeout=3600, require_all_nodes=True ): @@ -1178,6 +1249,7 @@ def timed_out() -> bool: log.warning("%s", msg) return {"status": "stuck_in_progress", "reason": msg} + # Helper function for verifying inference results def verify_inference_results(self, test_name, expected_result_dict): """ Validate inference benchmark results against expected performance @@ -1254,6 +1326,7 @@ def verify_inference_results(self, test_name, expected_result_dict): verify_dmesg_for_errors(self.b_phdl, self.inference_start_time, self.inference_end_time) log.info("%s", self.inference_results_dict) + # Helper function for counting occupied GPUs per prefill/decode node def sglang_disagg_gpu_counts(self, mem_threshold_mb=5000): """ After model load, count occupied GPUs per prefill/decode node via amd-smi. @@ -1316,3 +1389,138 @@ def _count_per_node(phdl): log.info("\n".join(lines)) return result + + # Helper function for verifying OpenAI-compatible endpoints + def verify_openai_compatible_endpoints(self) -> list[str]: + """ + Smoke-test OpenAI-compatible HTTP API on the proxy router (inside the + benchmark container via ``docker exec``): GET /v1/models, + POST /v1/chat/completions, POST /v1/completions, and structured JSON + (book) via chat completions. + """ + port = int(self.inf_dict["proxy_router_serv_port"]) + model_name = self.bp_dict["model"] + + probe_src = OpenAIProbe.probe_script(port, model_name) + b64 = base64.b64encode(probe_src.encode("utf-8")).decode("ascii") + cmd = f'''docker exec {self.container_name} /bin/bash -c " + mkdir -p {self.log_dir}/benchmark_node; \ + echo '{b64}' | base64 -d > /tmp/openai_mq_probe.py && \ + python3 /tmp/openai_mq_probe.py && \ + rm -f /tmp/openai_mq_probe.py" ''' + formatted_cmd = textwrap_for_yml(cmd) + log.info( + "OpenAI endpoint probe inside benchmark container (0.0.0.0:%r), same pattern as GSM8K/benchserv", + port, + ) + out_dict = self.b_phdl.exec( + formatted_cmd, + timeout=min(900, 480 + 180), + ) + bench_host = self.benchmark_serv_node[0] + raw_out = out_dict.get(bench_host) + if raw_out is None and out_dict: + raw_out = next(iter(out_dict.values())) + + probe_err: Optional[str] = None + results: dict[str, tuple[int, Any]] = {} + if not raw_out or not str(raw_out).strip(): + probe_err = f"OpenAI-compatible probe produced no output node {bench_host!r}: {out_dict!r}" + else: + lines_out = str(raw_out).strip().splitlines() + if not lines_out: + probe_err = f"OpenAI-compatible probe empty lines after strip on node {bench_host!r}: {raw_out!r}" + else: + last_line = lines_out[-1] + try: + parsed = json.loads(last_line) + except json.JSONDecodeError as e: + probe_err = f"OpenAI-compatible probe invalid JSON: {e!r} raw={raw_out!r}" + else: + if not isinstance(parsed, dict): + probe_err = f"OpenAI-compatible probe expected JSON object, got {type(parsed).__name__!r}" + else: + for step, val in parsed.items(): + if isinstance(val, (list, tuple)) and len(val) == 2: + results[step] = (int(val[0]), val[1]) + else: + probe_err = f"OpenAI-compatible probe bad shape at {step!r}: {val!r}" + break + + if probe_err is not None: + fail_test(probe_err) + return [] + + OpenAIProbe.log_results(results, log) + + ok, err = OpenAIProbe.check_results(results, port=port, logger=log) + if not ok: + summary = OpenAIProbe.summarize_results(results, ok, err) + fail_test(f"{err}") + return summary + + summary = OpenAIProbe.summarize_results(results, ok, err) + return summary + + # Helper functions for running LM-Eval benchmarks + def run_lm_eval_hellaswag_benchmark_test(self, _d_type="auto"): + return self.run_lm_eval_benchmark_test("lm_eval_hellaswag", _d_type=_d_type) + + # Helper functions for running GSM8K benchmarks + def run_lm_eval_gsm8k_benchmark_test(self, _d_type="auto"): + return self.run_lm_eval_benchmark_test("lm_eval_gsm8k", _d_type=_d_type) + + # Helper functions for running MMLU benchmarks + def run_lm_eval_mmlu_benchmark_test(self, _d_type="auto"): + return self.run_lm_eval_benchmark_test("lm_eval_mmlu", _d_type=_d_type) + + # Helper function for running LM-Eval benchmarks + def run_lm_eval_benchmark_test(self, bench_key: str, _d_type="auto"): + spec = LM_EVAL_SPECS[bench_key] + log.info("#================ * * * =========================#") + log.info("lm-eval %s benchmark", spec["display"]) + log.info("#================ * * * =========================#") + task_name = bench_key.removeprefix("lm_eval_") + i_dict = self.bp_dict["inference_tests"][bench_key] + inner_cmd, scoring = LmEvalBenchmark.prepare( + i_dict, + port=int(self.inf_dict["proxy_router_serv_port"]), + model_id=self.bp_dict["model"], + task_name=task_name, + default_tasks=task_name, + default_metric=spec["default_metric"], + default_metric_key=spec["default_metric_key"], + log_dir=self.log_dir, + log_basename=f"{bench_key}.log", + default_num_concurrent=spec["default_num_concurrent"], + ) + + cmd = f'''docker exec {self.container_name} /bin/bash -c " + mkdir -p {self.log_dir}/benchmark_node; \\ + source /tmp/benchmark_env_script.sh && \\ + {inner_cmd}" ''' + out_dict = self.b_phdl.exec(textwrap_for_yml(cmd), timeout=scoring["exec_timeout_sec"]) + time.sleep(5) + + check_kwargs = LmEvalBenchmark.check_kwargs_from_scoring(scoring) + summary = None + errors: list[str] = [] + + for node, text in out_dict.items(): + ok, node_summary, err = LmEvalBenchmark.check_results(text, **check_kwargs) + if node_summary is not None: + summary = node_summary + if not ok: + errors.append(f"lm-eval {spec['display']} on node {node!r}: {err}") + + if summary is None: + summary = LmEvalBenchmark.fallback_summary( + scoring, + error=errors[-1] if errors else "no benchmark nodes produced output to score", + ) + errors.append(f"lm-eval {spec['display']}: no benchmark nodes produced output to score") + + for msg in errors: + fail_test(msg) + + return summary diff --git a/cvs/lib/inference/unittests/__init__.py b/cvs/lib/inference/unittests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cvs/lib/inference/unittests/fake_orch.py b/cvs/lib/inference/unittests/fake_orch.py new file mode 100644 index 000000000..22e60b401 --- /dev/null +++ b/cvs/lib/inference/unittests/fake_orch.py @@ -0,0 +1,26 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Minimal stand-in for :class:`ContainerOrchestrator` in inference Job unit tests. + +Reusable by any suite's ``test_*_orch_parse.py`` — import ``FakeOrch`` instead of +copying the class into each test module. +''' + + +class FakeOrch: + def __init__(self, exec_return=None, hosts=None, exec_on_head_return=None): + self.hosts = list(hosts or ["node0"]) + self.exec_return = exec_return if exec_return is not None else {} + self.exec_on_head_return = exec_on_head_return if exec_on_head_return is not None else self.exec_return + self.commands = [] + self.exec_on_head_commands = [] + + def exec(self, cmd, hosts=None, **kwargs): + self.commands.append((cmd, hosts)) + return self.exec_return + + def exec_on_head(self, cmd, **kwargs): + self.exec_on_head_commands.append(cmd) + return self.exec_on_head_return diff --git a/cvs/lib/inference/unittests/fixtures/vllm_results_sample.json b/cvs/lib/inference/unittests/fixtures/vllm_results_sample.json new file mode 100644 index 000000000..842304822 --- /dev/null +++ b/cvs/lib/inference/unittests/fixtures/vllm_results_sample.json @@ -0,0 +1 @@ +{"date": "20260616-195845", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "amd/Llama-3.1-70B-Instruct-FP8-KV", "tokenizer_id": "amd/Llama-3.1-70B-Instruct-FP8-KV", "num_prompts": 12800, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 256, "duration": 2689.1403915379196, "completed": 12800, "failed": 0, "total_input_tokens": 1638400, "total_output_tokens": 26214400, "request_throughput": 4.7598853671896535, "request_goodput": null, "output_throughput": 9748.24523200441, "total_token_throughput": 10357.510559004686, "max_output_tokens_per_s": 10765.0, "max_concurrent_requests": 512, "rtfx": 0.0, "mean_ttft_ms": 668.48989902217, "median_ttft_ms": 639.2352399416268, "std_ttft_ms": 450.33186696146356, "p99_ttft_ms": 2471.584795164014, "mean_tpot_ms": 25.931252678514355, "median_tpot_ms": 25.9539894319472, "std_tpot_ms": 0.21713041382746362, "p99_tpot_ms": 26.324707068246756, "mean_itl_ms": 25.9318070919762, "median_itl_ms": 25.50674183294177, "std_itl_ms": 8.843604314389477, "p99_itl_ms": 32.19962654635305, "mean_e2el_ms": 53749.76413194105, "median_e2el_ms": 53752.437153831124, "std_e2el_ms": 298.81966449031785, "p99_e2el_ms": 55124.09303063527} \ No newline at end of file diff --git a/cvs/lib/inference/unittests/fixtures/vllm_results_widened.json b/cvs/lib/inference/unittests/fixtures/vllm_results_widened.json new file mode 100644 index 000000000..3e32870f9 --- /dev/null +++ b/cvs/lib/inference/unittests/fixtures/vllm_results_widened.json @@ -0,0 +1,52 @@ +{ + "date": "20260617-163232", + "endpoint_type": "vllm", + "backend": "vllm", + "label": null, + "model_id": "amd/Llama-3.1-70B-Instruct-FP8-KV", + "tokenizer_id": "amd/Llama-3.1-70B-Instruct-FP8-KV", + "num_prompts": 3200, + "request_rate": "inf", + "burstiness": 1.0, + "max_concurrency": 64, + "duration": 564.1147056743503, + "completed": 1791, + "failed": 1409, + "total_input_tokens": 229974, + "total_output_tokens": 2312408, + "request_throughput": 3.174886210170704, + "request_goodput": 3.174886210170704, + "output_throughput": 4099.180497760143, + "total_token_throughput": 4506.852904961594, + "max_output_tokens_per_s": 4590.0, + "max_concurrent_requests": 76, + "rtfx": 0.0, + "mean_ttft_ms": 291.88843878398956, + "median_ttft_ms": 73.2587007805705, + "std_ttft_ms": 1137.8571870825917, + "p50_ttft_ms": 73.2587007805705, + "p90_ttft_ms": 85.64653992652893, + "p95_ttft_ms": 91.81479038670659, + "p99_ttft_ms": 6259.152442589402, + "mean_tpot_ms": 15.032167084785439, + "median_tpot_ms": 15.03020008988302, + "std_tpot_ms": 0.3247818510561047, + "p50_tpot_ms": 15.03020008988302, + "p90_tpot_ms": 15.209271169796184, + "p95_tpot_ms": 15.24944565870449, + "p99_tpot_ms": 15.943741001209947, + "mean_itl_ms": 15.019441688915942, + "median_itl_ms": 14.628257602453232, + "std_itl_ms": 6.1097319902977505, + "p50_itl_ms": 14.628257602453232, + "p90_itl_ms": 15.559395030140879, + "p95_itl_ms": 17.392832040786708, + "p99_itl_ms": 27.511265948414778, + "mean_e2el_ms": 19668.871285726116, + "median_e2el_ms": 19835.17629932612, + "std_e2el_ms": 7829.601121737766, + "p50_e2el_ms": 19835.17629932612, + "p90_e2el_ms": 30145.559770055115, + "p95_e2el_ms": 31765.095902141184, + "p99_e2el_ms": 34034.53387795014 +} diff --git a/cvs/lib/inference/unittests/test_inference_suite_lifecycle.py b/cvs/lib/inference/unittests/test_inference_suite_lifecycle.py new file mode 100644 index 000000000..c9c914a83 --- /dev/null +++ b/cvs/lib/inference/unittests/test_inference_suite_lifecycle.py @@ -0,0 +1,29 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for inference_suite_lifecycle helpers. +''' + +import unittest + +from cvs.lib.inference.utils.cache_probe import du_bytes +from cvs.lib.inference.unittests.fake_orch import FakeOrch + + +class TestDuBytes(unittest.TestCase): + def test_sums_bytes_across_hosts(self): + orch = FakeOrch(exec_return={"node0": "1000", "node1": "2000"}) + self.assertEqual(du_bytes(orch, "/models"), 3000) + + def test_missing_path_returns_zero(self): + orch = FakeOrch(exec_return={"node0": "__MISSING__"}) + self.assertEqual(du_bytes(orch, "/models"), 0) + + def test_du_error_returns_none(self): + orch = FakeOrch(exec_return={"node0": "__DU_ERROR__"}) + self.assertIsNone(du_bytes(orch, "/models")) + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/inference/unittests/test_inferencex_atom_config_loader.py b/cvs/lib/inference/unittests/test_inferencex_atom_config_loader.py new file mode 100644 index 000000000..3b601e30c --- /dev/null +++ b/cvs/lib/inference/unittests/test_inferencex_atom_config_loader.py @@ -0,0 +1,439 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for cvs.lib.inference.inferencex_atom.inferencex_atom_config_loader. +''' + +import unittest +from pathlib import Path + +from cvs.lib.inference.inferencex_atom.inferencex_atom_config_loader import ( + InferenceXAtomVariantConfig, + expand_sweep, + expand_sweep_parametrize, + load_variant, + orchestrator_container_from_variant, + placeholder_gated_threshold_cell, + reuse_server_flag, + server_session_key, +) +from cvs.lib.inference.utils.inferencing_config_loader import Run, SeqCombo, Sweep + + +def _cluster_dict(): + return {"username": "testuser"} + + +class TestInferenceXAtomConfigLoader(unittest.TestCase): + def test_load_mi300x_sample_config(self): + root = Path(__file__).resolve().parents[3] + config = root / ( + "input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_gpt-oss-120b_bf16.json" + ) + variant = load_variant(config, _cluster_dict()) + self.assertEqual(variant.framework, "inferencex_atom") + self.assertEqual(variant.params.driver, "vllm") + self.assertEqual(variant.expected_cells(), ["ISL=7168,OSL=1024,TP=8,CONC=64"]) + self.assertIn("enforce-eager", variant.roles.server.serve_args) + + def test_deprecated_framework_alias_normalizes(self): + sweep = Sweep( + sequence_combinations=[SeqCombo(name="w1", isl="1024", osl="1024")], + runs=[Run(combo="w1", concurrency=128)], + ) + variant = InferenceXAtomVariantConfig( + schema_version=1, + framework="inferencex_atom_single", + gpu_arch="mi300x", + enforce_thresholds=False, + paths={ + "shared_fs": "/home/x", + "models_dir": "/home/x/models", + "log_dir": "/home/x/LOGS", + "hf_token_file": "/home/x/.hf", + }, + model={"id": "deepseek-ai/DeepSeek-R1-0528", "remote": 0, "precision": "fp8"}, + container={ + "name": "c", + "image": "img", + "runtime": {"name": "docker", "args": {"volumes": ["/home/x:/home/x"]}}, + }, + params={ + "driver": "atom", + "tensor_parallelism": "8", + "num_prompts": "100", + }, + sweep=sweep, + thresholds={}, + ) + self.assertEqual(variant.framework, "inferencex_atom") + + def test_load_w1_mi300x_atom_variant(self): + root = Path(__file__).resolve().parents[3] + config = root / ( + "input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_single.json" + ) + variant = load_variant(config, _cluster_dict()) + self.assertEqual(variant.threshold_json, "mi300x_inferencex-atom_deepseek-r1_fp8_single_threshold.json") + self.assertEqual(variant.gpu_arch, "mi300x") + self.assertEqual(variant.params.driver, "atom") + self.assertEqual(variant.params.metric_percentiles, "95,99") + self.assertEqual( + variant.roles.server.atom_args[:4], + ["-tp", "8", "--kv_cache_dtype", "fp8"], + ) + self.assertEqual( + variant.expected_cells(), + ["ISL=1024,OSL=1024,TP=8,CONC=128", "ISL=1024,OSL=1024,TP=8,CONC=256"], + ) + cell = "ISL=1024,OSL=1024,TP=8,CONC=128" + for key in ( + "client.per_gpu_throughput", + "client.output_tput_per_gpu", + "client.p99_ttft_ms", + "client.p99_tpot_ms", + "client.p95_tpot_ms", + ): + self.assertIn(key, variant.thresholds[cell]) + + def test_load_w1_mi300x_multinode_variant(self): + root = Path(__file__).resolve().parents[3] + config = root / ( + "input/config_file/inference/inferencex_atom/" + "mi300x_inferencex-atom_deepseek-r1_fp8_distributed.json" + ) + variant = load_variant(config, _cluster_dict()) + self.assertEqual(variant.params.nnodes, "2") + self.assertEqual(variant.params.driver, "vllm_atom") + self.assertEqual(variant.params.pipeline_parallel_size, "2") + self.assertEqual(variant.roles.server.ib_netdev, "auto") + self.assertEqual(variant.roles.server.ib_hca_devices, "auto") + self.assertEqual(variant.params.scaling_baseline_output_throughput, "1500") + self.assertTrue(variant.enforce_thresholds) + self.assertEqual(len(variant.expected_cells()), 15) + cell = "ISL=512,OSL=512,TP=8,PP=2,NNODES=2,CONC=16" + self.assertIn(cell, variant.expected_cells()) + self.assertEqual( + variant.thresholds[cell]["scaling.efficiency_pct"], + {"kind": "min", "value": 22}, + ) + + def test_load_w1_mi300x_multinode_sglang_variant(self): + root = Path(__file__).resolve().parents[3] + config = root / ( + "input/config_file/inference/inferencex_atom/" + "mi300x_inferencex-atom_deepseek-r1_fp8_sglang_distributed.json" + ) + variant = load_variant(config, _cluster_dict()) + self.assertEqual(variant.params.driver, "sglang") + self.assertEqual(variant.params.pipeline_parallel_size, "2") + self.assertFalse(variant.enforce_thresholds) + cell = "ISL=512,OSL=512,TP=8,PP=2,NNODES=2,CONC=16" + self.assertIn(cell, variant.expected_cells()) + + def test_load_w1_mi355x_multinode_variant(self): + root = Path(__file__).resolve().parents[3] + config = root / ( + "input/config_file/inference/inferencex_atom/" + "mi355x_inferencex-atom_deepseek-r1_fp8_distributed.json" + ) + variant = load_variant(config, _cluster_dict()) + self.assertEqual(variant.gpu_arch, "mi355x") + self.assertEqual(variant.params.nnodes, "2") + self.assertEqual(variant.params.driver, "vllm_atom") + self.assertEqual(variant.params.pipeline_parallel_size, "2") + self.assertEqual(variant.params.scaling_baseline_output_throughput, "4000") + self.assertFalse(variant.enforce_thresholds) + self.assertEqual(len(variant.expected_cells()), 15) + cell = "ISL=512,OSL=512,TP=8,PP=2,NNODES=2,CONC=16" + self.assertIn(cell, variant.expected_cells()) + self.assertEqual( + variant.thresholds[cell]["scaling.efficiency_pct"], + {"kind": "min", "value": 50}, + ) + + def test_load_baseline_sweep_mi300x_variant(self): + root = Path(__file__).resolve().parents[3] + config = root / ( + "input/config_file/inference/inferencex_atom/" + "mi300x_inferencex-atom_deepseek-r1_fp8_baseline_sweep.json" + ) + variant = load_variant(config, _cluster_dict()) + self.assertEqual(variant.params.max_model_length, "10240") + self.assertTrue(variant.enforce_thresholds) + self.assertEqual(len(variant.expected_cells()), 14) + self.assertIn("ISL=1024,OSL=1024,TP=8,CONC=4", variant.expected_cells()) + self.assertIn("ISL=8192,OSL=1024,TP=8,CONC=256", variant.expected_cells()) + cell = "ISL=8192,OSL=1024,TP=8,CONC=128" + self.assertIn("client.output_throughput", variant.thresholds[cell]) + self.assertEqual(variant.thresholds[cell]["client.success_rate"]["value"], 1) + + def test_load_baseline_sweep_multinode_mi300x_variant(self): + root = Path(__file__).resolve().parents[3] + config = root / ( + "input/config_file/inference/inferencex_atom/" + "mi300x_inferencex-atom_deepseek-r1_fp8_baseline_sweep_distributed.json" + ) + variant = load_variant(config, _cluster_dict()) + self.assertEqual(variant.params.nnodes, "2") + self.assertEqual(variant.params.driver, "vllm_atom") + self.assertEqual(variant.params.pipeline_parallel_size, "2") + self.assertEqual(variant.params.max_model_length, "10240") + self.assertEqual(variant.params.scaling_baseline_output_throughput, "1500") + self.assertTrue(variant.enforce_thresholds) + self.assertEqual(len(variant.expected_cells()), 14) + cell = "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=4" + self.assertIn(cell, variant.expected_cells()) + self.assertEqual( + variant.thresholds[cell]["scaling.efficiency_pct"], + {"kind": "min", "value": 9.0}, + ) + root = Path(__file__).resolve().parents[3] + config = root / ( + "input/config_file/inference/inferencex_atom/" + "mi355x_inferencex-atom_deepseek-r1_fp8_baseline_sweep.json" + ) + variant = load_variant(config, _cluster_dict()) + self.assertEqual(variant.gpu_arch, "mi355x") + self.assertFalse(variant.enforce_thresholds) + self.assertEqual(len(variant.expected_cells()), 14) + + def test_load_w1_mi355x_atom_single_variant_and_thresholds(self): + root = Path(__file__).resolve().parents[3] + config = root / ( + "input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_single.json" + ) + variant = load_variant(config, _cluster_dict()) + self.assertEqual(variant.gpu_arch, "mi355x") + self.assertIn("--trust-remote-code", variant.roles.server.atom_args) + self.assertEqual( + variant.expected_cells(), + ["ISL=1024,OSL=1024,TP=8,CONC=128", "ISL=1024,OSL=1024,TP=8,CONC=256"], + ) + cell = "ISL=1024,OSL=1024,TP=8,CONC=128" + self.assertEqual( + variant.thresholds[cell]["client.output_throughput"]["value"], + 4004.66, + ) + self.assertEqual( + variant.thresholds[cell]["client.mean_ttft_ms"]["value"], + 362.18, + ) + + def test_load_w1_mi355x_atom_mtp3_inline_bench_args(self): + root = Path(__file__).resolve().parents[3] + config = root / ( + "input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_mtp3.json" + ) + variant = load_variant(config, _cluster_dict()) + self.assertIn("--method", variant.roles.server.atom_args) + self.assertEqual(variant.params.bench_extra_args, "--use-chat-template") + + def test_load_w1_mi355x_atom_mtp3_thresholds(self): + root = Path(__file__).resolve().parents[3] + config = root / ( + "input/config_file/inference/inferencex_atom/mi355x_inferencex-atom_deepseek-r1_fp8_mtp3.json" + ) + variant = load_variant(config, _cluster_dict()) + cell = "ISL=1024,OSL=1024,TP=8,CONC=256" + self.assertEqual( + variant.thresholds[cell]["client.output_throughput"]["value"], + 6451.59, + ) + + def test_orchestrator_container_includes_server_env(self): + sweep = Sweep( + sequence_combinations=[SeqCombo(name="legacy_profile", isl="7168", osl="1024")], + runs=[Run(combo="legacy_profile", concurrency=64)], + ) + thresholds = { + "ISL=7168,OSL=1024,TP=8,CONC=64": placeholder_gated_threshold_cell(), + } + variant = InferenceXAtomVariantConfig( + schema_version=1, + framework="inferencex_atom", + gpu_arch="mi300x", + enforce_thresholds=False, + paths={ + "shared_fs": "/home/x", + "models_dir": "/home/x/models", + "log_dir": "/home/x/LOGS", + "hf_token_file": "/home/x/.hf", + }, + model={"id": "openai/gpt-oss-120b", "remote": 0, "precision": "bf16"}, + container={ + "name": "c", + "image": "img", + "runtime": {"name": "docker", "args": {"volumes": ["/home/x:/home/x"]}}, + }, + roles={"server": {"env": {"VLLM_ROCM_USE_AITER": "1"}}}, + params={"tensor_parallelism": "8"}, + sweep=sweep, + thresholds=thresholds, + ) + block = orchestrator_container_from_variant(variant) + self.assertEqual(block["env"]["VLLM_ROCM_USE_AITER"], "1") + + def test_expand_sweep_matches_w1_single(self): + root = Path(__file__).resolve().parents[3] + config = root / ( + "input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_single.json" + ) + import json + + raw = json.loads(config.read_text()) + cases, ids = expand_sweep(raw["sweep"]) + self.assertEqual(len(cases), 2) + self.assertEqual(ids[0], "w1_1k_1k-conc128") + self.assertEqual(ids[1], "w1_1k_1k-conc256") + self.assertEqual(cases[0][1], 128) + + def test_w1_single_threshold_health_gates_tight_when_enforcing(self): + root = Path(__file__).resolve().parents[3] + config = root / ( + "input/config_file/inference/inferencex_atom/mi300x_inferencex-atom_deepseek-r1_fp8_single.json" + ) + variant = load_variant(config, _cluster_dict()) + self.assertTrue(variant.enforce_thresholds) + cell = "ISL=1024,OSL=1024,TP=8,CONC=128" + self.assertEqual(variant.thresholds[cell]["client.success_rate"]["value"], 1) + self.assertEqual(variant.thresholds[cell]["client.failed"]["value"], 0) + + def test_placeholder_threshold_cell_covers_gated_metrics(self): + cell = placeholder_gated_threshold_cell() + from cvs.lib.inference.inferencex_atom.inferencex_atom_parsing import GATED_METRICS + + for short in GATED_METRICS: + self.assertIn(f"client.{short}", cell, short) + + def test_atom_driver_requires_inline_atom_args(self): + sweep = Sweep( + sequence_combinations=[SeqCombo(name="w1", isl="1024", osl="1024")], + runs=[Run(combo="w1", concurrency=128)], + ) + thresholds = {"ISL=1024,OSL=1024,TP=8,CONC=128": placeholder_gated_threshold_cell()} + with self.assertRaises(ValueError): + InferenceXAtomVariantConfig( + schema_version=1, + framework="inferencex_atom", + gpu_arch="mi300x", + enforce_thresholds=False, + paths={ + "shared_fs": "/home/x", + "models_dir": "/home/x/models", + "log_dir": "/home/x/LOGS", + "hf_token_file": "/home/x/.hf", + }, + model={"id": "deepseek-ai/DeepSeek-R1-0528", "remote": 0, "precision": "fp8"}, + container={ + "name": "c", + "image": "img", + "runtime": {"name": "docker", "args": {"volumes": ["/home/x:/home/x"]}}, + }, + roles={"server": {"env": {}}}, + params={"driver": "atom", "tensor_parallelism": "8"}, + sweep=sweep, + thresholds=thresholds, + ) + + def test_reuse_server_flag_and_session_key_helpers(self): + from types import SimpleNamespace + + self.assertFalse(reuse_server_flag(SimpleNamespace())) + variant = SimpleNamespace( + model=SimpleNamespace(id="m"), + params=SimpleNamespace(driver="atom", tensor_parallelism="8"), + roles=SimpleNamespace(server=SimpleNamespace(atom_args=("-tp", "8"))), + ) + self.assertNotEqual(server_session_key(variant, "1", "2"), server_session_key(variant, "3", "4")) + + def test_expand_sweep_parametrize_tier_ids(self): + sweep = { + "sequence_combinations": [{"name": "w1", "isl": "1024", "osl": "1024"}], + "runs": [{"combo": "w1", "concurrency": 128}], + } + _, _, ids = expand_sweep_parametrize(sweep, ("metric_tier",)) + self.assertIn("w1-conc128-throughput", ids) + + def test_ib_netdev_coerces_mlx5_hca_name_to_auto(self): + sweep = Sweep( + sequence_combinations=[SeqCombo(name="w1", isl="512", osl="512")], + runs=[Run(combo="w1", concurrency=16)], + ) + variant = InferenceXAtomVariantConfig( + schema_version=1, + framework="inferencex_atom", + gpu_arch="mi300x", + enforce_thresholds=False, + paths={ + "shared_fs": "/home/x", + "models_dir": "/home/x/models", + "log_dir": "/home/x/LOGS", + "hf_token_file": "/home/x/.hf", + }, + model={"id": "deepseek-ai/DeepSeek-R1-0528", "remote": 0, "precision": "fp8"}, + container={ + "name": "c", + "image": "img", + "runtime": {"name": "docker", "args": {"volumes": ["/home/x:/home/x"]}}, + }, + roles={"server": {"ib_netdev": "mlx5_0"}}, + params={ + "driver": "vllm_atom", + "tensor_parallelism": "8", + "pipeline_parallel_size": "2", + "nnodes": "2", + "master_addr": "10.0.0.1", + }, + sweep=sweep, + thresholds={}, + ) + self.assertEqual(variant.roles.server.ib_netdev, "auto") + + def test_server_env_strips_orchestrator_network_keys(self): + sweep = Sweep( + sequence_combinations=[SeqCombo(name="w1", isl="512", osl="512")], + runs=[Run(combo="w1", concurrency=16)], + ) + variant = InferenceXAtomVariantConfig( + schema_version=1, + framework="inferencex_atom", + gpu_arch="mi300x", + enforce_thresholds=False, + paths={ + "shared_fs": "/home/x", + "models_dir": "/home/x/models", + "log_dir": "/home/x/LOGS", + "hf_token_file": "/home/x/.hf", + }, + model={"id": "deepseek-ai/DeepSeek-R1-0528", "remote": 0, "precision": "fp8"}, + container={ + "name": "c", + "image": "img", + "runtime": {"name": "docker", "args": {"volumes": ["/home/x:/home/x"]}}, + }, + roles={ + "server": { + "env": { + "GLOO_SOCKET_IFNAME": "mlx5_0", + "NCCL_IB_HCA": "mlx5_0", + "NCCL_IB_GID_INDEX": "1", + } + } + }, + params={ + "driver": "vllm_atom", + "tensor_parallelism": "8", + "pipeline_parallel_size": "2", + "nnodes": "2", + "master_addr": "10.0.0.1", + }, + sweep=sweep, + thresholds={}, + ) + self.assertEqual(variant.roles.server.env, {"NCCL_IB_GID_INDEX": "1"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/inference/unittests/test_inferencex_atom_orch_parse.py b/cvs/lib/inference/unittests/test_inferencex_atom_orch_parse.py new file mode 100644 index 000000000..125f64d4e --- /dev/null +++ b/cvs/lib/inference/unittests/test_inferencex_atom_orch_parse.py @@ -0,0 +1,623 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for InferenceXAtomJob.parse_results (stock ``results`` artifact -> client.*). +No hardware: a fake orch returns committed fixture text. +''' + +import json +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from cvs.lib.inference.inferencex_atom.inferencex_atom_orch import InferenceXAtomJob +from cvs.lib.inference.unittests.fake_orch import FakeOrch + +_HERE = Path(__file__).parent +_FIXTURES = _HERE / "fixtures" +_ISL = 7168 +_OSL = 1024 +_TP = 8 + + +def _fake_variant( + *, driver="vllm", nnodes="1", pipeline_parallel_size="1", master_addr="", scaling_baseline_output_throughput="", ib_netdev="eth0", ib_hca_devices=None +): + params = SimpleNamespace( + driver=driver, + tensor_parallelism=str(_TP), + pipeline_parallel_size=pipeline_parallel_size, + nnodes=nnodes, + master_addr=master_addr, + master_port="29501", + scaling_baseline_output_throughput=scaling_baseline_output_throughput, + port_no="8000", + random_range_ratio="0.8", + random_prefix_len="0", + burstiness="1.0", + seed="0", + request_rate="inf", + tokenizer_mode="auto", + percentile_metrics="ttft,tpot,itl,e2el", + metric_percentiles="99", + base_url="http://0.0.0.0", + dataset_name="random", + backend="vllm", + max_model_length="8192", + bench_extra_args="", + result_filename="results", + ) + roles = SimpleNamespace( + server=SimpleNamespace( + serve_args={}, atom_args=[], sglang_args=[], env={}, ib_netdev=ib_netdev, ib_hca_devices=ib_hca_devices + ) + ) + paths = SimpleNamespace(log_dir="/LOGS", models_dir="/models") + model = SimpleNamespace(id="openai/gpt-oss-120b") + return SimpleNamespace(params=params, roles=roles, paths=paths, model=model) + + +class TestInferenceXAtomOrchParse(unittest.TestCase): + def test_parse_results_maps_client_metrics(self): + raw = json.loads((_FIXTURES / "vllm_results_sample.json").read_text()) + job = InferenceXAtomJob( + orch=FakeOrch(exec_return={"node0": json.dumps(raw)}), + variant=_fake_variant(driver="vllm"), + hf_token="tok", + isl=_ISL, + osl=_OSL, + concurrency=64, + num_prompts=100, + ) + out = job.parse_results() + metrics = out["node0"] + w = raw + self.assertIn("client.output_throughput", metrics) + self.assertIn("client.mean_ttft_ms", metrics) + self.assertAlmostEqual(metrics["client.per_gpu_throughput"], w["total_token_throughput"] / _TP) + self.assertAlmostEqual(metrics["client.output_tput_per_gpu"], w["output_throughput"] / _TP) + self.assertEqual(metrics["client.p99_ttft_ms"], w["p99_ttft_ms"]) + + def test_parse_results_w1_tail_metrics_from_widened_fixture(self): + raw = json.loads((_FIXTURES / "vllm_results_widened.json").read_text()) + job = InferenceXAtomJob( + orch=FakeOrch(exec_return={"node0": json.dumps(raw)}), + variant=_fake_variant(driver="atom"), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + metrics = job.parse_results()["node0"] + self.assertEqual(metrics["client.p95_tpot_ms"], raw["p95_tpot_ms"]) + self.assertEqual(metrics["client.p99_ttft_ms"], raw["p99_ttft_ms"]) + + def test_parse_results_atom_json_suffix(self): + raw = json.loads((_FIXTURES / "vllm_results_sample.json").read_text()) + job = InferenceXAtomJob( + orch=FakeOrch(exec_return={"node0": json.dumps(raw)}), + variant=_fake_variant(driver="atom"), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + self.assertTrue(job._result_artifact.endswith("/results.json")) + out = job.parse_results() + self.assertIn("client.output_throughput", out["node0"]) + + def test_run_client_clears_stale_result_artifact(self): + orch = FakeOrch() + job = InferenceXAtomJob( + orch=orch, + variant=_fake_variant(driver="atom"), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=1000, + ) + job.run_client() + rm_cmds = [c for c, _ in orch.commands if c.startswith("rm -f ")] + self.assertEqual(len(rm_cmds), 1) + self.assertIn(job._result_artifact, rm_cmds[0]) + self.assertTrue(any("benchmark_serving" in c for c, _ in orch.commands)) + + def test_parse_results_empty_artifact_raises(self): + job = InferenceXAtomJob( + orch=FakeOrch(exec_return={"node0": ""}), + variant=_fake_variant(driver="atom"), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + with self.assertRaisesRegex(RuntimeError, "empty/missing results artifact"): + job.parse_results() + + def test_parse_results_invalid_json_raises(self): + job = InferenceXAtomJob( + orch=FakeOrch(exec_return={"node0": "not-json"}), + variant=_fake_variant(driver="atom"), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + with self.assertRaisesRegex(RuntimeError, "unparseable results artifact"): + job.parse_results() + + def test_merged_serve_args_promotes_gpu_memory_util(self): + variant = _fake_variant(driver="vllm") + variant.roles.server.env = {"CVS_GPU_MEMORY_UTIL": "0.92"} + merged = InferenceXAtomJob._merged_serve_args(variant) + self.assertEqual(merged["gpu-memory-utilization"], "0.92") + + def test_merged_serve_args_skips_promotion_when_flag_present(self): + variant = _fake_variant(driver="vllm") + variant.roles.server.serve_args = {"gpu-memory-utilization": "0.75"} + variant.roles.server.env = {"CVS_GPU_MEMORY_UTIL": "0.92"} + merged = InferenceXAtomJob._merged_serve_args(variant) + self.assertEqual(merged["gpu-memory-utilization"], "0.75") + + def test_build_server_cmd_suppresses_gpu_memory_env_vars(self): + orch = FakeOrch() + variant = _fake_variant(driver="vllm") + variant.roles.server.env = { + "CVS_GPU_MEMORY_UTIL": "0.92", + "VLLM_GPU_MEMORY_UTIL": "0.91", + "VLLM_ENFORCE_EAGER": "1", + "CUSTOM_FLAG": "on", + } + job = InferenceXAtomJob( + orch=orch, + variant=variant, + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + job.build_server_cmd() + env_cmd = orch.commands[0][0] + self.assertNotIn("CVS_GPU_MEMORY_UTIL", env_cmd) + self.assertNotIn("VLLM_GPU_MEMORY_UTIL", env_cmd) + self.assertNotIn("VLLM_ENFORCE_EAGER", env_cmd) + self.assertIn("CUSTOM_FLAG", env_cmd) + + def test_client_log_failures_traceback(self): + job = InferenceXAtomJob( + orch=FakeOrch(exec_return={"node0": "Traceback (most recent call last):\n boom"}), + variant=_fake_variant(driver="atom"), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + failed = job._client_log_failures() + self.assertEqual(len(failed), 1) + self.assertIn("node0", failed[0][0]) + + def test_client_log_failures_launch_error(self): + job = InferenceXAtomJob( + orch=FakeOrch(exec_return={"node0": "error: argument --foo: invalid choice"}), + variant=_fake_variant(driver="atom"), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + self.assertEqual(len(job._client_log_failures()), 1) + + def test_client_log_failures_failed_requests_over_cap(self): + job = InferenceXAtomJob( + orch=FakeOrch(exec_return={"node0": "Failed requests: 3\n"}), + variant=_fake_variant(driver="atom"), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + job._bench_max_failed_requests = 0 + failed = job._client_log_failures() + self.assertEqual(len(failed), 1) + self.assertIn("Failed requests: 3", failed[0][1]) + + def test_client_log_failures_failed_requests_within_cap_warns(self): + job = InferenceXAtomJob( + orch=FakeOrch(exec_return={"node0": "Failed requests: 1\n"}), + variant=_fake_variant(driver="atom"), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + job._bench_max_failed_requests = 2 + failed = job._client_log_failures() + self.assertEqual(failed, []) + + def test_early_failure_regexes(self): + job = InferenceXAtomJob( + orch=FakeOrch(), + variant=_fake_variant(driver="atom"), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + self.assertTrue(job.FAILED_REQUESTS_RE.search("Failed requests: 2")) + self.assertTrue(job.CLIENT_CRASH_RE.search("Traceback (most recent call last)")) + self.assertTrue(job.CLIENT_LAUNCH_FAIL_RE.search("unrecognized arguments: --bad")) + self.assertTrue(job.EARLY_FAILURE_RE.search("No such file or directory")) + + def test_distributed_start_server_targets_each_host(self): + orch = FakeOrch(hosts=["10.0.0.1", "10.0.0.2"]) + job = InferenceXAtomJob( + orch=orch, + variant=_fake_variant(driver="atom", nnodes="2", pipeline_parallel_size="2", master_addr="10.0.0.1"), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + job.build_server_cmd(clear_atom_cache=False) + job.start_server() + launch_cmds = [c for c, hosts in orch.commands if hosts] + self.assertEqual(len(launch_cmds), 2) + self.assertNotIn("--node-rank", launch_cmds[0]) + self.assertNotIn("--distributed-executor-backend", launch_cmds[0]) + self.assertIn("openai_server", launch_cmds[0]) + + def test_distributed_atom_spmd_env_and_dp_when_tp_allows(self): + orch = FakeOrch(hosts=["10.0.0.1", "10.0.0.2"]) + variant = _fake_variant(driver="atom", nnodes="2", pipeline_parallel_size="2", master_addr="10.0.0.1") + variant.params.tensor_parallelism = "4" + variant.roles.server.atom_args = ["-tp", "4"] + job = InferenceXAtomJob( + orch=orch, + variant=variant, + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + job.start_server() + launch_cmds = [c for c, hosts in orch.commands if hosts] + self.assertIn("-dp 2", launch_cmds[0]) + self.assertIn("ATOM_DP_RANK=0", launch_cmds[0]) + self.assertIn("ATOM_DP_RANK=1", launch_cmds[1]) + self.assertIn("ATOM_DP_MASTER_IP=10.0.0.1", launch_cmds[0]) + + def test_distributed_atom_tp8_multinode_couples_spmd_dp(self): + orch = FakeOrch(hosts=["10.0.0.1", "10.0.0.2"]) + variant = _fake_variant(driver="atom", nnodes="2", pipeline_parallel_size="2", master_addr="10.0.0.1") + variant.params.tensor_parallelism = "8" + variant.roles.server.atom_args = ["-tp", "8"] + job = InferenceXAtomJob( + orch=orch, + variant=variant, + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + job.start_server() + launch_cmds = [c for c, hosts in orch.commands if hosts] + self.assertEqual(len(launch_cmds), 2) + self.assertIn("-dp 2", launch_cmds[0]) + self.assertIn("-dp 2", launch_cmds[1]) + self.assertIn("ATOM_DP_RANK=0", launch_cmds[0]) + self.assertIn("ATOM_DP_RANK=1", launch_cmds[1]) + self.assertIn("ATOM_DP_SIZE=2", launch_cmds[0]) + + def test_distributed_atom_tp8_multinode_never_passes_vllm_flags(self): + orch = FakeOrch(hosts=["10.0.0.1", "10.0.0.2"]) + variant = _fake_variant(driver="atom", nnodes="2", pipeline_parallel_size="2", master_addr="10.0.0.1") + variant.roles.server.atom_args = [ + "-tp", + "8", + "--node-rank", + "1", + "--pipeline-parallel-size", + "2", + ] + job = InferenceXAtomJob( + orch=orch, + variant=variant, + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + argv = job._atom_server_argv(rank=1) + joined = " ".join(argv) + self.assertNotIn("--node-rank", joined) + self.assertNotIn("--pipeline-parallel-size", joined) + self.assertNotIn("--master-addr", joined) + self.assertIn("-tp 8", joined) + job.start_server() + launch_cmds = [c for c, hosts in orch.commands if hosts] + self.assertNotIn("--node-rank", launch_cmds[1]) + self.assertNotIn("--pipeline-parallel-size", launch_cmds[1]) + + def test_distributed_client_uses_exec_on_head(self): + orch = FakeOrch(hosts=["10.0.0.1", "10.0.0.2"]) + job = InferenceXAtomJob( + orch=orch, + variant=_fake_variant(driver="atom", nnodes="2", pipeline_parallel_size="2"), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + job.run_client() + self.assertEqual(len(orch.exec_on_head_commands), 2) + self.assertTrue(any("benchmark_serving" in c for c in orch.exec_on_head_commands)) + + def test_distributed_vllm_atom_pp2_passes_vllm_executor_flags(self): + orch = FakeOrch(hosts=["10.0.0.1", "10.0.0.2"]) + variant = _fake_variant( + driver="vllm_atom", nnodes="2", pipeline_parallel_size="2", master_addr="10.0.0.1" + ) + job = InferenceXAtomJob( + orch=orch, + variant=variant, + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + argv0 = job._server_argv(rank=0) + argv1 = job._server_argv(rank=1) + joined0 = " ".join(argv0) + joined1 = " ".join(argv1) + self.assertIn("--pipeline-parallel-size 2", joined0) + self.assertIn("--node-rank 0", joined0) + self.assertIn("--node-rank 1", joined1) + self.assertIn("--headless", joined1) + self.assertNotIn("--headless", joined0) + job.start_server() + launch_cmds = [c for c, hosts in orch.commands if hosts] + self.assertIn("vllm serve", launch_cmds[0]) + self.assertIn("--pipeline-parallel-size 2", launch_cmds[1]) + + def test_distributed_sglang_pp2_passes_sglang_dist_flags(self): + orch = FakeOrch(hosts=["10.0.0.1", "10.0.0.2"]) + variant = _fake_variant( + driver="sglang", nnodes="2", pipeline_parallel_size="2", master_addr="10.0.0.1" + ) + variant.roles.server.sglang_args = ["--trust-remote-code"] + job = InferenceXAtomJob( + orch=orch, + variant=variant, + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + argv = job._sglang_server_argv(rank=1) + joined = " ".join(argv) + self.assertIn("sglang.launch_server", joined) + self.assertIn("--pp-size 2", joined) + self.assertIn("--node-rank 1", joined) + self.assertIn("--dist-init-addr 10.0.0.1:29501", joined) + client = " ".join(job._sglang_client_argv()) + self.assertIn("sglang.bench_serving", client) + + def test_parse_results_scaling_efficiency(self): + raw = json.loads((_FIXTURES / "vllm_results_sample.json").read_text()) + job = InferenceXAtomJob( + orch=FakeOrch(exec_on_head_return={"head": json.dumps(raw)}), + variant=_fake_variant( + driver="atom", + nnodes="2", + scaling_baseline_output_throughput="100", + ), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + metrics = job.parse_results()["head"] + self.assertIn("scaling.efficiency_pct", metrics) + expected = (raw["output_throughput"] / (100.0 * 2)) * 100.0 + self.assertAlmostEqual(metrics["scaling.efficiency_pct"], expected) + + +class TestInferenceXAtomBuildServerCmd(unittest.TestCase): + @staticmethod + def _env_script(orch): + return orch.commands[0][0] + + def test_nccl_ib_hca_line_present_only_when_ib_hcas_supplied(self): + cases = [ + (["mlx5_0", "mlx5_1"], True), + ([], False), + (None, False), + ] + for ib_hcas, present in cases: + with self.subTest(ib_hcas=ib_hcas): + orch = FakeOrch() + job = InferenceXAtomJob( + orch=orch, + variant=_fake_variant( + driver="vllm_atom", + nnodes="2", + pipeline_parallel_size="2", + master_addr="10.0.0.1", + ), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ib_hcas=ib_hcas, + ib_netdev="eth0", + ) + job.build_server_cmd() + script = self._env_script(orch) + if present: + self.assertIn("NCCL_IB_HCA", script) + self.assertIn("mlx5_0", script) + else: + self.assertNotIn("NCCL_IB_HCA", script) + + def test_socket_ifname_exports_present_only_when_distributed_ib_netdev_set(self): + orch = FakeOrch() + job = InferenceXAtomJob( + orch=orch, + variant=_fake_variant( + driver="vllm_atom", + nnodes="2", + pipeline_parallel_size="2", + master_addr="10.0.0.1", + ), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + job.build_server_cmd() + script = self._env_script(orch) + self.assertEqual(script.count("SOCKET_IFNAME"), 3) + self.assertIn("eth0", script) + + orch_single = FakeOrch() + job_single = InferenceXAtomJob( + orch=orch_single, + variant=_fake_variant(driver="vllm"), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + job_single.build_server_cmd() + script_single = self._env_script(orch_single) + self.assertNotIn("SOCKET_IFNAME", script_single) + + @patch("cvs.lib.utils.ib_discovery.resolve_multinode_fabric") + def test_build_server_cmd_resolves_topology_when_lifecycle_skipped(self, mock_resolve): + mock_resolve.return_value = (["mlx5_0", "mlx5_1"], "ens51f1np1") + orch = FakeOrch(hosts=["10.32.80.112", "10.32.80.113"]) + job = InferenceXAtomJob( + orch=orch, + variant=_fake_variant( + driver="vllm_atom", + nnodes="2", + pipeline_parallel_size="2", + master_addr="10.32.80.112", + ib_netdev="auto", + ib_hca_devices="auto", + ), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + job.build_server_cmd() + mock_resolve.assert_called_once() + script = self._env_script(orch) + self.assertIn("NCCL_IB_HCA", script) + self.assertIn("mlx5_0", script) + self.assertEqual(script.count("SOCKET_IFNAME"), 3) + self.assertIn("ens51f1np1", script) + + +class _RecordingOrch: + hosts = ["10.0.0.1", "10.0.0.2"] + + def __init__(self, responder=None, hosts=None): + self.calls = [] + self._responder = responder + if hosts is not None: + self.hosts = list(hosts) + + def exec(self, cmd, hosts=None, detailed=False, **kwargs): + self.calls.append((cmd, hosts)) + if self._responder is not None: + return self._responder(cmd, hosts, detailed) + return {} + + def exec_on_head(self, cmd, **kwargs): + return {} + + +def _readiness_responder(exit_code=0, empty=False): + def responder(cmd, hosts, detailed): + if empty: + return {} + host = hosts[0] if hosts else _RecordingOrch.hosts[0] + if detailed: + return {host: {"exit_code": exit_code, "output": "", "stdout": ""}} + return {host: ""} + + return responder + + +class TestInferenceXAtomIsReady(unittest.TestCase): + def test_multinode_vllm_atom_skips_worker_readiness_grep(self): + head, worker = _RecordingOrch.hosts + orch = _RecordingOrch(responder=_readiness_responder(exit_code=0)) + job = InferenceXAtomJob( + orch=orch, + variant=_fake_variant( + driver="vllm_atom", + nnodes="2", + pipeline_parallel_size="2", + master_addr=head, + ), + hf_token="tok", + isl="512", + osl="512", + concurrency=16, + num_prompts=128, + ) + self.assertTrue(job.is_ready()) + worker_calls = [hosts for _cmd, hosts in orch.calls if hosts == [worker]] + self.assertEqual(worker_calls, [], "headless worker must not be grepped for Uvicorn startup") + self.assertTrue(any(hosts == [head] for _cmd, hosts in orch.calls)) + + def test_multinode_vllm_atom_false_when_head_not_ready(self): + head = _RecordingOrch.hosts[0] + orch = _RecordingOrch(responder=_readiness_responder(exit_code=1)) + job = InferenceXAtomJob( + orch=orch, + variant=_fake_variant( + driver="vllm_atom", + nnodes="2", + pipeline_parallel_size="2", + master_addr=head, + ), + hf_token="tok", + isl="512", + osl="512", + concurrency=16, + num_prompts=128, + ) + self.assertFalse(job.is_ready()) + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/inference/unittests/test_inferencex_atom_parsing.py b/cvs/lib/inference/unittests/test_inferencex_atom_parsing.py new file mode 100644 index 000000000..485708c24 --- /dev/null +++ b/cvs/lib/inference/unittests/test_inferencex_atom_parsing.py @@ -0,0 +1,74 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. +''' + +import unittest + +from cvs.lib.inference.inferencex_atom.inferencex_atom_parsing import ( + CLIENT_METRICS, + ENFORCED_METRICS, + GATED_METRICS, + METRIC_TIERS, + tier_metric_specs, +) + + +class TestInferenceXAtomParsing(unittest.TestCase): + def test_gated_metrics_include_w1_ix_extras(self): + for name in ("per_gpu_throughput", "output_tput_per_gpu", "p99_tpot_ms", "p99_ttft_ms"): + self.assertIn(name, GATED_METRICS) + + def test_enforced_metrics_cover_all_tiers(self): + tiered = {m for names in METRIC_TIERS.values() for m in names} + self.assertEqual(ENFORCED_METRICS, frozenset(tiered)) + + def test_tier_metric_specs_throughput(self): + cell = { + "client.output_throughput": {"kind": "min_tok_s", "value": 1}, + "client.mean_ttft_ms": {"kind": "max_ms", "value": 2}, + } + specs = tier_metric_specs(cell, "throughput") + self.assertIn("client.output_throughput", specs) + self.assertNotIn("client.mean_ttft_ms", specs) + + def test_tier_metric_specs_tpot_uses_p99_tail(self): + cell = { + "client.mean_tpot_ms": {"kind": "max_ms", "value": 46.8}, + "client.p99_tpot_ms": {"kind": "max_ms", "value": 51.36}, + "client.p95_tpot_ms": {"kind": "max_ms", "value": 53.76}, + } + specs = tier_metric_specs(cell, "tpot") + self.assertIn("client.p99_tpot_ms", specs) + self.assertNotIn("client.p95_tpot_ms", specs) + + def test_tier_metric_specs_record_includes_non_tiered(self): + cell = { + "client.median_ttft_ms": {"kind": "max_ms", "value": 9}, + "client.output_throughput": {"kind": "min_tok_s", "value": 1}, + } + specs = tier_metric_specs(cell, "record") + self.assertIn("client.median_ttft_ms", specs) + self.assertNotIn("client.output_throughput", specs) + + def test_tier_metric_specs_scaling(self): + cell = { + "scaling.efficiency_pct": {"kind": "min", "value": 50}, + "client.output_throughput": {"kind": "min_tok_s", "value": 1}, + } + specs = tier_metric_specs(cell, "scaling") + self.assertEqual(specs, {"scaling.efficiency_pct": {"kind": "min", "value": 50}}) + + def test_gated_metrics_subset_of_client_metrics(self): + client_short = {short for short, _unit in CLIENT_METRICS} + missing = GATED_METRICS - client_short + self.assertEqual(missing, set(), f"GATED_METRICS not in CLIENT_METRICS: {missing}") + + def test_health_tier_metrics_in_enforced_set(self): + for name in ("success_rate", "failed"): + self.assertIn(name, ENFORCED_METRICS) + self.assertIn(name, METRIC_TIERS["health"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/inference/unittests/test_inferencex_atom_server_reuse.py b/cvs/lib/inference/unittests/test_inferencex_atom_server_reuse.py new file mode 100644 index 000000000..276d48068 --- /dev/null +++ b/cvs/lib/inference/unittests/test_inferencex_atom_server_reuse.py @@ -0,0 +1,87 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for InferenceX ATOM server-reuse helpers and sweep parametrization. +''' + +import unittest +from types import SimpleNamespace + +from cvs.lib.inference.inferencex_atom.inferencex_atom_config_loader import ( + expand_sweep_parametrize, + reuse_server_flag, + server_session_key, +) +from cvs.lib.inference.inferencex_atom.inferencex_atom_parsing import METRIC_TIER_ORDER + + +class TestServerReuseHelpers(unittest.TestCase): + def test_reuse_server_flag_truthy_values(self): + for raw in ("true", "1", "yes", "TRUE", " Yes "): + params = SimpleNamespace(reuse_server_across_sweep=raw) + self.assertTrue(reuse_server_flag(params), raw) + + def test_reuse_server_flag_falsey_values(self): + for raw in ("false", "0", "no", "", "maybe"): + params = SimpleNamespace(reuse_server_across_sweep=raw) + self.assertFalse(reuse_server_flag(params), raw) + + def test_reuse_server_flag_defaults_false_when_missing(self): + self.assertFalse(reuse_server_flag(SimpleNamespace())) + + def test_server_session_key_differs_for_model(self): + base = SimpleNamespace( + model=SimpleNamespace(id="model-a"), + params=SimpleNamespace(driver="atom", tensor_parallelism="8"), + roles=SimpleNamespace(server=SimpleNamespace(atom_args=("-tp", "8"))), + ) + other = SimpleNamespace( + model=SimpleNamespace(id="model-b"), + params=SimpleNamespace(driver="atom", tensor_parallelism="8"), + roles=SimpleNamespace(server=SimpleNamespace(atom_args=("-tp", "8"))), + ) + k1 = server_session_key(base, "1024", "1024") + k2 = server_session_key(other, "1024", "1024") + self.assertNotEqual(k1, k2) + + def test_server_session_key_differs_for_shape(self): + variant = SimpleNamespace( + model=SimpleNamespace(id="model-a"), + params=SimpleNamespace(driver="atom", tensor_parallelism="8"), + roles=SimpleNamespace(server=SimpleNamespace(atom_args=("-tp", "8"))), + ) + self.assertNotEqual( + server_session_key(variant, "1024", "1024"), + server_session_key(variant, "2048", "2048"), + ) + + +class TestExpandSweepParametrize(unittest.TestCase): + def test_metric_tier_expansion_multiplies_cases(self): + sweep = { + "sequence_combinations": [{"name": "w1_1k_1k", "isl": "1024", "osl": "1024"}], + "runs": [{"combo": "w1_1k_1k", "concurrency": 128}], + } + spec = expand_sweep_parametrize(sweep, ("metric_tier",)) + argnames, argvalues, ids = spec + self.assertEqual(argnames, "seq_combo,concurrency,metric_tier") + self.assertEqual(len(argvalues), len(METRIC_TIER_ORDER)) + self.assertEqual(len(ids), len(METRIC_TIER_ORDER)) + self.assertEqual(ids[0], "w1_1k_1k-conc128-throughput") + + def test_inference_only_parametrize_without_metric_tier(self): + sweep = { + "sequence_combinations": [{"name": "w1_1k_1k", "isl": "1024", "osl": "1024"}], + "runs": [ + {"combo": "w1_1k_1k", "concurrency": 128}, + {"combo": "w1_1k_1k", "concurrency": 256}, + ], + } + _, argvalues, ids = expand_sweep_parametrize(sweep, ("seq_combo", "concurrency")) + self.assertEqual(len(argvalues), 2) + self.assertEqual(ids, ["w1_1k_1k-conc128", "w1_1k_1k-conc256"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/inference/unittests/test_inferencing_config_loader.py b/cvs/lib/inference/unittests/test_inferencing_config_loader.py new file mode 100644 index 000000000..b5dfc4f7e --- /dev/null +++ b/cvs/lib/inference/unittests/test_inferencing_config_loader.py @@ -0,0 +1,371 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for cvs.lib.utils.config_loader (ModelSpec, BaseVariantConfig, +substitute_config) and cvs.lib.inference.utils.inferencing_config_loader +(Sweep, SeqCombo, GoodputSlo, Run, VariantConfig.expected_cells, +_check_thresholds_cover_sweep). No hardware. +''' + +import unittest +import warnings + +from pydantic import ValidationError + +from cvs.lib.inference.utils.inferencing_config_loader import ( + GoodputSlo, + Run, + SeqCombo, + Sweep, + VariantConfig, +) +from cvs.lib.utils.config_loader import ModelSpec +from cvs.lib.inference.utils.vllm_parsing import GATED_METRICS + + +def _combo(name, isl="128", osl="2048"): + return SeqCombo(name=name, isl=isl, osl=osl) + + +def _full_gated_specs(): + """A spec for every gated metric -- the minimum that satisfies coverage. + + Values are inert (a 0 floor / huge ceiling) so the set passes without + asserting anything; these tests pin the coverage gate, not the numbers. + """ + out = {} + for m in GATED_METRICS: + kind = "max_ms" if m.endswith("_ms") else "max" if m == "failed" else "min" + out[f"client.{m}"] = {"kind": kind, "value": 0 if kind == "min" else 1e12} + return out + + +def _variant(sweep, tp="8", thresholds=None, enforce_thresholds=False): + """A minimal VariantConfig carrying just enough to exercise expected_cells. + + remote=0 (the remote guard would otherwise reject it) and + enforce_thresholds=False so the empty threshold dict does not trip the + coverage check -- this test pins the selector expansion, not the gate. + """ + return VariantConfig( + schema_version=1, + framework="vllm_single", + gpu_arch="mi300x", + enforce_thresholds=enforce_thresholds, + threshold_json="", + paths={ + "shared_fs": "/home/x", + "models_dir": "/home/x/models", + "log_dir": "/home/x/LOGS", + "hf_token_file": "/home/x/.hf", + }, + model={"id": "amd/Llama-3.1-70B-Instruct-FP8-KV", "remote": 0}, + container={ + "name": "c", + "image": "rocm/vllm-dev:nightly-sshd", + "runtime": {"name": "docker"}, + }, + params={"tensor_parallelism": tp}, + sweep=sweep, + thresholds=thresholds or {}, + ) + + +class TestSweepValidator(unittest.TestCase): + def test_valid_runs_selector_constructs(self): + sw = Sweep( + sequence_combinations=[_combo("a"), _combo("b", osl="4096")], + runs=[Run(combo="a", concurrency=16), Run(combo="b", concurrency=32)], + ) + self.assertEqual([r.combo for r in sw.runs], ["a", "b"]) + + def test_unknown_run_combo_raises(self): + with self.assertRaises(ValidationError) as ctx: + Sweep( + sequence_combinations=[_combo("a")], + runs=[Run(combo="typo", concurrency=16)], + ) + self.assertIn("names no sequence_combination", str(ctx.exception)) + + def test_duplicate_combo_names_raise(self): + with self.assertRaises(ValidationError) as ctx: + Sweep( + sequence_combinations=[_combo("a"), _combo("a", osl="4096")], + runs=[Run(combo="a", concurrency=16)], + ) + self.assertIn("duplicate sequence_combination names", str(ctx.exception)) + + def test_concurrency_levels_is_rejected(self): + # The old cartesian key must be gone (extra=forbid): a config still + # carrying concurrency_levels should fail loudly, not silently ignore it. + with self.assertRaises(ValidationError): + Sweep( + sequence_combinations=[_combo("a")], + runs=[Run(combo="a", concurrency=16)], + concurrency_levels=[16], + ) + + +class TestExpectedCells(unittest.TestCase): + def test_runs_expand_to_exactly_their_cells(self): + sw = Sweep( + sequence_combinations=[_combo("a", isl="128", osl="2048"), _combo("b", isl="256", osl="4096")], + runs=[ + Run(combo="a", concurrency=16), + Run(combo="b", concurrency=32), + Run(combo="a", concurrency=64), + ], + ) + vc = _variant(sw) + self.assertEqual( + vc.expected_cells(), + [ + "ISL=128,OSL=2048,TP=8,CONC=16", + "ISL=256,OSL=4096,TP=8,CONC=32", + "ISL=128,OSL=2048,TP=8,CONC=64", + ], + ) + + def test_no_cartesian_blowup(self): + # Two combos + two runs must yield TWO cells, not 2x2=4 (the old bug). + sw = Sweep( + sequence_combinations=[_combo("a"), _combo("b", osl="4096")], + runs=[Run(combo="a", concurrency=16), Run(combo="b", concurrency=16)], + ) + self.assertEqual(len(_variant(sw).expected_cells()), 2) + + +class TestGatedMetricCoverage(unittest.TestCase): + """The gated-metric axis of _check_thresholds_cover_sweep.""" + + _CELL = "ISL=128,OSL=2048,TP=8,CONC=16" + + def _variant_with(self, thresholds, enforce): + sw = Sweep( + sequence_combinations=[_combo("a")], + runs=[Run(combo="a", concurrency=16)], + ) + return VariantConfig( + schema_version=1, + framework="vllm_single", + gpu_arch="mi300x", + enforce_thresholds=enforce, + threshold_json="", + paths={ + "shared_fs": "/home/x", + "models_dir": "/home/x/models", + "log_dir": "/home/x/LOGS", + "hf_token_file": "/home/x/.hf", + }, + model={"id": "amd/Llama-3.1-70B-Instruct-FP8-KV", "remote": 0}, + container={"name": "c", "image": "rocm/vllm-dev:nightly-sshd", "runtime": {"name": "docker"}}, + params={"tensor_parallelism": "8"}, + sweep=sw, + thresholds=thresholds, + ) + + def test_full_gated_set_constructs(self): + vc = self._variant_with({self._CELL: _full_gated_specs()}, enforce=True) + self.assertEqual(vc.enforce_thresholds, True) + + def test_missing_gated_metric_raises_when_enforced(self): + specs = _full_gated_specs() + del specs["client.p99_ttft_ms"] # drop one gated metric + with self.assertRaises(ValidationError) as ctx: + self._variant_with({self._CELL: specs}, enforce=True) + self.assertIn("missing gated-metric specs", str(ctx.exception)) + self.assertIn("client.p99_ttft_ms", str(ctx.exception)) + + def test_missing_gated_metric_warns_when_record_only(self): + specs = _full_gated_specs() + del specs["client.failed"] + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self._variant_with({self._CELL: specs}, enforce=False) + self.assertTrue(any("missing gated-metric specs" in str(x.message) for x in caught)) + + def test_extra_non_gated_spec_is_allowed(self): + # A spec for a non-gated metric (record-only display extra) must not + # trip coverage -- gating is a floor, not an allow-list. + specs = _full_gated_specs() + specs["client.num_prompts"] = {"kind": "min", "value": 0} + vc = self._variant_with({self._CELL: specs}, enforce=True) + self.assertIn("client.num_prompts", vc.thresholds[self._CELL]) + + +class TestModelSpecNoPrecision(unittest.TestCase): + """precision must not be accepted by ModelSpec (removed field).""" + + def test_precision_field_is_rejected(self): + with self.assertRaises(ValidationError): + ModelSpec(id="amd/Llama-3.1-70B", remote=0, precision="fp8") + + def test_valid_model_spec_without_precision(self): + ms = ModelSpec(id="amd/Llama-3.1-70B", remote=0) + self.assertEqual(ms.id, "amd/Llama-3.1-70B") + self.assertEqual(ms.remote, 0) + + +class TestThresholdJsonField(unittest.TestCase): + """threshold_json is a required field on BaseVariantConfig / VariantConfig.""" + + def _base_kwargs(self): + sw = Sweep( + sequence_combinations=[_combo("a")], + runs=[Run(combo="a", concurrency=16)], + ) + return dict( + schema_version=1, + framework="vllm_single", + gpu_arch="mi300x", + enforce_thresholds=False, + paths={ + "shared_fs": "/home/x", + "models_dir": "/home/x/models", + "log_dir": "/home/x/LOGS", + "hf_token_file": "/home/x/.hf", + }, + model={"id": "amd/Llama-3.1-70B", "remote": 0}, + container={"name": "c", "image": "img", "runtime": {"name": "docker"}}, + params={"tensor_parallelism": "8"}, + sweep=sw, + thresholds={}, + ) + + def test_missing_threshold_json_raises(self): + kwargs = self._base_kwargs() + # threshold_json deliberately absent + with self.assertRaises(ValidationError): + VariantConfig(**kwargs) + + def test_threshold_json_present_constructs(self): + kwargs = self._base_kwargs() + kwargs["threshold_json"] = "/some/absolute/path/threshold.json" + vc = VariantConfig(**kwargs) + self.assertEqual(vc.threshold_json, "/some/absolute/path/threshold.json") + + +class TestCellCoverageAxis(unittest.TestCase): + """Axis-1 of _check_thresholds_cover_sweep: cell vs threshold key mismatch.""" + + _CELL = "ISL=128,OSL=2048,TP=8,CONC=16" + + def _variant_with(self, thresholds, enforce=True): + sw = Sweep( + sequence_combinations=[_combo("a")], + runs=[Run(combo="a", concurrency=16)], + ) + return VariantConfig( + schema_version=1, + framework="vllm_single", + gpu_arch="mi300x", + enforce_thresholds=enforce, + threshold_json="", + paths={ + "shared_fs": "/home/x", + "models_dir": "/home/x/models", + "log_dir": "/home/x/LOGS", + "hf_token_file": "/home/x/.hf", + }, + model={"id": "amd/Llama-3.1-70B", "remote": 0}, + container={"name": "c", "image": "img", "runtime": {"name": "docker"}}, + params={"tensor_parallelism": "8"}, + sweep=sw, + thresholds=thresholds, + ) + + def test_sweep_cell_with_no_threshold_entry_raises(self): + # thresholds is empty -> sweep cell has no entry -> axis-1 fires + with self.assertRaises(ValidationError) as ctx: + self._variant_with(thresholds={}, enforce=True) + self.assertIn("sweep cells with no threshold entry", str(ctx.exception)) + self.assertIn(self._CELL, str(ctx.exception)) + + def test_threshold_key_matching_no_sweep_cell_raises(self): + # thresholds has the real cell PLUS a bogus key -> extra set is non-empty + specs = _full_gated_specs() + with self.assertRaises(ValidationError) as ctx: + self._variant_with( + thresholds={self._CELL: specs, "ISL=999,OSL=999,TP=8,CONC=99": specs}, + enforce=True, + ) + self.assertIn("threshold keys matching no sweep cell", str(ctx.exception)) + self.assertIn("ISL=999,OSL=999,TP=8,CONC=99", str(ctx.exception)) + + def test_cell_mismatch_warns_when_record_only(self): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self._variant_with(thresholds={}, enforce=False) + self.assertTrue(any("sweep cells with no threshold entry" in str(w.message) for w in caught)) + + +class TestExpectedCellsBoundaries(unittest.TestCase): + """Boundary cases for VariantConfig.expected_cells.""" + + def test_empty_runs_yields_empty_cells(self): + sw = Sweep(sequence_combinations=[_combo("a")], runs=[]) + self.assertEqual(_variant(sw).expected_cells(), []) + + def test_unreferenced_combo_not_in_expected_cells(self): + # combo 'unused' is declared but never referenced by any run + sw = Sweep( + sequence_combinations=[_combo("a"), _combo("unused", isl="999", osl="999")], + runs=[Run(combo="a", concurrency=16)], + ) + cells = _variant(sw).expected_cells() + self.assertEqual(len(cells), 1) + self.assertNotIn("ISL=999", cells[0]) + + +class TestGoodputSlo(unittest.TestCase): + """GoodputSlo is a _Forbid model with three required float fields.""" + + def test_valid_goodput_slo_constructs(self): + slo = GoodputSlo(ttft_ms=100.0, tpot_ms=50.0, e2el_ms=5000.0) + self.assertEqual(slo.ttft_ms, 100.0) + self.assertEqual(slo.tpot_ms, 50.0) + self.assertEqual(slo.e2el_ms, 5000.0) + + def test_missing_required_field_raises(self): + for missing in ("ttft_ms", "tpot_ms", "e2el_ms"): + with self.subTest(missing=missing): + kwargs = {"ttft_ms": 1.0, "tpot_ms": 1.0, "e2el_ms": 1.0} + del kwargs[missing] + with self.assertRaises(ValidationError): + GoodputSlo(**kwargs) + + def test_extra_key_raises(self): + with self.assertRaises(ValidationError): + GoodputSlo(ttft_ms=1.0, tpot_ms=1.0, e2el_ms=1.0, ttft_msec=1.0) + + def test_seq_combo_with_goodput_slo(self): + slo = GoodputSlo(ttft_ms=1000.0, tpot_ms=50.0, e2el_ms=10000.0) + combo = SeqCombo(name="a", isl="128", osl="2048", goodput_slo=slo) + self.assertIsNotNone(combo.goodput_slo) + self.assertEqual(combo.goodput_slo.e2el_ms, 10000.0) + + def test_seq_combo_without_goodput_slo(self): + combo = SeqCombo(name="a", isl="128", osl="2048") + self.assertIsNone(combo.goodput_slo) + + +class TestSeqComboForbid(unittest.TestCase): + """SeqCombo is _Forbid: missing required fields and extra keys must raise.""" + + def test_missing_required_field_raises(self): + for missing in ("name", "isl", "osl"): + with self.subTest(missing=missing): + kwargs = {"name": "a", "isl": "128", "osl": "2048"} + del kwargs[missing] + with self.assertRaises(ValidationError): + SeqCombo(**kwargs) + + def test_extra_key_raises(self): + with self.assertRaises(ValidationError): + SeqCombo(name="a", isl="128", osl="2048", unknown_field="x") + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/inference/unittests/test_vllm_job_ray_backend.py b/cvs/lib/inference/unittests/test_vllm_job_ray_backend.py new file mode 100644 index 000000000..313c57865 --- /dev/null +++ b/cvs/lib/inference/unittests/test_vllm_job_ray_backend.py @@ -0,0 +1,1354 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for the Ray distributed-executor-backend support added to +cvs.lib.inference.vllm_job.VllmJob and +cvs.lib.inference.utils.vllm_config_loader.VariantConfig. + +Impl-blind / spec-derived (greenfield): these tests are written from the +behavioral spec before the implementation exists and are committed RED. A +different agent makes them green and may NOT edit this file. + +Coverage map (spec AC -> test): + Config validator relaxation ...... AC1-6 -> TestVariantConfigRayConsistency + cell_key ray multi-node .......... AC7 -> TestCellKeyRayMultiNode + _is_ray_backend .................. AC8 -> TestIsRayBackend + _server_argv ray vs mp ........... AC16,17 + RC1,RC3 -> TestServerArgvRayVsMp + start_server bootstrap/order ..... AC9-15,26,27 -> TestStartServerRayBootstrap + stop_server teardown ............. AC18-21 -> TestStopServerRayTeardown + _check_early_failure ray skip .... AC22,23 -> TestCheckEarlyFailureRayWorkerSkip + server_signature ................. AC24,25 -> TestServerSignatureRay + lifecycle (transition table) ..... regr -> TestVllmJobRayLifecycle + +Coverage-gap additions (post-review, impl-blind against the same spec): + ray pp>1 keeps --pipeline-parallel-size ......... TestServerArgvRayVsMp + serve-launch EARLY_FAILURE (ray head, mp worker) TestStartServerRayBootstrap + bootstrap OR-branch matrix (bad-out head, exit!=0 worker) TestStartServerRayBootstrap + re-entrant start_server ......................... TestVllmJobRayLifecycle + node-rank strip strengthened (mp, non-vacuous) .. TestServerSignatureRay + worker master_addr regression (distinct from hosts[0]) TestStartServerRayBootstrap + +Round-2 coverage-gap additions (impl-blind against the same spec): + 6 empty/None exec silent-success guard ......... TestStartServerRayBootstrap + 7 nnodes=3 worker loop (all bootstrap; fail last) TestStartServerRayBootstrap + 8 ray+pp>1 through the REAL VariantConfig ....... TestVariantConfigRayConsistency + 9 stop-after-failed-start asserts teardown calls TestVllmJobRayLifecycle + 10 FATAL_LOG_RE grep -> "vllm server fatal error" TestCheckEarlyFailureRayWorkerSkip + +Post-review round additions (impl-blind against the same spec): + R1.1 bootstrap-fail return omitting 'output' key . TestStartServerRayBootstrap + R1.2 server_signature env tuple independent oracle TestServerSignatureRay + R2.1 mp dist-block flag VALUES (not just presence) TestServerArgvRayVsMp + R2.2 is_ready True/False/empty + multinode skip .. TestVllmJobIsReady + R2.3 parse_results empty/unparseable/delegation .. TestVllmJobParseResults + +Round-3 coverage-gap additions (impl-blind against the same spec): + R3.1 parse_results asserts RETURN value (not just delegation) TestVllmJobParseResults + R3.2 wait_ready poll state machine (return/timeout/order) .. TestVllmJobWaitReady + R3.3 build_server_cmd env-script + per-rank mkdir branches .. TestVllmJobBuildServerCmd + +Round-4 coverage-gap additions (impl-blind against the same spec): + R4.1 server_env pass-through pins KEY=VALUE (non-colliding) TestVllmJobBuildServerCmd + R4.2 ray server_signature invariant to nnodes (2 vs 3) .... TestServerSignatureRay + R4.3 cell_key pp>1 branch (PP= segment) + pp==1, subTest ... TestCellKeyRayMultiNode + R4.4 _flatten_serve_args list/tuple repeat branch ......... TestFlattenServeArgsBranches +''' + +import unittest +import unittest.mock as mock +from types import SimpleNamespace + +from pydantic import ValidationError + +from cvs.lib.inference.utils.vllm_config_loader import VariantConfig +from cvs.lib.inference.vllm_job import VllmJob + +RAY = {"distributed-executor-backend": "ray"} + +# EARLY_FAILURE_RE-matching / non-matching bootstrap outputs (spec Failure Modes). +_CLEAN = "Local node IP: 10.0.0.1" # confirmed NON-matching +_BAD = "command not found" # confirmed matching + + +# --------------------------------------------------------------------------- # +# Fakes / fixtures (per spec "Test fake orchestrator contract") +# --------------------------------------------------------------------------- # +class RecordingOrch: + """Records (cmd, hosts) per exec call so host-targeting and ordering ACs + are checkable. `responder(cmd, hosts, detailed) -> dict` controls returns.""" + + hosts = ["10.0.0.1", "10.0.0.2"] # index 0 = head/rank0, 1 = worker/rank1 + + def __init__(self, responder=None, hosts=None, head_responder=None): + self.calls = [] # list of (cmd, hosts) in call order + self.head_cmds = [] + self._responder = responder + # head_responder(cmd) -> return value for exec_on_head; None preserves the + # legacy {} return so existing tests that never inspect exec_on_head output + # are unaffected. Only parse_results (which fetches via exec_on_head) needs it. + self._head_responder = head_responder + if hosts is not None: + self.hosts = list(hosts) + + def exec(self, cmd, hosts=None, detailed=False, **k): + self.calls.append((cmd, hosts)) + if self._responder is not None: + return self._responder(cmd, hosts, detailed) + return {} + + def exec_on_head(self, cmd, *a, **k): + self.head_cmds.append(cmd) + if self._head_responder is not None: + return self._head_responder(cmd) + return {} + + +HEAD = RecordingOrch.hosts[0] +WORKER = RecordingOrch.hosts[1] +HOST2 = "10.0.0.3" # third host for nnodes=3 worker-loop coverage (not in default hosts) + + +def _responder_ok(): + """Every bootstrap succeeds (exit 0, clean output); serve launch is clean. + + A detailed `grep` (the _check_early_failure FATAL scan) returns exit_code 1 + = "no fatal pattern found"; every other detailed call (ray bootstrap) returns + exit_code 0 = success. Non-detailed calls (serve launch / tail) return clean + text that does not match EARLY_FAILURE_RE. + """ + + def r(cmd, hosts, detailed): + host = hosts[0] if hosts else HEAD + if detailed: + exit_code = 1 if "grep" in cmd else 0 + return {host: {"exit_code": exit_code, "output": _CLEAN, "stdout": ""}} + return {host: ""} + + return r + + +def _responder_bootstrap_fail(fail_map): + """fail_map: host -> detailed return dict for that host's bootstrap; all + other hosts succeed, serve launches are clean.""" + + def r(cmd, hosts, detailed): + host = hosts[0] if hosts else HEAD + if detailed: + return {host: fail_map.get(host, {"exit_code": 0, "output": _CLEAN, "stdout": _CLEAN})} + return {host: ""} + + return r + + +def _responder_serve_fail(bad_serve_hosts): + """Ray/mp bootstrap detailed calls all succeed (exit 0, clean output); the + NON-detailed `vllm serve` launch returns EARLY_FAILURE_RE-matching output for + hosts in `bad_serve_hosts`, clean otherwise. + + Exercises the post-bootstrap serve-launch EARLY_FAILURE check (the + "vllm server failed to launch on ... (rank N)" RuntimeError site), which is + distinct from the bootstrap failure sites covered by _responder_bootstrap_fail. + """ + + def r(cmd, hosts, detailed): + host = hosts[0] if hosts else HEAD + if detailed: + # No grep is issued by start_server; bootstrap detailed calls succeed. + exit_code = 1 if "grep" in cmd else 0 + return {host: {"exit_code": exit_code, "output": _CLEAN, "stdout": ""}} + if "vllm serve" in cmd and host in bad_serve_hosts: + return {host: _BAD} + return {host: ""} + + return r + + +def _responder_const(value): + """Every orch.exec call (bootstrap detailed AND serve non-detailed) returns + the SAME constant `value` -- used to drive the empty/None silent-success guard + (`(out or {}).items()`) in _bootstrap_ray_cluster and start_server. With + value={} or value=None the guard's iterable is empty, so no per-host failure + check runs and no false positive is raised.""" + + def r(cmd, hosts, detailed): + return value + + return r + + +# FATAL_LOG_RE-matching text confirmed by the class regex (see stub FATAL_LOG_RE: +# "...|Engine core initialization failed|..."). Distinct from EARLY_FAILURE_RE. +_FATAL = "Engine core initialization failed" + + +def _responder_fatal_grep(fatal_hosts, fatal_text=_FATAL): + """Detailed `grep` (the _check_early_failure FATAL_LOG_RE scan) returns + exit_code 0 (match found) with FATAL-matching `stdout` for hosts in + `fatal_hosts`; every other detailed call (and every host's grep otherwise) + returns exit_code 1 = no match. Non-detailed `tail` returns clean text that + does NOT match EARLY_FAILURE_RE, so the FATAL_LOG_RE branch -- not the tail + EARLY_FAILURE branch -- is the one that fires.""" + + def r(cmd, hosts, detailed): + host = hosts[0] if hosts else HEAD + if detailed: + if "grep" in cmd and host in fatal_hosts: + return {host: {"exit_code": 0, "stdout": fatal_text, "output": fatal_text}} + return {host: {"exit_code": 1, "stdout": "", "output": _CLEAN}} + return {host: ""} + + return r + + +def _responder_readiness(exit_code=0, empty=False): + """is_ready() greps each non-skipped rank's readiness log via + orch.exec(detailed=True) and returns {host: {"exit_code": ...}}; exit_code 0 + means the readiness pattern was found (server ready). empty=True returns {} + to exercise the `not out` (empty result) False path. Non-detailed calls + return clean text (unused by is_ready).""" + + def r(cmd, hosts, detailed): + if empty: + return {} + host = hosts[0] if hosts else HEAD + if detailed: + return {host: {"exit_code": exit_code, "output": "", "stdout": ""}} + return {host: ""} + + return r + + +def _variant(serve_args=None, nnodes="2", pp="2", ib_netdev="enp159s0np0", tp="8", master_addr="10.0.0.1", env=None): + """Minimal SimpleNamespace variant mirroring _variant() in the reuse suite.""" + params = SimpleNamespace( + tensor_parallelism=tp, + pipeline_parallel_size=pp, + master_addr=master_addr, + master_port="29501", + nnodes=nnodes, + port_no="8000", + random_range_ratio="0.0", + random_prefix_len="0", + burstiness="1.0", + seed="0", + request_rate="inf", + tokenizer_mode="auto", + percentile_metrics="ttft,tpot,itl,e2el", + metric_percentiles="50,90,95,99", + base_url="http://0.0.0.0", + dataset_name="random", + backend="vllm", + ) + return SimpleNamespace( + params=params, + model=SimpleNamespace(id="/models/test-model"), + paths=SimpleNamespace(log_dir="/logs", models_dir="/models"), + roles=SimpleNamespace( + server=SimpleNamespace(serve_args=dict(serve_args or {}), env=dict(env or {}), ib_netdev=ib_netdev) + ), + ) + + +def _job( + orch=None, + serve_args=None, + nnodes="2", + pp="2", + ib_netdev="enp159s0np0", + concurrency=16, + isl="1024", + osl="1024", + tp="8", + master_addr="10.0.0.1", + env=None, + ib_hcas=None, +): + orch = RecordingOrch() if orch is None else orch + return VllmJob( + orch=orch, + variant=_variant(serve_args, nnodes, pp, ib_netdev, tp, master_addr, env), + hf_token="tok", + isl=isl, + osl=osl, + concurrency=concurrency, + num_prompts="640", + ib_hcas=ib_hcas, + ) + + +def _vc(nnodes="2", pp="1", serve_args=None, ib_netdev="eth0", tp="8"): + """A real pydantic VariantConfig exercising _check_distributed_consistency. + + enforce_thresholds=False so the (independent) threshold-coverage validator + only warns and never masks the distributed-consistency error under test. + """ + return VariantConfig( + schema_version=1, + framework="vllm", + gpu_arch="mi300x", + enforce_thresholds=False, + paths={ + "shared_fs": "/home/x", + "models_dir": "/home/x/models", + "log_dir": "/home/x/LOGS", + "hf_token_file": "/home/x/.hf", + }, + model={"id": "/models/test-model", "remote": 0}, + params={"tensor_parallelism": tp, "pipeline_parallel_size": pp, "nnodes": nnodes}, + roles={"server": {"serve_args": dict(serve_args or {}), "env": {}, "ib_netdev": ib_netdev}}, + sweep={ + "sequence_combinations": [{"name": "a", "isl": "1024", "osl": "1024"}], + "runs": [{"combo": "a", "concurrency": 16}], + }, + thresholds={}, + ) + + +# --------------------------------------------------------------------------- # +# helpers for argv / call inspection +# --------------------------------------------------------------------------- # +def _value_after(argv, flag): + """Return the element following `flag` in argv, or None if flag absent.""" + for i, a in enumerate(argv): + if a == flag: + return argv[i + 1] if i + 1 < len(argv) else None + return None + + +def _calls_to(orch, host): + return [cmd for cmd, hosts in orch.calls if hosts == [host]] + + +def _first_index(orch, predicate): + for i, (cmd, hosts) in enumerate(orch.calls): + if predicate(cmd, hosts): + return i + return -1 + + +def _all_cmds(orch): + return [c for c, _ in orch.calls] + list(orch.head_cmds) + + +# --------------------------------------------------------------------------- # +# Config validator: VariantConfig._check_distributed_consistency (AC1-6) +# --------------------------------------------------------------------------- # +class TestVariantConfigRayConsistency(unittest.TestCase): + """The ray relaxation applies ONLY to the (nn>1 & pp==1) rule and ONLY for + the exact string 'ray'. ib_netdev and the (pp>1 & nn==1) rule are untouched.""" + + def test_accepts_valid(self): + # (nnodes, pp, serve_args, ib_netdev) that must construct without error. + cases = [ + ("1", "1", {}, None), # baseline: unrelaxed single-node default path + ("2", "1", RAY, "eth0"), # AC1: ray relaxation permits nn>1 & pp==1 + ("2", "2", {}, "eth0"), # AC3: mp multi-node path unchanged + ("2", "2", RAY, "eth0"), # finding 8: ray + pp>1 is legal (nn>1 & pp>1 + # is valid for ANY backend; the ray relaxation only special-cases pp==1, + # it never REJECTS ray+pp>1). Validated through the REAL VariantConfig + # validator, not just the SimpleNamespace fake used by _server_argv tests. + ] + for nn, pp, sa, ib in cases: + with self.subTest(nnodes=nn, pp=pp, serve_args=sa): + try: + _vc(nnodes=nn, pp=pp, serve_args=sa, ib_netdev=ib) + except ValidationError as e: # pragma: no cover - failure path + self.fail(f"unexpected ValidationError: {e}") + + def test_rejects_invalid(self): + # (nnodes, pp, serve_args, ib_netdev, field-token-in-message) + cases = [ + ("2", "1", {}, "eth0", "pipeline_parallel_size"), # AC2 no ray key + ("1", "2", RAY, "eth0", "pipeline_parallel_size"), # AC4 pp>1 & nn==1 never relaxed + ("2", "1", RAY, None, "ib_netdev"), # AC5 ib_netdev not relaxed by ray + ("2", "1", {"distributed-executor-backend": "RAY"}, "eth0", "pipeline_parallel_size"), # AC6 case-sensitive + ("2", "1", {"distributed-executor-backend": "Ray"}, "eth0", "pipeline_parallel_size"), # AC6 case-sensitive + ] + for nn, pp, sa, ib, token in cases: + with self.subTest(nnodes=nn, pp=pp, serve_args=sa, ib_netdev=ib): + with self.assertRaises(ValidationError) as ctx: + _vc(nnodes=nn, pp=pp, serve_args=sa, ib_netdev=ib) + self.assertIn(token, str(ctx.exception)) + + +class TestCellKeyRayMultiNode(unittest.TestCase): + """AC7: ray multi-node has pp=1, so cell_key uses the single-node format + (no PP= segment), identical to a genuine single-node cell. + + Round-4 finding 3: cell_key has two branches -- pp==1 (no PP= segment) and + pp>1 (a "PP=," segment inserted before CONC). The pp>1 branch had zero + coverage anywhere for THIS VariantConfig class, so both branches are now + pinned together in one subTest table (discipline rule B), asserting the exact + segment position/value/comma placement, not just presence.""" + + def test_cell_key_format_both_pp_branches(self): + # (nnodes, pp, serve_args, ib_netdev, expected_key) + cases = [ + # AC7: ray multi-node, pp==1 -> single-node format, NO PP= segment. + ("2", "1", RAY, "eth0", "ISL=1024,OSL=1024,TP=8,CONC=16"), + # pp>1 branch -> "PP=2," inserted immediately before CONC. pp>1 requires + # nnodes>1 (the pp>1 & nn==1 rule always fires), so this is a valid mp + # multi-node config; the PP segment is what distinguishes it from the + # pp==1 key above. + ("2", "2", {}, "eth0", "ISL=1024,OSL=1024,TP=8,PP=2,CONC=16"), + ] + for nn, pp, sa, ib, expected in cases: + with self.subTest(nnodes=nn, pp=pp): + vc = _vc(nnodes=nn, pp=pp, serve_args=sa, ib_netdev=ib, tp="8") + self.assertEqual(vc.cell_key(isl="1024", osl="1024", concurrency="16"), expected) + + +# --------------------------------------------------------------------------- # +# _is_ray_backend (AC8) +# --------------------------------------------------------------------------- # +class TestIsRayBackend(unittest.TestCase): + def test_backend_detection_is_exact_string(self): + # (serve_args, expected) + cases = [ + ({"distributed-executor-backend": "ray"}, True), + ({}, False), + ({"distributed-executor-backend": "mp"}, False), + ({"distributed-executor-backend": "RAY"}, False), + ({"distributed-executor-backend": "Ray"}, False), + ] + for sa, expected in cases: + with self.subTest(serve_args=sa): + job = _job(serve_args=sa, nnodes="1", pp="1") + self.assertIs(job._is_ray_backend, expected) + + def test_property_reflects_live_serve_args_not_a_cached_snapshot(self): + job = _job(serve_args={}, nnodes="1", pp="1") + self.assertIs(job._is_ray_backend, False) + job.serve_args["distributed-executor-backend"] = "ray" + self.assertIs(job._is_ray_backend, True) + + +# --------------------------------------------------------------------------- # +# _server_argv (AC16, AC17, RC1, RC3) +# --------------------------------------------------------------------------- # +class TestServerArgvRayVsMp(unittest.TestCase): + _DRIVER_DIST_FLAGS = [ + "--node-rank", + "--headless", + "--pipeline-parallel-size", + "--master-addr", + "--master-port", + "--nnodes", + ] + + def test_ray_multinode_omits_all_driver_dist_flags(self): + # AC16: the mp block is skipped entirely under ray. + argv = _job(serve_args=RAY, nnodes="2", pp="1")._server_argv(0) + for flag in self._DRIVER_DIST_FLAGS: + with self.subTest(flag=flag): + self.assertNotIn(flag, argv) + + def test_ray_multinode_backend_arrives_via_serve_args(self): + # AC17: --distributed-executor-backend ray comes from _flatten_serve_args. + argv = _job(serve_args=RAY, nnodes="2", pp="1")._server_argv(0) + self.assertIn("--distributed-executor-backend", argv) + self.assertEqual(_value_after(argv, "--distributed-executor-backend"), "ray") + + def test_mp_multinode_injects_full_dist_block(self): + # RC1: mp multi-node keeps the driver-injected block + hardcoded mp backend. + # Round-2 finding 1: assert the VALUE after each flag, not just presence. A + # mutant that emits the right flag names but wrong/hardcoded values -- e.g. + # swapping master_addr/master_port, hardcoding --nnodes 1, or dropping the + # pipeline width -- would pass a presence-only check while breaking the + # launch. Each expected value is pinned to the job's real attribute (read + # from variant.params, not re-read from the produced argv), so the check is + # independent of the argv it is validating. + job = _job(serve_args={}, nnodes="2", pp="2") + argv = job._server_argv(0) + expected = [ + ("--node-rank", "0"), # the rank argument passed to _server_argv(0) + ("--master-addr", job.master_addr), + ("--master-port", job.master_port), + ("--nnodes", job.nnodes), + ("--pipeline-parallel-size", job.pp), + ("--distributed-executor-backend", "mp"), # hardcoded on the mp path + ] + for flag, val in expected: + with self.subTest(flag=flag): + self.assertIn(flag, argv) + self.assertEqual(_value_after(argv, flag), val) + + def test_mp_worker_rank_is_headless(self): + # RC1: rank>0 mp worker additionally carries --headless; rank 0 does not. + # Round-2 finding 1: also pin --node-rank's VALUE to the actual rank arg, so + # a mutant that always emits "--node-rank 0" regardless of rank (breaking + # multi-node distribution) is caught -- not merely flag/--headless presence. + job = _job(serve_args={}, nnodes="2", pp="2") + argv0 = job._server_argv(0) + argv1 = job._server_argv(1) + self.assertNotIn("--headless", argv0) + self.assertIn("--headless", argv1) + self.assertEqual(_value_after(argv0, "--node-rank"), "0") + self.assertEqual(_value_after(argv1, "--node-rank"), "1") + + def test_single_node_ray_passthrough_no_driver_flags(self): + # RC3 / Edge: single-node omits all driver-injected dist flags, but the + # user's serve_args backend still passes through verbatim. + argv = _job(serve_args=RAY, nnodes="1", pp="1")._server_argv(0) + for flag in self._DRIVER_DIST_FLAGS: + with self.subTest(flag=flag): + self.assertNotIn(flag, argv) + self.assertEqual(_value_after(argv, "--distributed-executor-backend"), "ray") + + def test_ray_multinode_pp_gt_1_keeps_pipeline_parallel_size(self): + # Coverage-gap (finding 1): VariantConfig permits a ray backend with + # nnodes>1 AND pp>1 (the ray relaxation only special-cases pp==1; the + # nn>1 & pp>1 combo is legal for any backend). The mp block that normally + # carries "--pipeline-parallel-size" is skipped for every ray job, so a + # ray+pp=2 config must NOT silently drop the pipeline-parallel width: the + # head's single `vllm serve` still has to be told pp=2 (via the flag with + # value self.pp) or the cluster silently runs at pp=1. A mutant that drops + # the flag under ray (the current guard `nnodes>1 and not _is_ray_backend`) + # is caught here. + argv = _job(serve_args=RAY, nnodes="2", pp="2")._server_argv(0) + self.assertIn("--pipeline-parallel-size", argv) + self.assertEqual(_value_after(argv, "--pipeline-parallel-size"), "2") + # Backend is still ray (contributed by serve_args passthrough), not mp. + self.assertEqual(_value_after(argv, "--distributed-executor-backend"), "ray") + # Ray still manages rendezvous, so the torchrun-style mp flags stay absent + # even though pp>1 (ray does not use --node-rank/--master-*/--nnodes/--headless). + for flag in ("--node-rank", "--headless", "--master-addr", "--master-port", "--nnodes"): + with self.subTest(flag=flag): + self.assertNotIn(flag, argv) + + +# --------------------------------------------------------------------------- # +# _flatten_serve_args: the four equivalence classes (RC8) +# --------------------------------------------------------------------------- # +class TestFlattenServeArgsBranches(unittest.TestCase): + """RC8: _flatten_serve_args has four branches -- True (bare flag), False + (omitted), list/tuple (flag repeated per element), scalar (flag + str(value)). + True/False/scalar are covered in test_vllm_job_server_reuse.py; the list/tuple + repeat branch (Round-4 finding 4) is covered here so all four cells of this + pure function's table are pinned (discipline rule B). A bug in the repeat branch + (wrong flag repeated, values not str()-cast, wrong order) is caught.""" + + def test_list_and_tuple_values_repeat_the_flag_per_element(self): + # (value, expected) -- list and tuple both repeat "--" before each + # element, in order; non-string elements are str()-cast. + cases = [ + (["a.b.C", "d.e.F"], ["--middleware", "a.b.C", "--middleware", "d.e.F"]), + (("a.b.C", "d.e.F"), ["--middleware", "a.b.C", "--middleware", "d.e.F"]), + ([1, 2], ["--middleware", "1", "--middleware", "2"]), # str()-cast + ] + for value, expected in cases: + with self.subTest(value=value): + self.assertEqual(VllmJob._flatten_serve_args({"middleware": value}), expected) + + +# --------------------------------------------------------------------------- # +# start_server: ray bootstrap ordering, host targeting, failure (AC9-15,26,27) +# --------------------------------------------------------------------------- # +class TestStartServerRayBootstrap(unittest.TestCase): + def test_head_bootstrap_command_and_target(self): + # AC9 + orch = RecordingOrch(responder=_responder_ok()) + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1").start_server() + head_ray = [c for c in _calls_to(orch, HEAD) if "ray start" in c] + self.assertTrue(head_ray, "expected a ray start command targeting the head") + cmd = head_ray[0] + for token in ("ray start", "--head", "--port=29501"): + with self.subTest(token=token): + self.assertIn(token, cmd) + + def test_worker_bootstrap_command_and_target(self): + # AC10 + orch = RecordingOrch(responder=_responder_ok()) + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1").start_server() + worker_ray = [c for c in _calls_to(orch, WORKER) if "ray start" in c] + self.assertTrue(worker_ray, "expected a ray start command targeting the worker") + cmd = worker_ray[0] + self.assertIn("ray start", cmd) + self.assertIn("--address=10.0.0.1:29501", cmd) + + def test_worker_bootstrap_targets_master_addr_not_head_host(self): + # AC10 disambiguation / REGRESSION: the worker's Ray rendezvous --address + # must be self.master_addr (the data-plane IP the head actually started + # with via `ray start --head --port=...`), NOT self.orch.hosts[0] (the + # SSH/management host). The default fixture sets master_addr == hosts[0] + # ("10.0.0.1"), so the plain AC10 test above passes regardless of which + # field the impl uses. Here master_addr is DISTINCT from hosts[0] + # (hosts=["10.0.0.1","10.0.0.2"], master_addr="172.16.0.1"), so only an + # impl that targets master_addr passes; one that targets hosts[0] fails. + orch = RecordingOrch(responder=_responder_ok()) # hosts[0]=HEAD=10.0.0.1 + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1", master_addr="172.16.0.1").start_server() + worker_ray = [c for c in _calls_to(orch, WORKER) if "ray start" in c] + self.assertTrue(worker_ray, "expected a ray start command targeting the worker") + cmd = worker_ray[0] + self.assertIn( + "--address=172.16.0.1:29501", + cmd, + "worker rendezvous must target master_addr (data-plane IP), not hosts[0]", + ) + self.assertNotIn( + "--address=10.0.0.1:29501", + cmd, + "worker must NOT rendezvous against the SSH/management host hosts[0]", + ) + + def test_bootstrap_precedes_serve_launch(self): + # AC11: every ray start bootstrap (head AND worker) precedes the vllm serve launch. + orch = RecordingOrch(responder=_responder_ok()) + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1").start_server() + first_head_bootstrap = _first_index(orch, lambda c, h: "ray start" in c and h == [HEAD]) + first_worker_bootstrap = _first_index(orch, lambda c, h: "ray start" in c and h == [WORKER]) + first_serve = _first_index(orch, lambda c, h: "vllm serve" in c) + self.assertNotEqual(first_head_bootstrap, -1, "no head ray start call recorded") + self.assertNotEqual(first_worker_bootstrap, -1, "no worker ray start call recorded") + self.assertNotEqual(first_serve, -1, "no vllm serve call recorded") + self.assertLess(first_head_bootstrap, first_serve) + self.assertLess(first_worker_bootstrap, first_serve) + + def test_no_serve_on_worker_under_ray(self): + # AC12: vllm serve runs only on the head under ray. + orch = RecordingOrch(responder=_responder_ok()) + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1").start_server() + self.assertEqual([c for c in _calls_to(orch, WORKER) if "vllm serve" in c], []) + self.assertTrue([c for c in _calls_to(orch, HEAD) if "vllm serve" in c]) + + def test_mp_serves_every_host_no_ray_start(self): + # AC13: mp multi-node serves on every host (incl. worker), no ray start. + orch = RecordingOrch(responder=_responder_ok()) + _job(orch=orch, serve_args={}, nnodes="2", pp="2").start_server() + self.assertTrue([c for c in _calls_to(orch, HEAD) if "vllm serve" in c]) + self.assertTrue([c for c in _calls_to(orch, WORKER) if "vllm serve" in c]) + self.assertEqual([c for c, _ in orch.calls if "ray start" in c], []) + + def test_single_node_ray_no_bootstrap(self): + # AC14: 1-node ray issues no ray start and exactly one vllm serve launch. + orch = RecordingOrch(responder=_responder_ok(), hosts=[HEAD]) + _job(orch=orch, serve_args=RAY, nnodes="1", pp="1", ib_netdev=None).start_server() + self.assertEqual([c for c in _all_cmds(orch) if "ray start" in c], []) + serves = [c for c in _all_cmds(orch) if "vllm serve" in c] + self.assertEqual(len(serves), 1, f"expected exactly one serve launch, got {serves}") + + def test_happy_path_launches_serve_on_head(self): + # AC15: clean bootstrap -> no exception + serve on head via exec(hosts=[head]). + orch = RecordingOrch(responder=_responder_ok()) + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1").start_server() + self.assertTrue([c for c in _calls_to(orch, HEAD) if "vllm serve" in c]) + + def test_head_bootstrap_failure_aborts_before_serve(self): + # AC26: head exit_code!=0 -> RuntimeError(rank 0), no serve, no worker bootstrap. + orch = RecordingOrch( + responder=_responder_bootstrap_fail({HEAD: {"exit_code": 1, "output": "something went wrong"}}) + ) + with self.assertRaises(RuntimeError) as ctx: + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1").start_server() + msg = str(ctx.exception) + self.assertIn("ray bootstrap failed on", msg) + self.assertIn("rank 0", msg) + self.assertEqual([c for c, _ in orch.calls if "vllm serve" in c], []) + self.assertEqual([c for c in _calls_to(orch, WORKER) if "ray start" in c], []) + + def test_worker_bootstrap_failure_bad_output(self): + # AC27: head ok, worker exit 0 but output matches EARLY_FAILURE_RE -> rank 1. + orch = RecordingOrch(responder=_responder_bootstrap_fail({WORKER: {"exit_code": 0, "output": _BAD}})) + with self.assertRaises(RuntimeError) as ctx: + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1").start_server() + msg = str(ctx.exception) + self.assertIn("ray bootstrap failed on", msg) + self.assertIn("rank 1", msg) + self.assertEqual([c for c, _ in orch.calls if "vllm serve" in c], []) + + # ---- Coverage-gap (finding 5): the bootstrap failure check is an OR of + # (exit_code != 0) OR (EARLY_FAILURE_RE matches output), applied to BOTH head + # and worker. Existing tests cover only exit!=0-on-head and bad-output-on-worker; + # the two mirror combinations below (bad-output-on-head, exit!=0-on-worker) close + # the OR-branch matrix so a mutant dropping either half on either host is killed. + def test_head_bootstrap_failure_bad_output_exit0(self): + # head: exit_code 0 but output matches EARLY_FAILURE_RE -> RuntimeError rank 0, + # aborting before any worker bootstrap and before serve launch. + orch = RecordingOrch(responder=_responder_bootstrap_fail({HEAD: {"exit_code": 0, "output": _BAD}})) + with self.assertRaises(RuntimeError) as ctx: + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1").start_server() + msg = str(ctx.exception) + self.assertIn("ray bootstrap failed on", msg) + self.assertIn("rank 0", msg) + self.assertEqual([c for c in _calls_to(orch, WORKER) if "ray start" in c], []) + self.assertEqual([c for c, _ in orch.calls if "vllm serve" in c], []) + + def test_worker_bootstrap_failure_nonzero_exit_clean_output(self): + # worker: exit_code != 0 with CLEAN (non-EARLY_FAILURE) output -> RuntimeError + # rank 1. The head bootstrap succeeded, so the failure is attributed to rank 1. + orch = RecordingOrch(responder=_responder_bootstrap_fail({WORKER: {"exit_code": 1, "output": _CLEAN}})) + with self.assertRaises(RuntimeError) as ctx: + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1").start_server() + msg = str(ctx.exception) + self.assertIn("ray bootstrap failed on", msg) + self.assertIn("rank 1", msg) + + # ---- Coverage-gap (Round-1 finding 1): the bootstrap-failure detailed return + # dict may OMIT the content key entirely. The spec's Failure Modes section is + # explicit: the check reads r.get("output", ""), so a return dict missing the + # content key is treated as empty string -- "no KeyError, no false positive". + # Every other responder in this file always supplies an "output" key, so a + # regression that read r["output"] (KeyError on a real orchestrator response + # missing that key) would slip through. Here the failing head returns a dict + # with exit_code=1 and NO "output" key at all: start_server() must still raise + # the normal RuntimeError naming rank 0 (the empty-output path), NOT a KeyError. + # assertRaises(RuntimeError) does not catch KeyError, so an r["output"] mutant + # surfaces as a test error/failure rather than a false pass. + def test_head_bootstrap_failure_output_key_absent_no_keyerror(self): + orch = RecordingOrch(responder=_responder_bootstrap_fail({HEAD: {"exit_code": 1}})) + try: + with self.assertRaises(RuntimeError) as ctx: + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1").start_server() + except KeyError as e: # pragma: no cover - regression guard + self.fail(f"missing 'output' key must be treated as empty string, not raise KeyError: {e!r}") + msg = str(ctx.exception) + self.assertIn("ray bootstrap failed on", msg) + self.assertIn("rank 0", msg) + # Aborts before any serve launch, exactly like the with-output failure path. + self.assertEqual([c for c, _ in orch.calls if "vllm serve" in c], []) + self.assertEqual([c for c, _ in orch.calls if "vllm serve" in c], []) + + # ---- Coverage-gap (finding 4): the post-bootstrap `vllm serve` launch has its + # own EARLY_FAILURE_RE check (RuntimeError "vllm server failed to launch on ... + # (rank N)"), distinct from the bootstrap checks above. No existing test returns + # EARLY_FAILURE output for the non-detailed serve launch, so these two sites -- + # the ray head launch and the mp non-head-rank launch -- were never exercised. + def test_ray_head_serve_launch_failure_raises_rank0(self): + # ray path: bootstrap (head + worker) succeeds, but the head's post-bootstrap + # vllm serve launch output matches EARLY_FAILURE_RE -> RuntimeError rank 0. + orch = RecordingOrch(responder=_responder_serve_fail({HEAD})) + with self.assertRaises(RuntimeError) as ctx: + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1").start_server() + msg = str(ctx.exception) + self.assertIn("vllm server failed to launch on", msg) + self.assertIn("rank 0", msg) + + def test_mp_worker_serve_launch_failure_raises_rank1(self): + # mp path (else branch): the non-head-rank (rank 1) vllm serve launch output + # matches EARLY_FAILURE_RE -> RuntimeError rank 1. Confirms the serve-launch + # failure check fires for a worker on the mp path, not just the head. + orch = RecordingOrch(responder=_responder_serve_fail({WORKER})) + with self.assertRaises(RuntimeError) as ctx: + _job(orch=orch, serve_args={}, nnodes="2", pp="2").start_server() + msg = str(ctx.exception) + self.assertIn("vllm server failed to launch on", msg) + self.assertIn("rank 1", msg) + + # ---- Coverage-gap (finding 6): the per-host failure check iterates + # `(out or {}).items()` in BOTH _bootstrap_ray_cluster and the serve-launch + # scan in start_server. When orch.exec returns {} or None (an empty/omitted + # result -- which the real orchestrator can produce, and which the spec says + # must be treated as empty output "no KeyError, no false positive"), the + # iterable is empty, no host entry is examined, and the code proceeds as a + # silent success. No prior responder ever returned {}/None, so this guard was + # never exercised. Intended behavior: start_server does NOT raise and the ray + # start + vllm serve calls are still issued (returns are recorded regardless). + def test_empty_or_none_bootstrap_result_is_silent_success(self): + for value in ({}, None): + with self.subTest(exec_return=value): + orch = RecordingOrch(responder=_responder_const(value)) + try: + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1").start_server() + except Exception as e: # pragma: no cover - failure path + self.fail(f"empty/None exec return must be silent-success, raised: {e!r}") + # The calls were still dispatched (their empty returns just yield no + # failure to detect): head+worker ray start and a head vllm serve. + self.assertTrue([c for c in _calls_to(orch, HEAD) if "ray start" in c]) + self.assertTrue([c for c in _calls_to(orch, WORKER) if "ray start" in c]) + self.assertTrue([c for c in _calls_to(orch, HEAD) if "vllm serve" in c]) + + # ---- Coverage-gap (finding 7): the worker bootstrap loop was only ever + # exercised with exactly ONE worker (nnodes=2). A loop that bootstraps only + # hosts[1] (off-by-one / no loop) would pass every nnodes=2 test. These two + # nnodes=3 cases pin the loop: (a) EVERY worker is bootstrapped on the happy + # path; (b) a failure injected on the LAST worker still aborts with the + # correct rank, proving the loop reaches it (no short-circuit after rank 1). + def test_three_node_ray_bootstraps_every_worker(self): + orch = RecordingOrch(responder=_responder_ok(), hosts=[HEAD, WORKER, HOST2]) + _job(orch=orch, serve_args=RAY, nnodes="3", pp="1").start_server() + # Both workers (rank 1 and rank 2) get a ray start; the head gets one too. + self.assertTrue([c for c in _calls_to(orch, HEAD) if "ray start" in c], "head ray start missing") + self.assertTrue([c for c in _calls_to(orch, WORKER) if "ray start" in c], "rank-1 worker ray start missing") + self.assertTrue([c for c in _calls_to(orch, HOST2) if "ray start" in c], "rank-2 worker ray start missing") + # Ray still serves only on the head; no serve on either worker. + self.assertTrue([c for c in _calls_to(orch, HEAD) if "vllm serve" in c]) + self.assertEqual([c for c in _calls_to(orch, WORKER) if "vllm serve" in c], []) + self.assertEqual([c for c in _calls_to(orch, HOST2) if "vllm serve" in c], []) + + def test_three_node_ray_failure_on_last_worker_aborts_rank2(self): + # Failure on the LAST worker (rank 2), not the first -- confirms the loop + # does not short-circuit at rank 1. The head and rank-1 worker bootstrap + # cleanly; rank-2 fails, so the RuntimeError names rank 2 and no serve runs. + orch = RecordingOrch( + responder=_responder_bootstrap_fail({HOST2: {"exit_code": 1, "output": _BAD}}), + hosts=[HEAD, WORKER, HOST2], + ) + with self.assertRaises(RuntimeError) as ctx: + _job(orch=orch, serve_args=RAY, nnodes="3", pp="1").start_server() + msg = str(ctx.exception) + self.assertIn("ray bootstrap failed on", msg) + self.assertIn("rank 2", msg) + # The loop DID reach the earlier ranks before failing at the last worker. + self.assertTrue([c for c in _calls_to(orch, HEAD) if "ray start" in c], "head must have bootstrapped") + self.assertTrue( + [c for c in _calls_to(orch, WORKER) if "ray start" in c], "rank-1 worker must have bootstrapped" + ) + # No serve launched anywhere after the abort. + self.assertEqual([c for c, _ in orch.calls if "vllm serve" in c], []) + + +# --------------------------------------------------------------------------- # +# stop_server: ray teardown (AC18-21) +# --------------------------------------------------------------------------- # +@mock.patch("cvs.lib.inference.vllm_job.time.sleep") +class TestStopServerRayTeardown(unittest.TestCase): + def test_ray_multinode_broadcasts_single_ray_stop(self, mock_sleep): + # AC18: exactly one broadcast (hosts=None) ray stop. + orch = RecordingOrch() + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1").stop_server() + ray_stops = [(c, h) for c, h in orch.calls if "ray stop" in c] + self.assertEqual(len(ray_stops), 1, f"expected one ray stop, got {ray_stops}") + self.assertIsNone(ray_stops[0][1], "ray stop must be broadcast (hosts=None)") + + def test_pkill_precedes_ray_stop(self, mock_sleep): + # AC19: pkill vllm serve broadcast comes before ray stop. + orch = RecordingOrch() + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1").stop_server() + pkill_idx = _first_index(orch, lambda c, h: "pkill" in c and "vllm serve" in c) + ray_idx = _first_index(orch, lambda c, h: "ray stop" in c) + self.assertNotEqual(pkill_idx, -1, "no pkill vllm serve call recorded") + self.assertNotEqual(ray_idx, -1, "no ray stop call recorded") + self.assertLess(pkill_idx, ray_idx) + + def test_mp_multinode_no_ray_stop(self, mock_sleep): + # AC20 / RC4 + orch = RecordingOrch() + _job(orch=orch, serve_args={}, nnodes="2", pp="2").stop_server() + self.assertEqual([c for c, _ in orch.calls if "ray stop" in c], []) + + def test_single_node_ray_no_ray_stop(self, mock_sleep): + # AC21 + orch = RecordingOrch(hosts=[HEAD]) + _job(orch=orch, serve_args=RAY, nnodes="1", pp="1", ib_netdev=None).stop_server() + self.assertEqual([c for c, _ in orch.calls if "ray stop" in c], []) + + +# --------------------------------------------------------------------------- # +# _check_early_failure: Ray worker skip (AC22, AC23) +# --------------------------------------------------------------------------- # +class TestCheckEarlyFailureRayWorkerSkip(unittest.TestCase): + def test_ray_worker_is_skipped(self): + # AC22: ray workers have no per-rank server log -> no tail/grep on worker. + orch = RecordingOrch(responder=_responder_ok()) + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1")._check_early_failure() + self.assertEqual(_calls_to(orch, WORKER), [], "ray worker must not be tailed/grepped") + self.assertTrue(_calls_to(orch, HEAD), "rank 0 head must still be checked") + + def test_mp_worker_is_checked(self): + # AC23: mp workers DO produce a per-rank log -> rank-1 worker is checked. + orch = RecordingOrch(responder=_responder_ok()) + _job(orch=orch, serve_args={}, nnodes="2", pp="2")._check_early_failure() + self.assertTrue(_calls_to(orch, WORKER), "mp rank-1 worker must be tailed/grepped") + + def test_fatal_log_match_raises_with_rank(self): + # Coverage-gap (finding 10): the FATAL_LOG_RE grep branch (detailed grep + # returns exit_code 0 with stdout matching FATAL_LOG_RE) raises a + # RuntimeError "vllm server fatal error". Every prior fixture returned + # exit_code 1 / no-match for the grep, so this RuntimeError site was never + # exercised. Single-host job so exactly rank 0 is inspected; the tail + # returns clean text so the EARLY_FAILURE_RE branch does NOT pre-empt the + # FATAL_LOG_RE branch under test. + orch = RecordingOrch(responder=_responder_fatal_grep({HEAD}), hosts=[HEAD]) + job = _job(orch=orch, serve_args={}, nnodes="1", pp="1", ib_netdev=None) + with self.assertRaises(RuntimeError) as ctx: + job._check_early_failure() + msg = str(ctx.exception) + self.assertIn("vllm server fatal error", msg) + self.assertIn("rank 0", msg) + + def test_fatal_log_match_on_mp_worker_reports_rank1(self): + # Companion to finding 10: the FATAL_LOG_RE match on an mp rank-1 worker + # (which IS inspected, unlike a ray worker) must attribute the fatal error + # to rank 1 -- confirming the rank is threaded into the message, not + # hard-coded to 0. Head grep is clean; only the worker's grep matches. + orch = RecordingOrch(responder=_responder_fatal_grep({WORKER})) + job = _job(orch=orch, serve_args={}, nnodes="2", pp="2") + with self.assertRaises(RuntimeError) as ctx: + job._check_early_failure() + msg = str(ctx.exception) + self.assertIn("vllm server fatal error", msg) + self.assertIn("rank 1", msg) + + +# --------------------------------------------------------------------------- # +# server_signature (AC24, AC25) +# --------------------------------------------------------------------------- # +class TestServerSignatureRay(unittest.TestCase): + def test_hashable_and_stable(self): + # AC24 + job = _job(serve_args=RAY, nnodes="2", pp="1") + sig = job.server_signature() + self.assertEqual(hash(sig), hash(job.server_signature())) + self.assertEqual(sig, job.server_signature()) + + def test_invariant_to_concurrency(self): + # AC25: concurrency is client-only; two ray jobs differing only in it match. + self.assertEqual( + _job(serve_args=RAY, nnodes="2", pp="1", concurrency=4).server_signature(), + _job(serve_args=RAY, nnodes="2", pp="1", concurrency=64).server_signature(), + ) + + def test_ray_signature_invariant_to_nnodes(self): + # Round-4 finding 2 / spec Edge Cases (line 299, called out as "intentional"): + # two ray jobs differing ONLY in nnodes (2 vs 3) must produce EQUAL + # server_signature() values, because ray's _server_argv(0) never emits + # --nnodes (ray manages cluster size, not the `vllm serve` command). This + # is what lets a reused server span differently-sized ray clusters. A + # regression that leaks nnodes / worker count into the ray argv (e.g. a + # future _bootstrap change reusing _server_argv) would break server reuse + # and is caught here -- the mirror of the concurrency-invariance test above. + self.assertEqual( + _job(serve_args=RAY, nnodes="2", pp="1").server_signature(), + _job(serve_args=RAY, nnodes="3", pp="1").server_signature(), + ) + + def test_node_rank_strip_removes_flag_and_value(self): + # Strengthened (finding 3): the old test only asserted --node-rank absent + # from a ray signature, which is vacuous (ray argv never contains it, so no + # implementation could fail it -- redundant with AC16). Here we exercise the + # strip loop where it CAN fail: an mp multi-node job's _server_argv(0) DOES + # contain "--node-rank ", and server_signature() must remove exactly that + # flag+value pair (two elements) while leaving every other token intact. A + # mutant that strips nothing, strips only the flag, or strips extra tokens + # is caught. The ray no-op (no --node-rank to strip) is also re-asserted. + mp_job = _job(serve_args={}, nnodes="2", pp="2") + argv = list(mp_job._server_argv(0)) + self.assertIn("--node-rank", argv) # precondition: mp argv has it + i = argv.index("--node-rank") + expected = argv[:i] + argv[i + 2 :] # argv minus the flag+value pair + sig_argv = list(mp_job.server_signature()[0]) + self.assertNotIn("--node-rank", sig_argv) + self.assertEqual(sig_argv, expected) + self.assertEqual(len(sig_argv), len(argv) - 2) + # Ray path: no --node-rank is ever present, so the strip is a documented no-op. + ray_sig = _job(serve_args=RAY, nnodes="2", pp="1").server_signature() + self.assertNotIn("--node-rank", ray_sig[0]) + + def test_signature_pins_actual_argv_content_not_a_constant(self): + # Rejects a hard-coded/degenerate server_signature(): the signature must + # actually contain the job's real server argv (rank-0, --node-rank + # stripped) and the real env map, not an opaque constant. + job = _job(serve_args=RAY, nnodes="2", pp="1") + expected_argv = list(job._server_argv(0)) + if "--node-rank" in expected_argv: + i = expected_argv.index("--node-rank") + del expected_argv[i : i + 2] + expected_env = tuple(sorted((str(k), str(v)) for k, v in job.server_env.items())) + sig = job.server_signature() + self.assertEqual(sig, (tuple(expected_argv), expected_env)) + self.assertIn("--tensor-parallel-size", sig[0]) + self.assertIn("--distributed-executor-backend", sig[0]) + + def test_signature_env_is_independently_sorted_and_str_cast(self): + # Round-1 finding 2: the pin test above derives its expected env tuple with + # the SAME sorted((str(k),str(v)) ...) expression as production, so the env + # half of that assertion is tautological -- a bug in that exact transform + # (wrong sort key, missing str() cast, unsorted output) would reproduce in + # both sides and still pass. Here the expected env is an INDEPENDENTLY + # hard-coded literal, and server_env is populated with multiple out-of-order + # keys plus a non-string value, so the assertion actually verifies: (a) keys + # are sorted, (b) both key and value are str()-cast (the int 3 -> "3"), (c) + # the result is a tuple of (str, str) pairs. No other test in the suite + # exercises server_env with >1 entry, so this is the sole real coverage of + # that transform. + job = _job(serve_args=RAY, nnodes="2", pp="1") + # Deliberately out-of-order insertion order; "MID" maps to an int to force str(). + job.server_env = {"ZEBRA": "z1", "ALPHA": "a1", "MID": 3} + # Independently-constructed literal oracle (not re-derived from server_env). + expected_env = (("ALPHA", "a1"), ("MID", "3"), ("ZEBRA", "z1")) + sig = job.server_signature() + self.assertEqual(sig[1], expected_env) + + def test_differing_tensor_parallelism_yields_different_ray_signature(self): + # A ray job differing in a server-affecting field (tp) must NOT share a + # signature with another ray job — otherwise an incompatible server + # would be wrongly reused across cells. + self.assertNotEqual( + _job(serve_args=RAY, nnodes="2", pp="1", tp="4").server_signature(), + _job(serve_args=RAY, nnodes="2", pp="1", tp="8").server_signature(), + ) + + def test_differing_model_id_yields_different_ray_signature(self): + # model_id is always in argv regardless of backend (unlike master_addr, + # which ray legitimately omits per AC16 / _DRIVER_DIST_FLAGS above). + job_a = _job(serve_args=RAY, nnodes="2", pp="1") + job_b = _job(serve_args=RAY, nnodes="2", pp="1") + job_b.model_id = "/models/a-different-model" + self.assertNotEqual(job_a.server_signature(), job_b.server_signature()) + + +# --------------------------------------------------------------------------- # +# is_ready (Round-2 finding 2: previously zero direct coverage) +# --------------------------------------------------------------------------- # +class TestVllmJobIsReady(unittest.TestCase): + """is_ready() greps rank-0's readiness log via orch.exec(detailed=True) and + returns True iff the collected result is non-empty AND every grepped rank's + exit_code == 0 (exit 0 = readiness pattern found). rank>0 workers are skipped + when int(nnodes) > 1 (the pre-existing guard, NOT ray-gated -- spec RC9). + These tests pin the True path, both False paths (non-zero exit, empty result), + and the multi-node worker-skip; none of it was exercised before (is_ready is + never called by the start_server tests).""" + + def test_true_when_readiness_found(self): + orch = RecordingOrch(responder=_responder_readiness(exit_code=0), hosts=[HEAD]) + job = _job(orch=orch, serve_args={}, nnodes="1", pp="1", ib_netdev=None) + self.assertTrue(job.is_ready()) + + def test_false_when_readiness_absent(self): + orch = RecordingOrch(responder=_responder_readiness(exit_code=1), hosts=[HEAD]) + job = _job(orch=orch, serve_args={}, nnodes="1", pp="1", ib_netdev=None) + self.assertFalse(job.is_ready()) + + def test_false_when_result_empty(self): + # `not out` branch: an empty/None exec result must read as NOT ready. + orch = RecordingOrch(responder=_responder_readiness(empty=True), hosts=[HEAD]) + job = _job(orch=orch, serve_args={}, nnodes="1", pp="1", ib_netdev=None) + self.assertFalse(job.is_ready()) + + def test_multinode_skips_workers_and_only_checks_rank0(self): + # nnodes=2: only rank 0 (head) is grepped; the rank-1 worker is skipped + # (rank>0 & nnodes>1 guard), so readiness is decided from rank 0 alone. + orch = RecordingOrch(responder=_responder_readiness(exit_code=0)) + job = _job(orch=orch, serve_args={}, nnodes="2", pp="2") + self.assertTrue(job.is_ready()) + self.assertEqual(_calls_to(orch, WORKER), [], "rank>0 worker must be skipped by is_ready under nnodes>1") + self.assertTrue(_calls_to(orch, HEAD), "rank 0 head readiness log must be grepped") + + +# --------------------------------------------------------------------------- # +# parse_results (Round-2 finding 3: previously zero coverage) +# --------------------------------------------------------------------------- # +class TestVllmJobParseResults(unittest.TestCase): + """parse_results() fetches the client results artifact via orch.exec_on_head + (which returns {host: content}), json-loads it, and returns + to_client_metrics(raw, tp=self.tp, isl=self.isl) per host. Two documented + exception modes: empty/missing artifact -> RuntimeError; unparseable JSON -> + RuntimeError. Exception assertions pin the TYPE only (message text is an + implementation detail per the authoring anti-patterns). The happy path pins the + delegation to to_client_metrics with the correct keyword-only tp/isl.""" + + def test_empty_artifact_raises_runtimeerror(self): + orch = RecordingOrch(head_responder=lambda cmd: {HEAD: ""}, hosts=[HEAD]) + job = _job(orch=orch, serve_args={}, nnodes="1", pp="1", ib_netdev=None) + with self.assertRaises(RuntimeError): + job.parse_results() + + def test_unparseable_json_raises_runtimeerror(self): + orch = RecordingOrch(head_responder=lambda cmd: {HEAD: "not-json{"}, hosts=[HEAD]) + job = _job(orch=orch, serve_args={}, nnodes="1", pp="1", ib_netdev=None) + with self.assertRaises(RuntimeError): + job.parse_results() + + def test_valid_artifact_delegates_to_to_client_metrics_with_tp_isl(self): + # tp and isl are keyword-only in to_client_metrics, so they MUST arrive as + # kwargs; raw (the json-loaded artifact) arrives positionally. Patching the + # symbol as imported into vllm_job keeps this impl-blind on the metric math. + # + # Round-3 finding 1: capture and assert the RETURN VALUE, not just that the + # mock was called with the right args. Production threads the metric result + # back out as {host: to_client_metrics(...)}; a mutant that calls + # to_client_metrics for its side effect but then stores `raw` (or the wrong + # host key, or returns early) would satisfy a call-args-only check while + # breaking the actual output. The mock's return_value is the independent + # oracle for what must appear under the head host key. + import json as _json + + raw = {"output_throughput": 1234.0, "request_goodput": 10.0} + sentinel = {"client.sentinel": 1} + orch = RecordingOrch(head_responder=lambda cmd: {HEAD: _json.dumps(raw)}, hosts=[HEAD]) + job = _job(orch=orch, serve_args={}, nnodes="1", pp="1", ib_netdev=None, isl="1024") + with mock.patch("cvs.lib.inference.vllm_job.to_client_metrics") as m_tcm: + m_tcm.return_value = sentinel + result = job.parse_results() + self.assertTrue(m_tcm.called, "parse_results must delegate to to_client_metrics") + args, kwargs = m_tcm.call_args + self.assertEqual(kwargs.get("tp"), job.tp) + self.assertEqual(kwargs.get("isl"), job.isl) + self.assertEqual(args[0], raw, "raw must be the json-loaded artifact passed positionally") + # The metric result must be threaded back out under the head host key -- + # NOT the raw artifact, and NOT dropped/re-keyed. + self.assertEqual(result, {HEAD: sentinel}) + + +# --------------------------------------------------------------------------- # +# wait_ready (Round-3 finding 2: the readiness polling state machine had zero +# coverage -- it is the real caller of is_ready() and _check_early_failure()) +# --------------------------------------------------------------------------- # +@mock.patch("cvs.lib.inference.vllm_job.time.sleep") +class TestVllmJobWaitReady(unittest.TestCase): + """wait_ready() drives the sequence: precheck-wait -> early-failure-check -> + warmup-wait -> early-failure-check -> poll-loop(is_ready) -> RuntimeError on + timeout. is_ready() and _check_early_failure() are unit-tested in isolation + elsewhere; here they are mocked on the instance so the ORCHESTRATION itself is + what is exercised: that is_ready is actually polled, that an exhausted poll + budget raises (not swallowed), that the early-failure check runs before the + poll loop, and that a failure surfaced during warmup aborts before polling. + time.sleep is patched at the module seam so no real waiting occurs.""" + + def test_returns_when_ready_and_stops_polling(self, mock_sleep): + # Happy path: is_ready flips True on the 3rd poll; wait_ready must return + # (no raise) and must stop polling immediately once ready (the side_effect + # list has no 4th element, so a spurious extra poll raises StopIteration). + job = _job(serve_args={}, nnodes="1", pp="1", ib_netdev=None) + job._check_early_failure = mock.Mock() + job.is_ready = mock.Mock(side_effect=[False, False, True]) + try: + job.wait_ready() + except Exception as e: # pragma: no cover - failure path + self.fail(f"wait_ready must return once is_ready() is True, raised: {e!r}") + self.assertEqual(job.is_ready.call_count, 3, "wait_ready must poll is_ready until it returns True, then stop") + self.assertTrue(job._check_early_failure.called, "wait_ready must run the early-failure check") + + def test_timeout_raises_after_exhausting_poll_budget(self, mock_sleep): + # Liveness/termination: is_ready never becomes True. wait_ready must NOT + # spin forever and must NOT swallow the failure -- it raises RuntimeError + # once the poll budget (server_poll_count) is exhausted, having polled + # is_ready exactly server_poll_count times. + # server_poll_count is bound at construction, so set it via the documented + # constructor parameter (a small budget keeps the test fast and pins the + # expected poll count without depending on the internal attribute name). + poll_count = 3 + job = VllmJob( + orch=RecordingOrch(responder=_responder_ok(), hosts=[HEAD]), + variant=_variant(serve_args={}, nnodes="1", pp="1", ib_netdev=None), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=16, + num_prompts="640", + server_poll_count=poll_count, + ) + job._check_early_failure = mock.Mock() + job.is_ready = mock.Mock(return_value=False) + with self.assertRaises(RuntimeError): + job.wait_ready() + self.assertEqual( + job.is_ready.call_count, + poll_count, + "on timeout wait_ready must have polled is_ready exactly server_poll_count times", + ) + + def test_early_failure_check_runs_before_polling(self, mock_sleep): + # Ordering: the early-failure check must precede the is_ready poll loop, so a + # crash detectable in the log is surfaced before spending the poll budget. + job = _job(serve_args={}, nnodes="1", pp="1", ib_netdev=None) + order = [] + job._check_early_failure = mock.Mock(side_effect=lambda *a, **k: order.append("check")) + job.is_ready = mock.Mock(side_effect=lambda: (order.append("ready"), True)[1]) + job.wait_ready() + self.assertIn("check", order, "the early-failure check must be invoked") + self.assertIn("ready", order, "is_ready must be polled") + self.assertEqual(order[0], "check", "early-failure check must run before the first is_ready poll") + + def test_failure_detected_during_warmup_aborts_before_polling(self, mock_sleep): + # If _check_early_failure raises (a fatal log line found during precheck/ + # warmup), wait_ready must propagate it and NOT proceed to poll is_ready -- + # the server is already known dead. + job = _job(serve_args={}, nnodes="1", pp="1", ib_netdev=None) + job._check_early_failure = mock.Mock(side_effect=RuntimeError("vllm server fatal error (rank 0)")) + job.is_ready = mock.Mock(return_value=True) + with self.assertRaises(RuntimeError): + job.wait_ready() + self.assertFalse( + job.is_ready.called, + "a failure surfaced by _check_early_failure must abort wait_ready before the poll loop", + ) + + +# --------------------------------------------------------------------------- # +# build_server_cmd (Round-3 finding 3: env-script construction had zero direct +# coverage -- multiple conditional branches forming clear equivalence classes) +# --------------------------------------------------------------------------- # +class TestVllmJobBuildServerCmd(unittest.TestCase): + """build_server_cmd() writes an env-script (broadcast to all nodes) and issues + per-rank mkdir commands. Its documented equivalence classes (per discipline + rule B, driven by subTest tables): ib_hcas present vs empty/None (the + NCCL_IB_HCA line is emitted or not), ib_netdev present vs None (the socket + interface exports are emitted or not), the server_env pass-through loop, and the + per-rank mkdir loop bounded by nnodes. Assertions target stable, named tokens + (NCCL_IB_HCA, SOCKET_IFNAME, the env keys, mkdir) rather than exact shell + formatting, and are collected from every command the method emits (whether + issued via orch.exec or returned) so the test does not couple to the transport + detail of how the script is delivered.""" + + @staticmethod + def _script(orch, ret): + parts = list(_all_cmds(orch)) + if isinstance(ret, str): + parts.append(ret) + elif isinstance(ret, (list, tuple)): + parts.extend(str(x) for x in ret) + return "\n".join(parts) + + def test_nccl_ib_hca_line_present_only_when_ib_hcas_supplied(self): + # (ib_hcas, hca_present) -- the NCCL_IB_HCA export is gated on a non-empty + # ib_hcas list; empty list and None must NOT emit it. + cases = [ + (["mlx5_0", "mlx5_1"], True), + ([], False), + (None, False), + ] + for ib_hcas, present in cases: + with self.subTest(ib_hcas=ib_hcas): + orch = RecordingOrch() + job = _job(orch=orch, serve_args={}, nnodes="2", pp="2", ib_hcas=ib_hcas) + ret = job.build_server_cmd() + script = self._script(orch, ret) + if present: + self.assertIn("NCCL_IB_HCA", script) + # The supplied HCA name must actually reach the export value. + self.assertIn("mlx5_0", script) + else: + self.assertNotIn("NCCL_IB_HCA", script) + + def test_socket_ifname_exports_present_only_when_ib_netdev_set(self): + # ib_netdev set -> the socket-interface exports are emitted (all three name + # the device); ib_netdev None -> none are emitted. + orch_set = RecordingOrch() + _job(orch=orch_set, serve_args={}, nnodes="2", pp="2", ib_netdev="eth0").build_server_cmd() + script_set = self._script(orch_set, None) + self.assertEqual( + script_set.count("SOCKET_IFNAME"), + 3, + "ib_netdev must emit exactly the three socket-ifname exports", + ) + self.assertIn("eth0", script_set, "the configured ib_netdev must reach the export value") + + orch_none = RecordingOrch() + _job(orch=orch_none, serve_args={}, nnodes="2", pp="2", ib_netdev=None).build_server_cmd() + script_none = self._script(orch_none, None) + self.assertNotIn("SOCKET_IFNAME", script_none, "no ib_netdev -> no socket-ifname exports") + + def test_server_env_entries_passed_through(self): + # Every server_env key/value must appear in the emitted env-script (the + # pass-through loop). Two entries so a single-entry short-circuit is caught. + # + # Round-4 finding 1: the values MUST be distinctive strings that cannot + # collide with any boilerplate line the env-script also emits. A bare value + # like "1" trivially matches elsewhere (e.g. "...AITER_UNIFIED_ATTENTION=1"), + # so assertIn("1", script) is vacuous -- a mutant that hard-codes a wrong + # value or drops the CUSTOM_A line entirely still passes. Using unique + # values AND asserting the "KEY=VALUE" pairing (not the bare value) pins + # both the presence and the key/value association without coupling to the + # exact "export " prefix formatting. + orch = RecordingOrch() + job = _job( + orch=orch, + serve_args={}, + nnodes="2", + pp="2", + env={"CUSTOM_A": "CUSTOM_A_VALUE_XYZ", "CUSTOM_B": "CUSTOM_B_VALUE_QRS"}, + ) + ret = job.build_server_cmd() + script = self._script(orch, ret) + for key, val in (("CUSTOM_A", "CUSTOM_A_VALUE_XYZ"), ("CUSTOM_B", "CUSTOM_B_VALUE_QRS")): + with self.subTest(key=key): + self.assertIn(f"{key}={val}", script, f"server_env {key} must be exported paired with its value") + + def test_mkdir_count_scales_with_nnodes(self): + # The per-rank mkdir loop is bounded by nnodes: a 3-node job must issue + # strictly more mkdir commands than a single-node job (all else equal). + orch1 = RecordingOrch(hosts=[HEAD]) + _job(orch=orch1, serve_args={}, nnodes="1", pp="1", ib_netdev=None).build_server_cmd() + orch3 = RecordingOrch(hosts=[HEAD, WORKER, HOST2]) + _job(orch=orch3, serve_args={}, nnodes="3", pp="1", ib_netdev="eth0").build_server_cmd() + mk1 = self._script(orch1, None).count("mkdir") + mk3 = self._script(orch3, None).count("mkdir") + self.assertGreater(mk1, 0, "build_server_cmd must create at least the rank-0 log dir") + self.assertGreater(mk3, mk1, "per-rank mkdir loop must scale with nnodes") + + +# --------------------------------------------------------------------------- # +# Lifecycle (transition table) +# --------------------------------------------------------------------------- # +# | from state | event | to state / effect | +# |----------------------|---------------------------|---------------------------------------| +# | constructed | start_server() [ok ray] | bootstrap head+worker, serve on head | +# | started | start_server() again | re-entrant: no raise, re-launch serve | +# | started | stop_server() | pkill + ray stop broadcast -> down | +# | bootstrap-failed | start_server() [head bad] | RuntimeError, no serve (illegal txn) | +# | bootstrap-failed | stop_server() | must NOT raise (partial cleanup) | +# | down | stop_server() again | idempotent no-op, no raise | +@mock.patch("cvs.lib.inference.vllm_job.time.sleep") +class TestVllmJobRayLifecycle(unittest.TestCase): + def test_legal_start_then_stop(self, mock_sleep): + orch = RecordingOrch(responder=_responder_ok()) + job = _job(orch=orch, serve_args=RAY, nnodes="2", pp="1") + job.start_server() # constructed -> started + job.stop_server() # started -> down + self.assertTrue([c for c in _calls_to(orch, HEAD) if "vllm serve" in c]) + self.assertTrue([c for c, _ in orch.calls if "ray stop" in c]) + + def test_illegal_start_on_bad_bootstrap_is_rejected(self, mock_sleep): + orch = RecordingOrch(responder=_responder_bootstrap_fail({HEAD: {"exit_code": 1, "output": _BAD}})) + job = _job(orch=orch, serve_args=RAY, nnodes="2", pp="1") + with self.assertRaises(RuntimeError): + job.start_server() + + def test_stop_after_failed_start_does_not_raise(self, mock_sleep): + # Regression: partial bootstrap -> caller must be able to stop_server safely. + orch = RecordingOrch(responder=_responder_bootstrap_fail({HEAD: {"exit_code": 1, "output": _BAD}})) + job = _job(orch=orch, serve_args=RAY, nnodes="2", pp="1") + with self.assertRaises(RuntimeError): + job.start_server() + calls_before_stop = len(orch.calls) + try: + job.stop_server() # must be robust after a failed/partial start + except Exception as e: # pragma: no cover - failure path + self.fail(f"stop_server after failed start raised: {e!r}") + # Finding 9: not merely "no exception" -- assert stop_server actually did + # the full teardown after the partial start. Both broadcasts (hosts=None) + # must be issued: the pkill vllm serve, then (nnodes>1 & ray) ray stop. + stop_calls = orch.calls[calls_before_stop:] + pkill = [(c, h) for c, h in stop_calls if "pkill" in c and "vllm serve" in c] + ray_stop = [(c, h) for c, h in stop_calls if "ray stop" in c] + self.assertEqual(len(pkill), 1, f"expected one pkill broadcast on stop, got {pkill}") + self.assertIsNone(pkill[0][1], "pkill must be a broadcast (hosts=None)") + self.assertEqual(len(ray_stop), 1, f"expected one ray stop broadcast on stop, got {ray_stop}") + self.assertIsNone(ray_stop[0][1], "ray stop must be a broadcast (hosts=None)") + + def test_reentrant_start_is_not_rejected(self, mock_sleep): + # Coverage-gap (finding 2): the transition table had a legal start and an + # idempotent stop-reentry, but no started -> start_server()-again case. The + # spec documents no idempotency guard / no documented error on re-entrant + # start, so a second start_server() must not raise and re-runs the bootstrap + # + serve sequence (mirroring the re-run semantics of the stop reentry test: + # each teardown issues its own ray stop, so each start issues its own serve). + orch = RecordingOrch(responder=_responder_ok()) + job = _job(orch=orch, serve_args=RAY, nnodes="2", pp="1") + job.start_server() # constructed -> started + try: + job.start_server() # started -> start again (re-entrant) + except Exception as e: # pragma: no cover - failure path + self.fail(f"re-entrant start_server raised: {e!r}") + head_serves = [c for c in _calls_to(orch, HEAD) if "vllm serve" in c] + self.assertEqual( + len(head_serves), 2, f"each start_server must (re-)launch vllm serve on the head; got {head_serves}" + ) + + def test_idempotent_stop_reentry(self, mock_sleep): + orch = RecordingOrch(responder=_responder_ok()) + job = _job(orch=orch, serve_args=RAY, nnodes="2", pp="1") + job.start_server() + job.stop_server() + try: + job.stop_server() # cleanup twice must not raise + except Exception as e: # pragma: no cover - failure path + self.fail(f"second stop_server raised: {e!r}") + # Each teardown issues its own ray stop broadcast. + self.assertEqual(len([c for c, _ in orch.calls if "ray stop" in c]), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/inference/unittests/test_vllm_job_server_reuse.py b/cvs/lib/inference/unittests/test_vllm_job_server_reuse.py new file mode 100644 index 000000000..98ae38af2 --- /dev/null +++ b/cvs/lib/inference/unittests/test_vllm_job_server_reuse.py @@ -0,0 +1,276 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for cvs.lib.inference.vllm_job.VllmJob server-command construction: + - the duplicate --max-model-len fix (config-pin suppresses the derived value) + - server_signature(), which gates cross-cell server reuse + - _flatten_serve_args boolean handling and log-level pass-through + - _check_early_failure tail emission and CLI parse error detection + - RoleServer.serve_args log-level validator +''' + +import unittest +import unittest.mock as mock +from types import SimpleNamespace + +import pydantic + +from cvs.lib.inference.utils.vllm_config_loader import RoleServer +from cvs.lib.inference.vllm_job import VllmJob + +_TP = 8 +_PP = 2 +_NNODES = 2 + + +class FakeOrch: + hosts = ["10.0.0.1", "10.0.0.2"] + + def __init__(self): + self.head_cmds = [] + + def exec(self, *a, **k): + return {} + + def exec_on_head(self, cmd, *a, **k): + self.head_cmds.append(cmd) + return {} + + +class FakeOrchWithOutput: + """Single-rank fake orch that returns controllable tail/grep output.""" + + hosts = ["10.0.0.1"] + + def __init__(self, tail_output="", grep_exit=1): + self.head_cmds = [] + self._tail_output = tail_output + self._grep_exit = grep_exit # 1 = no match (safe), 0 = match found + + def exec(self, cmd, hosts=None, detailed=False): + if detailed: + return {"10.0.0.1": {"exit_code": self._grep_exit, "stdout": ""}} + return {"10.0.0.1": self._tail_output} + + def exec_on_head(self, cmd, *a, **k): + self.head_cmds.append(cmd) + return {} + + +def _make_job_for_check(tail_output="", grep_exit=1): + """Construct a VllmJob suitable for testing _check_early_failure.""" + variant = mock.MagicMock() + variant.params.tensor_parallelism = "8" + variant.params.pipeline_parallel_size = "1" + variant.params.master_addr = "localhost" + variant.params.master_port = "29501" + variant.params.nnodes = "1" + variant.params.port_no = "8000" + variant.params.random_range_ratio = "0.0" + variant.params.random_prefix_len = "0" + variant.params.burstiness = "1.0" + variant.params.seed = "0" + variant.params.request_rate = "inf" + variant.params.tokenizer_mode = "auto" + variant.params.percentile_metrics = "ttft,tpot,itl,e2el" + variant.params.metric_percentiles = "50,90,95,99" + variant.params.base_url = "http://0.0.0.0" + variant.params.dataset_name = "random" + variant.params.backend = "vllm" + variant.model.id = "/models/test-model" + variant.paths.log_dir = "/tmp/test_logs" + variant.paths.models_dir = "/tmp/models" + variant.roles.server.serve_args = {} + variant.roles.server.env = {} + variant.roles.server.ib_netdev = None + orch = FakeOrchWithOutput(tail_output=tail_output, grep_exit=grep_exit) + return VllmJob( + orch=orch, + variant=variant, + hf_token="tok", + isl="1024", + osl="1024", + concurrency="8", + num_prompts="100", + ) + + +def _variant(serve_args=None): + params = SimpleNamespace( + tensor_parallelism=str(_TP), + pipeline_parallel_size=str(_PP), + master_addr="10.0.0.1", + master_port="29501", + nnodes=str(_NNODES), + port_no="8000", + random_range_ratio="0.8", + random_prefix_len="0", + burstiness="1.0", + seed="0", + request_rate="inf", + tokenizer_mode="auto", + percentile_metrics="ttft,tpot,itl,e2el", + metric_percentiles="50,90,95,99", + base_url="http://0.0.0.0", + dataset_name="random", + backend="vllm", + ) + return SimpleNamespace( + params=params, + model=SimpleNamespace(id="/models/Kimi-K2.5-W4A8"), + paths=SimpleNamespace(log_dir="/logs", models_dir="/models"), + roles=SimpleNamespace( + server=SimpleNamespace( + serve_args=dict(serve_args or {}), + env={"VLLM_ROCM_USE_AITER": "1"}, + ib_netdev="enp159s0np0", + ) + ), + ) + + +def _job(isl, osl, conc, serve_args=None): + return VllmJob( + orch=FakeOrch(), + variant=_variant(serve_args), + hf_token="tok", + isl=isl, + osl=osl, + concurrency=conc, + num_prompts="640", + ) + + +class TestMaxModelLenNoDuplicate(unittest.TestCase): + def test_config_pin_wins_and_no_duplicate(self): + argv = _job("1024", "1024", 16, serve_args={"max-model-len": "16384"})._server_argv(0) + idxs = [i for i, a in enumerate(argv) if a == "--max-model-len"] + self.assertEqual(len(idxs), 1, "config-pinned max-model-len must appear exactly once") + self.assertEqual(argv[idxs[0] + 1], "16384", "config value must win") + + def test_derived_emitted_when_not_pinned(self): + argv = _job("1024", "1024", 16, serve_args={})._server_argv(0) + idxs = [i for i, a in enumerate(argv) if a == "--max-model-len"] + self.assertEqual(len(idxs), 1, "derived max-model-len must still be emitted when unpinned") + # 1024+1024 worst-case derived value, definitely not the 16384 config value + self.assertNotEqual(argv[idxs[0] + 1], "16384") + + +class TestServerSignatureReuse(unittest.TestCase): + def test_invariant_to_concurrency(self): + # Pinned max-model-len: cells differing only in concurrency share a server. + sa = {"max-model-len": "16384"} + self.assertEqual( + _job("1024", "1024", 4, sa).server_signature(), + _job("1024", "1024", 64, sa).server_signature(), + ) + + def test_pinned_mml_shares_across_isl_osl(self): + # With a fixed max-model-len, ISL/OSL never reach the server argv, so all + # cells legitimately share one server (ISL/OSL are client-only knobs). + sa = {"max-model-len": "16384"} + self.assertEqual( + _job("1024", "1024", 16, sa).server_signature(), + _job("8192", "1024", 16, sa).server_signature(), + ) + + def test_derived_mml_distinguishes_osl(self): + # Without a pin, max-model-len is derived per (isl+osl); different OSL must + # change the signature so a real restart happens. + self.assertNotEqual( + _job("1024", "1024", 16, serve_args={}).server_signature(), + _job("1024", "8192", 16, serve_args={}).server_signature(), + ) + + def test_signature_strips_node_rank_and_is_hashable(self): + job = _job("1024", "1024", 16, serve_args={"max-model-len": "16384"}) + self.assertIn("--node-rank", job._server_argv(0)) + sig = job.server_signature() + self.assertNotIn("--node-rank", sig[0]) + # hashable + stable + self.assertEqual(hash(sig), hash(job.server_signature())) + + +class TestRunClientEnsuresOutDir(unittest.TestCase): + """The server-reuse path skips build_server_cmd (which creates the per-cell + out_dir), so run_client must create its own out_dir or the client's + client.log/results writes fail with 'No such file or directory'.""" + + def test_run_client_mkdirs_out_dir(self): + job = _job("1024", "1024", 8, serve_args={"max-model-len": "16384"}) + job.run_client() + mkdir_cmds = [c for c in job.orch.head_cmds if "mkdir -p" in c and job.out_dir in c] + self.assertTrue( + mkdir_cmds, + f"run_client must mkdir -p its out_dir ({job.out_dir}) so the reuse path " + f"(which skips build_server_cmd) can still write client.log; head cmds: {job.orch.head_cmds}", + ) + + +class TestRunClientTrustRemoteCode(unittest.TestCase): + """Models with a custom tokenizer (e.g. Kimi-K2.6's auto_map) need the bench + client to pass --trust-remote-code, mirroring the server's serve_args, or the + client's tokenizer load raises ValueError before any request is sent.""" + + def _bench_cmd(self, job): + job.run_client() + bench = [c for c in job.orch.head_cmds if "vllm" in c and "bench" in c] + self.assertTrue(bench, f"no bench client command issued; head cmds: {job.orch.head_cmds}") + return bench[-1] + + def test_trust_remote_code_passed_when_server_enables_it(self): + job = _job("1024", "1024", 8, serve_args={"max-model-len": "16384", "trust-remote-code": True}) + self.assertIn("--trust-remote-code", self._bench_cmd(job)) + + def test_trust_remote_code_absent_when_server_omits_it(self): + job = _job("1024", "1024", 8, serve_args={"max-model-len": "16384"}) + self.assertNotIn("--trust-remote-code", self._bench_cmd(job)) + + +class TestFlattenServeArgsFalse(unittest.TestCase): + def test_false_value_omitted(self): + result = VllmJob._flatten_serve_args({"enable-prefix-caching": False, "tensor-parallel-size": "8"}) + self.assertNotIn("--enable-prefix-caching", result) + self.assertNotIn("False", result) + self.assertEqual(result, ["--tensor-parallel-size", "8"]) + + def test_true_value_emits_flag_only(self): + result = VllmJob._flatten_serve_args({"enforce-eager": True}) + self.assertEqual(result, ["--enforce-eager"]) + + def test_log_level_passed_through(self): + result = VllmJob._flatten_serve_args({"log-level": "debug"}) + self.assertEqual(result, ["--log-level", "debug"]) + + +class TestCheckEarlyFailureEmitTail(unittest.TestCase): + def test_emit_tail_true_logs_content(self): + job = _make_job_for_check(tail_output="INFO engine loading\nINFO weights done") + with mock.patch("cvs.lib.inference.vllm_job.log") as mock_log: + job._check_early_failure(emit_tail=True) + logged_lines = [call.args[3] for call in mock_log.info.call_args_list if len(call.args) >= 4] + self.assertIn("INFO engine loading", logged_lines) + self.assertIn("INFO weights done", logged_lines) + + def test_raises_on_cli_parse_error(self): + job = _make_job_for_check(tail_output="vllm: error: unrecognized arguments: False") + with self.assertRaises(RuntimeError): + job._check_early_failure() + + +class TestRoleServerLogLevelValidator(unittest.TestCase): + def test_invalid_log_level_rejected(self): + with self.assertRaises(pydantic.ValidationError) as ctx: + RoleServer(serve_args={"log-level": "verbose"}) + msg = str(ctx.exception) + self.assertIn("log-level", msg) + self.assertIn("verbose", msg) + + def test_valid_log_level_accepted(self): + rs = RoleServer(serve_args={"log-level": "debug"}) + self.assertEqual(rs.serve_args["log-level"], "debug") + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/inference/unittests/test_vllm_report_preset.py b/cvs/lib/inference/unittests/test_vllm_report_preset.py new file mode 100644 index 000000000..664b080af --- /dev/null +++ b/cvs/lib/inference/unittests/test_vllm_report_preset.py @@ -0,0 +1,102 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. +''' + +import unittest + +from cvs.lib.inference.utils.vllm_parsing import ( + CLIENT_METRICS, + GATED_METRICS, + METRIC_TIER_ORDER, + METRIC_TIERS, + VLLM_RESULTS_COLUMNS, + tier_metric_specs, +) +from cvs.lib.report.presets.vllm import VLLM_REPORT_CONFIG + + +class TestVllmReportPreset(unittest.TestCase): + def test_results_columns_fixed_positional_prefix(self): + fixed = VLLM_RESULTS_COLUMNS[:7] + self.assertEqual( + fixed, + ( + ("Model", None), + ("GPU", None), + ("ISL", None), + ("OSL", None), + ("Policy", None), + ("Conc", None), + ("Host", None), + ), + ) + + def test_metric_tiers_subset_of_tier_order(self): + self.assertTrue(set(METRIC_TIERS) <= set(METRIC_TIER_ORDER)) + + def test_gated_metrics_partitioned_exactly_once(self): + tiered = [m for names in METRIC_TIERS.values() for m in names] + # No duplicates across tiers. + self.assertEqual(len(tiered), len(set(tiered))) + # Every gated metric lands in exactly one non-record tier. + self.assertEqual(set(tiered), set(GATED_METRICS)) + + def test_gated_metrics_subset_of_client_metrics(self): + client_short = {short for short, _unit in CLIENT_METRICS} + missing = GATED_METRICS - client_short + self.assertEqual(missing, set(), f"GATED_METRICS not in CLIENT_METRICS: {missing}") + + def test_tier_metric_specs_throughput(self): + cell = { + "client.output_throughput": {"kind": "min_tok_s", "value": 1}, + "client.mean_ttft_ms": {"kind": "max_ms", "value": 2}, + } + specs = tier_metric_specs(cell, "throughput") + self.assertIn("client.output_throughput", specs) + self.assertNotIn("client.mean_ttft_ms", specs) + + def test_tier_metric_specs_record_includes_non_tiered(self): + cell = { + "client.num_prompts": {"kind": "within", "value": 100}, + "client.output_throughput": {"kind": "min_tok_s", "value": 1}, + } + specs = tier_metric_specs(cell, "record") + self.assertIn("client.num_prompts", specs) + self.assertNotIn("client.output_throughput", specs) + + def test_preset_config_identity(self): + self.assertEqual(VLLM_REPORT_CONFIG.suite_id, "vllm") + self.assertEqual(VLLM_REPORT_CONFIG.inference_test_substring, "test_vllm_inference") + self.assertEqual(VLLM_REPORT_CONFIG.row_card_test_names, ("test_metric",)) + + def test_preset_lifecycle_labels_match_what_suite_records(self): + # Guard against drift: the vLLM suite (cvs/tests/inference/vllm/vllm.py) + # records exactly these session-level stages via lifecycle.record(...). + suite_recorded = { + "container_launch", + "topology_discovery", + "model_fetch", + "server_ready", + "teardown", + } + self.assertTrue(set(VLLM_REPORT_CONFIG.session_lifecycle_labels) <= suite_recorded) + self.assertTrue(set(VLLM_REPORT_CONFIG.cell_lifecycle_labels) <= suite_recorded) + + def test_auto_register_resolves_vllm_stem(self): + from cvs.lib.report.auto_register import try_auto_register_inference_suite_report + from cvs.lib.report.registry import get_suite_report_config + + class _FakeConfig: + pass + + cfg = _FakeConfig() + cfg._suite_name = "vllm" + cfg._suite_report_config = None + registered = try_auto_register_inference_suite_report(cfg) + self.assertTrue(registered) + self.assertIs(get_suite_report_config(cfg), VLLM_REPORT_CONFIG) + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/inference/utils/AGENTS.md b/cvs/lib/inference/utils/AGENTS.md new file mode 100644 index 000000000..2a99bfc74 --- /dev/null +++ b/cvs/lib/inference/utils/AGENTS.md @@ -0,0 +1,325 @@ +# cvs/lib/inference/utils — inference-specific config and parsing + +**Boundary**: this is the serving/inference half of the config machinery. +The generic half (`BaseVariantConfig`, `substitute_config`, `evaluate_all`, `Paths`, `ContainerSpec`) +lives in `cvs/lib/utils/` — import from there, never duplicate it here. + +--- + +## Files + +### `inferencing_config_loader.py` + +#### Schema classes + +**`RoleServer`** (`_Forbid`): per-model server overrides. +- `serve_args: Dict[str, Any]` — extra `vllm serve` flags; scalar → `--flag value`, + `True` → bare `--flag`, list → flag repeated per element +- `env: Dict[str, str]` — env vars merged over orchestrator defaults +- Both default empty; fp8-kv cells set `--kv-cache-dtype` here to keep the generic driver model-agnostic + +**`Roles`** (`_Forbid`): wraps `RoleServer`. +- `server: RoleServer` — defaults to empty `RoleServer()` + +**`GoodputSlo`** (`_Forbid`): per-combo goodput gate, in milliseconds. +- `ttft_ms: float`, `tpot_ms: float`, `e2el_ms: float` +- **INPUT to the run** (passed to `vllm bench serve --goodput`), NOT a threshold to assert. + Lives in the sweep, not `threshold.json`. `_Forbid` ensures a typo'd SLO key fails load + rather than silently drop the SLO and run with the wrong gate on hardware. + +**`SeqCombo`** (`_Forbid`): one named sequence-length combination. +- `name: str` — the join key referenced by `Run.combo` +- `isl: str`, `osl: str` +- `goodput_slo: Optional[GoodputSlo]` — omit when no goodput gate is needed + +**`Run`** (`_Forbid`): one sweep cell — a named combo at a single concurrency. +- `combo: str` — references a `SeqCombo.name` +- `concurrency: int` +- Explicit `runs[]` replaces the old NxM cartesian (`sequence_combinations × concurrency_levels`); + you enumerate exactly the cells you want + +**`Sweep`** (`_Forbid`): the full sweep selector. +- `sequence_combinations: List[SeqCombo]` +- `runs: List[Run]` +- `@model_validator(mode="after")` delegates to `validate_sweep_selector` + +**`Params`** (`_Forbid`): `vllm bench serve` CLI flags; all fields are `str`. + +| Field | Default | Notes | +|---|---|---| +| `backend` | `"vllm"` | | +| `base_url` | `"http://0.0.0.0"` | | +| `port_no` | `"8888"` | | +| `dataset_name` | `"random"` | | +| `burstiness` | `"1.0"` | | +| `seed` | `"0"` | | +| `request_rate` | `"inf"` | | +| `random_range_ratio` | `"0.8"` | | +| `random_prefix_len` | `"0"` | | +| `tensor_parallelism` | `"1"` | used in `cell_key` and `per_gpu_throughput` | +| `tokenizer_mode` | `"auto"` | | +| `percentile_metrics` | `"ttft,tpot,itl,e2el"` | | +| `metric_percentiles` | `"50,90,95,99"` | | +| `num_prompts` | `"3200"` | overridden per-cell by `_num_prompts_for` | +| `client_poll_count` | `"20"` | see below | + +`client_poll_count` semantics: total client wait budget = +**(client_poll_count × 60 s) + 120 s initial wait**. +The poll loop exits as soon as the client finishes, so raising this never slows down fast cells. +Raise it for high-osl cells where large-output runs take longer to complete. (Regression: REG-20260609-001) + +**`VariantConfig(BaseVariantConfig)`**: the full typed config. +- Adds: `framework: Literal["vllm_single"]`, `gpu_arch: str`, `roles: Roles = Roles()`, + `params: Params`, `sweep: Sweep` +- Implements: `cell_key(isl, osl, concurrency)`, `expected_cells()` +- Has: `@model_validator(mode="after") _check_thresholds_cover_sweep` + +--- + +#### Public functions + +**`load_variant(config_path, cluster_dict) -> VariantConfig`** + +The function a suite's `variant_config` fixture calls. + +1. Delegates file read + 3-pass placeholder substitution to `substitute_config` +2. Attaches thresholds returned by `substitute_config` to the raw dict +3. Builds and returns a typed, validated `VariantConfig` + +Does not reimplement file reading or substitution — always calls `substitute_config`. +See `cvs/lib/utils/AGENTS.md` for the full `substitute_config` contract. + +**`validate_sweep_selector(combo_names, run_combo_refs)`** — PUBLIC ENTRY POINT + +Shared rule called by **both**: +- the typed `Sweep` validator at load time +- `pytest_generate_tests` at collection time (reads raw JSON before the loader runs) + +Checks: +- combo names are unique (duplicate → `ValueError`) +- every `run.combo` names a known `sequence_combination` (unknown → `ValueError`) + +Operates on plain `list[str]` so both call sites feed it without the full typed schema. +If you add a sweep check, add it here so both paths enforce it without drift. + +--- + +#### `_check_thresholds_cover_sweep` — two-axis coverage check + +`@model_validator(mode="after")` on `VariantConfig`. Fails at load time if the threshold file +does not match the sweep matrix. + +**Axis 1 — cell coverage** +- Every sweep cell produced by `expected_cells()` has an entry in `threshold.json` +- No threshold key names a non-existent cell (catches typos in threshold key names) + +**Axis 2 — gated-metric coverage** +- Every cell present in both sets has a spec for every `client.` key + (e.g. `client.total_token_throughput`, not `total_token_throughput`) +- Without this, a gated metric with no spec falls through `test_metric`'s `spec is None` + record-only branch and reports PASS with zero assertions even under `enforce_thresholds=true` +- Only checked for cells present in both expected and threshold sets (missing cells are already + reported by axis 1; no double-reporting) + +When `enforce_thresholds=false`: both failures become warnings, not errors. +The config loads as a record-only scaffold (metrics captured, nothing asserted). + +See `docs/cell-key-format.md` for the exact key format used in `threshold.json`. + +--- + +### `vllm_parsing.py` + +**`to_client_metrics(raw, *, tp, isl) -> dict`** + +Pure — no I/O, no orchestration. `raw` is the already-parsed JSON the load generator +writes to its `--result-dir` results artifact. Caller is responsible for fetching and +`json.load`-ing the artifact. + +- All stock scalars namespaced 1:1 as `client.` +- `client.goodput` = alias for stock's `request_goodput` (the name threshold files reference) +- Derived metrics (all guarded by `_safe_div`; degrade to `None` on missing/`None`/zero-divisor): + +| Metric | Formula | +|---|---| +| `client.per_gpu_throughput` | `total_token_throughput / tp` | +| `client.normalized_ttft_ms_per_tok` | `mean_ttft_ms / isl` | +| `client.decode_latency_ratio` | `p99_itl_ms / p50_itl_ms` | +| `client.decode_throughput_p50` | `1000.0 / median_tpot_ms` | +| `client.success_rate` | `completed / (completed + failed)` | + +See `docs/derived-metrics.md` for per-metric inputs, None conditions, and gated/record-only rationale. + +**`CLIENT_METRICS`** — ordered `list[(short_name, unit)]`. + +The display surface: one HTML row per metric per cell. Single definition shared by all vLLM +flavours — do not re-list per suite. + +**`CLIENT_METRIC_UNITS`** — `dict` form of `CLIENT_METRICS`. + +**`GATED_METRICS`** — the asserted subset of `CLIENT_METRICS`. + +Membership = "out of range means FAILURE". + +Closed-world default: a new metric added to `CLIENT_METRICS` is record-only until its name +is explicitly added to `GATED_METRICS`. The loader's coverage check then forces a spec for +that metric in every cell before the suite can run green. + +Currently gated: + +| Category | Members | +|---|---| +| Throughput | `total_token_throughput`, `output_throughput` | +| TTFT latency | `mean`, `median`, `p90`, `p95`, `p99` | +| TPOT latency | `mean`, `median`, `p90`, `p95`, `p99` | +| ITL latency | `mean`, `median`, `p95`, `p99` (no p90 producer) | +| E2EL latency | `mean`, `median`, `p90`, `p95`, `p99` | +| Run health | `success_rate` (floor), `failed` (ceiling) | + +Record-only by design: inputs (`num_prompts`), totals (`total_input_tokens`, +`total_output_tokens`), secondary throughputs (`per_gpu_throughput`, `request_throughput`, +`goodput`, `decode_throughput_p50`, `max_output_tokens_per_s`), diagnostic derivations +(`normalized_ttft_ms_per_tok`, `decode_latency_ratio`). + +--- + +## The sweep selector + +Named combos + explicit `runs[]` list replaces old NxM cartesian +(`sequence_combinations × concurrency_levels`). One `Run` = one `(combo, concurrency)` cell. +The sweep enumerates exactly the cells you want. + +`sequence_combinations` names each ISL/OSL shape once (with an optional goodput SLO); +`runs` references those names at specific concurrencies. This lets you include only the +cells that matter for a given model — no silent NxM explosion, no empty cells. + +--- + +## The serving-generic / vllm-specific seam + +`Params` is the only vllm-specific class. Everything else (`Sweep`, `SeqCombo`, `GoodputSlo`, +`Roles`, `cell_key`) is serving-generic and reusable when a second serving framework lands. + +**Second serving framework checklist:** +1. Subclass `Params` with your framework's CLI flags +2. Reuse `Sweep`/`SeqCombo`/`GoodputSlo`/`Roles` unchanged +3. Reuse `validate_sweep_selector` in your `pytest_generate_tests` +4. Write your own metric vocabulary (`_parsing.py`) +5. Define your own `GATED_METRICS` + +--- + +## Lifecycle-as-tests model + +Each stage of the test run is an **independent pytest test**, not fixture body code. +Each stage appears as a timed, independently pass/fail row in the HTML report. + +Standard lifecycle order (pinned in `pytest_collection_modifyitems`): + +| Rank | Test | Action | +|---|---|---| +| 0 | `test_launch_container` | `setup_containers()`; asserts container is running | +| 1 | `test_setup_sshd` | `setup_sshd()`; probes `:2224` for multinode only | +| 2 | `test_model_fetch` | ensures model bytes present; polls or downloads if remote | +| 3 | `test_vllm_inference` | benchmark loop per cell; stores results in `inf_res_dict` | +| 4 | `test_metric` | one test per metric per cell; reads `inf_res_dict`; asserts verdict | +| 5 | `test_print_results_table` | summary log; must run after all cells | +| 6 | `test_teardown` | `teardown_containers()`; sets `lifecycle.torn_down`; **never skips** | + +Rules: +- Every test except `test_launch_container`, `test_teardown`, and `test_print_results_table` + checks `lifecycle.failed` and skips if true. `test_launch_container` is the first stage + and is itself responsible for setting `lifecycle.failed`; it has no prior stage to guard + against. `test_teardown` must run even on failure. `test_print_results_table` guards only + on whether `inf_res_dict` is empty and logs whatever results were recorded. +- `test_vllm_inference` catches exceptions, sets `lifecycle.failed = True`, re-raises +- `test_teardown` never skips — must run even on failure; sets `lifecycle.torn_down = True` + to suppress the `orch` fixture's leak-guard finalizer (prevents double teardown) + +`test_metric` verdict pattern: +- Reads value from `inf_res_dict`; attaches to `user_properties` for HTML rendering +- If `enforce_thresholds` and a spec exists for this cell+metric: calls `evaluate_all` + with **the full per-cell actuals dict** (not just the single metric value) so a + `min_ratio` spec can resolve its reference metric +- Otherwise: record-only PASS + +--- + +## conftest fixtures + +All fixtures are `scope="module"`. + +| Fixture | Owns | Key detail | +|---|---|---| +| `cluster_dict` | reads `--cluster_file` JSON; resolves placeholders | calls `resolve_cluster_config_placeholders` | +| `variant_config` | calls `load_variant(config_file, cluster_dict)` | the sole entry point to the typed schema | +| `lifecycle` | `_Lifecycle` instance | shared cross-test state: `failed`, `torn_down`, `report` | +| `orch` | builds `ContainerOrchestrator`; registers leak-guard finalizer | deep-merges variant container block onto cluster container block | +| `hf_token` | reads `variant_config.paths.hf_token_file` | skips if file absent | +| `inf_res_dict` | module-scoped `{}` keyed by `(model_id, gpu_arch, isl, osl, combo_name, concurrency)` | populated by `test_vllm_inference`; consumed by `test_metric` | + +`_deep_merge` helper: `OrchestratorConfig.from_configs` does a top-level `dict.update`, +so a bare variant container block wipes the cluster file's container settings. The conftest +deep-merges the variant block onto the cluster block so cluster-set scalar/dict keys survive +with the variant winning on conflicts. List keys (e.g. runtime args, volume mounts) are +replaced at the merge step and recombined additively downstream in `container.py`'s getters. + +Required pytest hooks (all in `conftest.py`): +- `pytest_collection_modifyitems` — pins lifecycle order; imported functions (e.g. + `test_print_results_table` from `_shared.py`) sort by source line, not insertion order, + so explicit pinning is mandatory +- `pytest_runtest_makereport` — attaches lifecycle timing rows to the HTML detail panel +- `pytest_html_results_table_header` / `pytest_html_results_table_row` — adds Value/Unit + columns; populated for `test_metric` rows, blank for lifecycle/inference rows + +--- + +## `pytest_generate_tests` mirror rule + +`pytest_generate_tests` runs at **collection time**, before any fixtures exist. +It reads the raw config JSON directly — it cannot use the `variant_config` fixture. + +To keep collection-time validation aligned with load-time validation: + +1. Call `validate_sweep_selector` on the raw combo names and run combo refs — mirrors the + typed `Sweep` validator so duplicate names and unknown refs fail collection, not silently drop +2. Validate each raw `goodput_slo` dict through `GoodputSlo(**combo["goodput_slo"])` — + mirrors the `_Forbid` model so a typo'd SLO key fails collection, not runs with the wrong gate + +**Rule**: if you add a check to `Sweep`, add it to `validate_sweep_selector` (or an equivalent +call in `pytest_generate_tests`) so both paths enforce it without drift. + +--- + +## Gotchas + +- **`cell_key` is the single source of truth** — the loader coverage check (`expected_cells`) + and the test verdict lookup (`test_metric`) both call it; change the format in one place and + everything keyed on it moves together. See `docs/cell-key-format.md` for the exact format. + +- **Both axes of `_check_thresholds_cover_sweep` must pass** — without axis 2 a gated metric + with no spec reports zero-assertion PASS even under `enforce_thresholds=true`; the silent + green is indistinguishable from a real pass. + +- **`pytest_generate_tests` reads raw JSON** — it runs before fixtures exist; mirror every + `Sweep` validator via `validate_sweep_selector` or the two paths drift. + +- **`GoodputSlo` is an INPUT, not a threshold** — typo'd key fails load (`_Forbid`); + lives in the sweep config, not `threshold.json`. + +- **`to_client_metrics` is deliberately I/O-free** — the fetch lives in the job class; + callers hand in an already-parsed dict. + +- **Derived metrics degrade to `None` via `_safe_div`, never crash** — `None` renders as + `-` in the HTML table; if the metric is gated, `evaluate_all` will report a loud violation. + +- **All gated-metric threshold keys must use the `client.` prefix** — the axis 2 check + resolves `client.` in the threshold dict, so a bare key (e.g. `total_token_throughput`) + is treated as absent and triggers a coverage failure even though the entry is present. + +- **`client.goodput` is an alias** for stock's `request_goodput` — the names differ; + threshold files must use `client.goodput`, not `client.request_goodput`. + +- **`client_poll_count` controls the total client wait budget**; too low and a long-osl + run times out before the client finishes. Raising it never slows down fast cells. + (Regression: REG-20260609-001) diff --git a/cvs/lib/inference/utils/__init__.py b/cvs/lib/inference/utils/__init__.py new file mode 100644 index 000000000..d3438a6e8 --- /dev/null +++ b/cvs/lib/inference/utils/__init__.py @@ -0,0 +1,4 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. +''' diff --git a/cvs/lib/inference/utils/cache_probe.py b/cvs/lib/inference/utils/cache_probe.py new file mode 100644 index 000000000..4c6457b14 --- /dev/null +++ b/cvs/lib/inference/utils/cache_probe.py @@ -0,0 +1,39 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Model-cache size probing helpers (no pytest dependency). +''' + +from __future__ import annotations + +import shlex + + +def du_bytes(orch, path): + """Bytes under ``path`` in the container. + + Returns 0 when absent or empty. Returns ``None`` when ``du`` cannot run so + callers do not treat infrastructure failure as "model not present". + """ + quoted = shlex.quote(path) + cmd = ( + f"if [ ! -e {quoted} ]; then echo __MISSING__; " + f"elif bytes=$(du -sb {quoted} 2>/dev/null | cut -f1) && [ -n \"$bytes\" ]; " + f"then echo \"$bytes\"; else echo __DU_ERROR__; fi" + ) + out = orch.exec(f"bash -c {shlex.quote(cmd)}") + total = 0 + saw_marker = False + for text in (out or {}).values(): + text = (text or "").strip() + if text == "__DU_ERROR__": + return None + if text == "__MISSING__": + saw_marker = True + continue + if text.isdigit(): + total += int(text) + if saw_marker and total == 0: + return 0 + return total diff --git a/cvs/lib/inference/utils/docs/cell-key-format.md b/cvs/lib/inference/utils/docs/cell-key-format.md new file mode 100644 index 000000000..cb4f7f919 --- /dev/null +++ b/cvs/lib/inference/utils/docs/cell-key-format.md @@ -0,0 +1,116 @@ +# Cell key format + +The cell key is a string that uniquely identifies one sweep cell. It is the single source +of truth shared between: +- `VariantConfig.cell_key()` — generates the key +- `threshold.json` top-level keys — what the key must match +- The test's verdict lookup — reads threshold specs from `variant_config.thresholds` keyed by this string +- pytest parametrize ids — `pytest_generate_tests` builds ids as + `combo_name + "-conc" + concurrency + "-" + metric_short` + (e.g. `w1_isl=1000_osl=1000-conc16-mean_ttft_ms`); the cell key string never + appears in pytest parametrize ids + +--- + +## Format specification + +``` +ISL={isl},OSL={osl},TP={tensor_parallelism},CONC={concurrency} +``` + +**All four fields are required, in this exact order, with no spaces.** + +| Field | Source | Description | +|---|---|---| +| `ISL=` | `SeqCombo.isl` (string from config) | Input sequence length in tokens | +| `OSL=` | `SeqCombo.osl` (string from config) | Output sequence length in tokens | +| `TP=` | `Params.tensor_parallelism` (string from config) | Tensor parallelism degree; comes from `params`, not from the run | +| `CONC=` | `Run.concurrency` (integer from config) | Number of concurrent requests for this run | + +Values are interpolated verbatim from the config fields — no numeric normalization. +`isl` and `osl` are string-typed in the schema; `concurrency` is an integer but +Python's f-string renders it directly. What appears in the config is what appears in +the key. + +**Implementation** (from `VariantConfig.cell_key`): +```python +f"ISL={isl},OSL={osl},TP={self.params.tensor_parallelism},CONC={concurrency}" +``` + +--- + +## Where it is used + +**`threshold.json` top-level keys** — each key names one sweep cell. The coverage +check (`_check_thresholds_cover_sweep`) enforces a two-way match at load time: +every cell key produced by `expected_cells()` must appear as a threshold key, and +every threshold key must name a cell the sweep actually runs. Axis 2 of the same +check verifies every present cell has a spec for every `GATED_METRICS` member. +The spec keys the coverage check looks for are prefixed with `client.` +(e.g., `client.mean_ttft_ms`, `client.output_throughput`) — bare `GATED_METRICS` +names without this prefix will not satisfy the check. + +**`test_metric` verdict lookup** — the test calls +`variant_config.cell_key(isl, osl, concurrency)` to build the key it looks up in +the threshold specs (`variant_config.thresholds.get(cell)`). A format mismatch means +no spec is found, and the test falls through to the record-only branch — PASS with +zero assertions, silently, even under `enforce_thresholds=true`. + +Note: `inf_res_dict` is keyed by the tuple +`(model_id, gpu_arch, isl, osl, combo_name, concurrency)`, which `test_metric` +builds independently before `cell_key` is called. The cell_key string is not used +to index `inf_res_dict`. + +--- + +## Worked example — `w1_llama31_70b_fp8_config.json` + +The config declares `tensor_parallelism: "8"` in `params`, and the sweep has five +cells all at `concurrency: 16`: + +```json +"params": { "tensor_parallelism": "8" }, +"sweep": { + "sequence_combinations": [ + { "name": "w1_isl=1000_osl=1000", "isl": "1000", "osl": "1000" }, + { "name": "w1_isl=8000_osl=1000", "isl": "8000", "osl": "1000" }, + { "name": "w1_isl=1000_osl=8000", "isl": "1000", "osl": "8000" }, + { "name": "w1_isl=1000_osl=4000", "isl": "1000", "osl": "4000" }, + { "name": "w1_isl=5000_osl=1024", "isl": "5000", "osl": "1024" } + ], + "runs": [ + { "combo": "w1_isl=1000_osl=1000", "concurrency": 16 }, + { "combo": "w1_isl=8000_osl=1000", "concurrency": 16 }, + { "combo": "w1_isl=1000_osl=8000", "concurrency": 16 }, + { "combo": "w1_isl=1000_osl=4000", "concurrency": 16 }, + { "combo": "w1_isl=5000_osl=1024", "concurrency": 16 } + ] +} +``` + +`expected_cells()` returns — and `llama31_70b_fp8_threshold.json` must use as top-level keys: + +``` +ISL=1000,OSL=1000,TP=8,CONC=16 +ISL=8000,OSL=1000,TP=8,CONC=16 +ISL=1000,OSL=8000,TP=8,CONC=16 +ISL=1000,OSL=4000,TP=8,CONC=16 +ISL=5000,OSL=1024,TP=8,CONC=16 +``` + +--- + +## Common mistakes + +| Mistake | Effect | +|---|---| +| Adding a space: `ISL=1000, OSL=1000,...` | Key mismatch — no threshold found, no verdict | +| Changing field order: `TP=8,ISL=1000,...` | Key mismatch — no threshold found, no verdict | +| Using a different separator: `ISL=1000\|OSL=1000\|...` | Key mismatch — no threshold found, no verdict | +| Normalizing values: `ISL=1024` when config says `"1000"` | Key mismatch — values are verbatim from the config string | +| Typo in threshold.json key | Axis 1 of `_check_thresholds_cover_sweep` catches this at load time | + +Any key mismatch silently drops the cell — the test finds no spec and falls through to the +record-only branch, reporting PASS with zero assertions. The load-time coverage check +(`_check_thresholds_cover_sweep`) is what catches this before the test runs: when +`enforce_thresholds=true` it raises `ValueError`; when false it emits a warning. diff --git a/cvs/lib/inference/utils/docs/derived-metrics.md b/cvs/lib/inference/utils/docs/derived-metrics.md new file mode 100644 index 000000000..99596a270 --- /dev/null +++ b/cvs/lib/inference/utils/docs/derived-metrics.md @@ -0,0 +1,159 @@ +# Derived metrics + +`to_client_metrics` in `vllm_parsing.py` appends five derived metrics on top of +the stock `vllm bench serve` scalars. Every derived metric is guarded by `_safe_div`. + +--- + +## `_safe_div` contract + +```python +def _safe_div(num, den): + if num is None or den is None: + return None + try: + den = float(den) + if den == 0: + return None + return float(num) / den + except (TypeError, ValueError): + return None +``` + +Returns `None` when: + +- Either operand is `None` (missing key or explicit null in the artifact) +- The divisor is `0` +- Either operand cannot be converted to `float` (`TypeError` or `ValueError`) + +Never raises `TypeError` or `ValueError`; never returns a bogus `0`. Note: `OverflowError` is not caught — callers must ensure artifact values are within the representable float range. + +### What `None` means downstream + +| Context | Behaviour | +|---|---| +| HTML results table | Displayed as `-` | +| Record-only metric (not in `GATED_METRICS`) | Silently recorded; no assertion | +| Gated metric with a threshold spec | `evaluate_all` raises `ThresholdViolation` — the violation string names the metric and states the value is `None` (metric unavailable for this run) | + +A `None` gated metric is always a loud failure, never a silent skip. + +--- + +## Derived metrics + +### `client.per_gpu_throughput` + +**Formula**: `total_token_throughput / tp` + +**Inputs from artifact**: `total_token_throughput` (stock scalar) + +**Out-of-band input**: `tp` — tensor parallelism, passed by the caller from +`params.tensor_parallelism` + +**None when**: `total_token_throughput` is missing or `None`, `tp` is `0` or +unconvertible to `float`. + +**Gated or record-only**: record-only. Secondary throughput used for per-GPU +capacity analysis. `total_token_throughput` (the primary throughput) is gated +instead. + +--- + +### `client.normalized_ttft_ms_per_tok` + +**Formula**: `mean_ttft_ms / isl` + +**Inputs from artifact**: `mean_ttft_ms` (stock scalar) + +**Out-of-band input**: `isl` — input sequence length, passed by the caller from +the sweep `SeqCombo.isl` + +**None when**: `mean_ttft_ms` is missing or `None`, `isl` is `0` or +unconvertible. + +**Gated or record-only**: record-only. Diagnostic derivation — normalises TTFT +by input length to expose prefill efficiency across cells with different ISLs. +The per-quantile `*_ttft_ms` metrics are gated instead. + +--- + +### `client.decode_latency_ratio` + +**Formula**: `p99_itl_ms / p50_itl_ms` + +**Inputs from artifact**: `p99_itl_ms` (stock scalar), `p50_itl_ms` (looked up +via `raw.get("p50_itl_ms")`) + +**Important**: real `vllm bench serve` artifacts emit `median_itl_ms` for the +50th-percentile ITL, **not** `p50_itl_ms`. The source calls +`raw.get("p50_itl_ms")`, so this key is absent from every real artifact and the +denominator is always `None`. As a result, `client.decode_latency_ratio` is +**always `None` on real runs** — it is record-only by design and the `None` +value is silently recorded (no assertion, displayed as `-` in the results +table). + +**None when**: `p50_itl_ms` is absent from the artifact (always, with real +runs), or either input is `None`, or `p50_itl_ms` is `0`. + +**Gated or record-only**: record-only. Diagnostic derivation — intended to +measure tail latency inflation (how much worse p99 ITL is than the median). The +individual quantile ITL metrics (`p95_itl_ms`, `p99_itl_ms`, etc.) are gated +instead. + +--- + +### `client.decode_throughput_p50` + +**Formula**: `1000.0 / median_tpot_ms` + +**Inputs from artifact**: `median_tpot_ms` (stock scalar) + +**None when**: `median_tpot_ms` is missing, `None`, or `0`. + +**Gated or record-only**: record-only. Secondary throughput — converts median +time-per-output-token to tokens-per-second for human-readable throughput +comparisons. `median_tpot_ms` (the latency metric) is gated instead. + +--- + +### `client.success_rate` + +**Formula**: `completed / (completed + failed)` + +**Inputs from artifact**: `completed`, `failed` (both stock scalars) + +**Implementation note**: the denominator is computed as +`completed + failed` only when both are non-`None`; if either is `None`, the +total is set to `None` and `_safe_div` returns `None`. + +```python +total_req = None if completed is None or failed is None else completed + failed +m["client.success_rate"] = _safe_div(completed, total_req) +``` + +**None when**: either `completed` or `failed` is missing or `None`. + +**Gated or record-only**: **gated**. Run health: a floor on the fraction of +requests that completed without error. A success rate below threshold indicates +infrastructure failures, OOM, or timeouts — not a performance regression. + +--- + +## `client.goodput` alias + +```python +m["client.goodput"] = raw.get("request_goodput") +``` + +Stock `vllm bench serve` emits the key `request_goodput`. The results table and +`threshold.json` reference it as `client.goodput` (without the `request_` prefix) +for readability. The alias is set unconditionally alongside the stock +`client.request_goodput` entry that the 1:1 namespace copy also produces. + +If `--goodput` was not passed to `vllm bench serve`, stock emits +`request_goodput: null`, so `client.goodput` will be `None`. + +**Gated or record-only**: record-only. Goodput is only meaningful when a +`GoodputSlo` was configured in the sweep, and the value reflects the SLO input +rather than an independent throughput measurement. diff --git a/cvs/lib/inference/utils/docs/inferencex-atom-parsing.md b/cvs/lib/inference/utils/docs/inferencex-atom-parsing.md new file mode 100644 index 000000000..2954accc5 --- /dev/null +++ b/cvs/lib/inference/utils/docs/inferencex-atom-parsing.md @@ -0,0 +1,37 @@ +# InferenceX ATOM parsing + +`inferencex_atom_parsing.py` owns the **IX W1 SLO contract** (`GATED_METRICS`) and +ATOM-specific derived metrics. It reuses `vllm_parsing.to_client_metrics` for the +stock `benchmark_serving` / `vllm bench serve` JSON scalars because ATOM and vLLM +bench artifacts share the same keys. + +## Drivers and artifacts + +| `params.driver` | Result artifact | Parser entry | +| --- | --- | --- | +| `atom` | `{result_stem}.json` from `benchmark_serving` | `to_client_metrics` | +| `vllm`, `vllm_atom` | vLLM bench `{result_stem}` (no `.json` suffix) | `to_client_metrics` | +| `sglang` | SGLang bench log / artifact (client poll on log today) | `to_client_metrics` when JSON present | + +Multinode **scaling** adds `scaling.efficiency_pct` when +`params.scaling_baseline_output_throughput` is set (multinode PP configs). + +## Why not `vllm_parsing` alone? + +- `vllm_single` keeps its own `GATED_METRICS` (legacy suite). +- W1 IX gates (`per_gpu_throughput`, `output_tput_per_gpu`, tail percentiles) live here. +- Driver choice affects orchestration, not the `client.*` namespace once metrics are parsed. + +## Derived metrics (IX-only display + gates) + +| Metric | Formula | +| --- | --- | +| `client.per_gpu_throughput` | `total_token_throughput / tp` (from shared parser) | +| `client.output_tput_per_gpu` | `output_throughput / tp` (added in this module) | +| `scaling.efficiency_pct` | `output_throughput / (baseline × nnodes) × 100` when baseline set | + +## Consumers + +- `inferencex_atom_orch.InferenceXAtomJob.parse_results` +- `inferencex_atom_config_loader` threshold coverage (`gated_metrics=GATED_METRICS`) +- `cvs.tests.inference.inferencex_atom.inferencex_atom` — `test_cell_metrics` tiers (throughput, ttft, tpot, health, record) diff --git a/cvs/lib/inference/utils/inference_suite_lifecycle.py b/cvs/lib/inference/utils/inference_suite_lifecycle.py new file mode 100644 index 000000000..f81c17440 --- /dev/null +++ b/cvs/lib/inference/utils/inference_suite_lifecycle.py @@ -0,0 +1,225 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Reusable **lifecycle-as-tests** helpers for DTNI inference suites. + +``inferencex_atom`` imports the stage tests from here today; other suites +(``vllm_single``, future IX parity frameworks) can reuse the same module instead +of copying launch / sshd / model-fetch / teardown blocks. + +**Suite module** — import stage tests so pytest collects them:: + + from cvs.lib.inference.utils.inference_suite_lifecycle import ( + test_launch_container, + test_model_fetch, + test_setup_sshd, + test_teardown, + ) + +**conftest.py** — wire shared fixtures and HTML hooks:: + + from cvs.lib.inference.utils.inference_suite_lifecycle import ( + InferenceLifecycle, + attach_lifecycle_html_table, + html_metric_table_header, + html_metric_table_row, + sort_lifecycle_items, + ) + +Also provides ``sweep_cell_result_key``; see :mod:`cvs.lib.inference.utils.cache_probe` for ``du_bytes``. + +Optional HTML/JSON suite report: add ``cvs/lib/report/presets/.py`` (see +``cvs/lib/report/README.md``); root ``cvs/conftest.py`` auto-wires hooks when ``--html`` is set. +''' + +from __future__ import annotations + +import shlex +import time + +import pytest + +try: + import pytest_html +except ImportError: + pytest_html = None + +from cvs.lib import globals +from cvs.lib.inference.utils.cache_probe import du_bytes + +log = globals.log + +FETCH_POLL_COUNT = 80 +FETCH_POLL_WAIT_S = 30 +FETCH_PRESENCE_RETRIES = 5 + + +class InferenceLifecycle: + """Cross-test state shared by lifecycle stage tests in one module scope.""" + + def __init__(self): + self.failed = False + self.torn_down = False + self.report = {} + + def record(self, nodeid, label, value, unit="s"): + self.report.setdefault(nodeid, []).append((label, value, unit)) + + +def sweep_cell_result_key(variant_config, seq_combo, isl, osl, concurrency): + """Canonical ``inf_res_dict`` key for one sweep cell.""" + return ( + variant_config.model.id, + variant_config.gpu_arch, + isl, + osl, + seq_combo.get("name", "default"), + concurrency, + ) + + +def test_launch_container(orch, variant_config, lifecycle, request): + """Stage 1: launch the container.""" + t = time.monotonic() + ok = orch.setup_containers() + lifecycle.record(request.node.nodeid, "container_launch", time.monotonic() - t) + if not ok: + lifecycle.failed = True + name = orch.get_container_name(orch.container_config, orch.container_config["image"]) + pytest.fail(f"setup_containers() returned False for {name}") + name = orch.get_container_name(orch.container_config, orch.container_config["image"]) + if not orch.verify_containers_running(name): + lifecycle.failed = True + pytest.fail(f"container {name} not running after setup_containers()") + + +def test_setup_sshd(orch, lifecycle, request): + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + t = time.monotonic() + ok = orch.setup_sshd() + lifecycle.record(request.node.nodeid, "sshd_setup", time.monotonic() - t) + if not ok: + lifecycle.failed = True + pytest.fail("setup_sshd() returned False") + if len(orch.hosts) > 1: + probe = orch.exec( + "bash -c '" + "(ss -ltn 2>/dev/null || netstat -lnt 2>/dev/null) | grep -q :2224 && echo OK || " + "(pgrep -f \"sshd.*2224\" >/dev/null && echo OK) || echo NO'" + ) + if not any("OK" in (v or "") for v in (probe or {}).values()): + lifecycle.failed = True + pytest.fail("sshd not listening on 2224 after setup_sshd()") + + +def test_model_fetch(orch, variant_config, lifecycle, request): + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + models_dir = variant_config.paths.models_dir + if not models_dir: + pytest.skip("paths.models_dir unset; cannot locate/verify the HF cache") + + remote = getattr(variant_config.model, "remote", 0) + t = time.monotonic() + orch.exec(f"mkdir -p {shlex.quote(models_dir)}") + + if not remote: + final = 0 + for it in range(FETCH_PRESENCE_RETRIES): + cur = du_bytes(orch, models_dir) + if cur is None: + lifecycle.failed = True + pytest.fail(f"could not measure model cache size under {models_dir} (du error)") + final = cur + log.info("[fetch presence %d] size=%.1fGB", it, final / 1e9) + if final > 0: + break + time.sleep(FETCH_POLL_WAIT_S) + else: + fetch = ( + f"HF_HUB_CACHE={shlex.quote(models_dir)} " + f"nohup hf download {shlex.quote(variant_config.model.id)} " + f"> /tmp/hf_fetch.log 2>&1 &" + ) + orch.exec("bash -c " + shlex.quote(fetch)) + prev = -1 + stable = 0 + final = du_bytes(orch, models_dir) + if final is None: + lifecycle.failed = True + pytest.fail(f"could not measure model cache size under {models_dir} (du error)") + for it in range(FETCH_POLL_COUNT): + cur = du_bytes(orch, models_dir) + if cur is None: + lifecycle.failed = True + pytest.fail(f"could not measure model cache size under {models_dir} (du error)") + final = cur + log.info("[fetch poll %d] size=%.1fGB", it, cur / 1e9) + if cur > 0 and cur == prev: + stable += 1 + if stable >= 2: + break + else: + stable = 0 + prev = cur + time.sleep(FETCH_POLL_WAIT_S) + + lifecycle.record(request.node.nodeid, "model_fetch", time.monotonic() - t) + lifecycle.record(request.node.nodeid, "model_size", final / 1e9, "GB") + if final <= 0: + lifecycle.failed = True + pytest.fail(f"no model bytes under {models_dir} after fetch") + + +def test_teardown(orch, lifecycle, request): + name = orch.get_container_name(orch.container_config, orch.container_config["image"]) + t = time.monotonic() + orch.teardown_containers() + lifecycle.record(request.node.nodeid, "teardown", time.monotonic() - t) + if orch.verify_containers_running(name): + pytest.fail(f"container {name} still running after teardown_containers()") + lifecycle.torn_down = True + + +def sort_lifecycle_items(items, rank): + items.sort(key=lambda it: rank.get(it.originalname or it.name.split("[")[0], 99)) + + +def attach_lifecycle_html_table(item, report): + if report.when != "call": + return + lc = item.funcargs.get("lifecycle") + rows = getattr(lc, "report", {}).get(item.nodeid) if lc else None + if not rows: + return + if pytest_html is None: + return + body = "".join(f"{label}{value:.1f}{unit}" for label, value, unit in rows) + html = f"{body}
stagevalueunit
" + extras = getattr(report, "extras", []) + extras.append(pytest_html.extras.html(html)) + report.extras = extras + + +def html_metric_table_header(cells): + cells.insert(-1, "Value") + cells.insert(-1, "Unit") + + +def html_metric_table_row(report, cells): + props = dict(getattr(report, "user_properties", ())) + has = "metric_value" in props + val = props.get("metric_value") + unit = props.get("metric_unit", "") if has else "" + if not has: + shown = "" + elif val is None: + shown = "-" + elif isinstance(val, float): + shown = f"{val:.3f}" + else: + shown = str(val) + cells.insert(-1, f"{shown}") + cells.insert(-1, f"{unit}") diff --git a/cvs/lib/inference/utils/inference_suite_results_table.py b/cvs/lib/inference/utils/inference_suite_results_table.py new file mode 100644 index 000000000..39adf0f2f --- /dev/null +++ b/cvs/lib/inference/utils/inference_suite_results_table.py @@ -0,0 +1,93 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Reusable end-of-session results table for DTNI inference suites. + +Pair with ``cvs.lib.report`` suite reports for an optional HTML/JSON dashboard +that uses the same column preset and ``inf_res_dict`` keys. + +Bind a column preset with ``make_print_results_table(columns)`` and export the +returned callable as ``test_print_results_table`` from the suite module (see +``cvs/tests/inference/inferencex_atom/_shared.py``). Other suites can supply +their own ``(header_label, client.*_key)`` tuples without duplicating the +tabulate loop. +''' + +from __future__ import annotations + +from cvs.lib import globals + +log = globals.log + +# Column tuple: ``(header label, client.* metric key or None for fixed key fields)``. +# First seven columns are always Model, GPU, ISL, OSL, Policy, Conc, Host. + +INFERENCEX_ATOM_RESULTS_COLUMNS = ( + ("Model", None), + ("GPU", None), + ("ISL", None), + ("OSL", None), + ("Policy", None), + ("Conc", None), + ("Host", None), + ("Output tok/s", "client.output_throughput"), + ("Total tok/s", "client.total_token_throughput"), + ("Mean TTFT (ms)", "client.mean_ttft_ms"), + ("Mean TPOT (ms)", "client.mean_tpot_ms"), + ("P99 ITL (ms)", "client.p99_itl_ms"), + ("Scaling eff. (%)", "scaling.efficiency_pct"), +) + +# Optional preset for suites that want vLLM-style columns (not wired in vllm_single yet). +VLLM_SINGLE_RESULTS_COLUMNS = ( + ("Model", None), + ("GPU", None), + ("ISL", None), + ("OSL", None), + ("Policy", None), + ("Conc", None), + ("Host", None), + ("Req/s", "client.request_throughput"), + ("Total tok/s", "client.total_token_throughput"), + ("Mean TTFT (ms)", "client.mean_ttft_ms"), + ("P95 TTFT (ms)", "client.p95_ttft_ms"), + ("Mean TPOT (ms)", "client.mean_tpot_ms"), + ("P95 TPOT (ms)", "client.p95_tpot_ms"), + ("P99 ITL (ms)", "client.p99_itl_ms"), + ("Goodput (req/s)", "client.goodput"), +) + + +def _cell(metrics, key): + v = metrics.get(key) + return "-" if v is None else v + + +def print_results_table(inf_res_dict, columns): + from tabulate import tabulate + + if not inf_res_dict: + log.info("inf_res_dict empty, nothing to print") + return + headers = [label for label, _key in columns] + metric_keys = [key for _label, key in columns] + rows = [] + for key, host_dict in inf_res_dict.items(): + model, gpu, isl, osl, policy, conc = key + fixed = [model, gpu, isl, osl, policy, conc] + for host, metrics in host_dict.items(): + row = list(fixed) + row.append(host) + row.extend(_cell(metrics, mk) if mk else "-" for mk in metric_keys[7:]) + rows.append(row) + log.info("\n" + tabulate(rows, headers=headers, tablefmt="github")) + + +def make_print_results_table(columns): + """Return a ``test_print_results_table(inf_res_dict)`` callable for one column spec.""" + + def test_print_results_table(inf_res_dict): + print_results_table(inf_res_dict, columns) + + return test_print_results_table diff --git a/cvs/lib/inference/utils/inferencing_config_loader.py b/cvs/lib/inference/utils/inferencing_config_loader.py new file mode 100644 index 000000000..725f9f1af --- /dev/null +++ b/cvs/lib/inference/utils/inferencing_config_loader.py @@ -0,0 +1,218 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Inference-specific config schema for the vllm_single suite. + +The framework-agnostic machinery (paths/model/image/container schema, the +3-pass placeholder substitution, the `enforce_thresholds` gate, and the +`substitute_config` file-read helper) lives in `cvs.lib.utils.config_loader`. +This module holds the inference half: the sweep selector +(`SeqCombo`/`Run`/`Sweep`), the goodput SLO, the framework `Params`, the +`server` role, and `VariantConfig(BaseVariantConfig)` with the +ISL/OSL/TP/CONC `cell_key` and its sweep-coverage check. + +`Sweep`/`SeqCombo`/`GoodputSlo`/`Roles`/`cell_key` are inference-generic (any +serving framework sweeps sequence shapes at concurrencies); only `Params` (the +`vllm bench serve` flags) is framework-flavored and is the seam to subclass +when a second serving framework lands. +''' + +from __future__ import annotations + +import warnings +from collections import Counter +from typing import Any, Dict, List, Optional + +from pydantic import model_validator +from typing_extensions import Literal + +from cvs.lib.utils.config_loader import BaseVariantConfig, _Forbid, substitute_config +from cvs.lib.inference.utils.vllm_parsing import GATED_METRICS + + +# ---------- pydantic models (inference) ---------- + + +class RoleServer(_Forbid): + # The `vllm serve` command is built in Python (cvs.lib.inference.vllm_single), + # not cloned from a `.sh` script, so a run is self-contained. Per-model + # server quirks live here: extra `vllm serve` flags (serve_args) and env vars + # merged over the defaults the orchestrator sets. Both default empty; the + # fp8-kv cell sets its --kv-cache-dtype via serve_args, kept out of the + # generic driver so it stays model-agnostic. A {flag: value} map (flag + # without the leading --): a scalar renders `--flag value`, True renders a + # bare `--flag`, and a list renders the flag once per element. Cleaner than + # a flat [flag, value, flag, value] list and still covers vllm's bare and + # repeatable flags. + serve_args: Dict[str, Any] = {} + env: Dict[str, str] = {} + + +class Roles(_Forbid): + server: RoleServer = RoleServer() + + +class GoodputSlo(_Forbid): + # Per-cell SLOs for the goodput gate, in milliseconds. An INPUT to the run + # (passed to `vllm bench serve --goodput`), NOT a threshold to assert -- so + # it lives in the sweep, not threshold.json. Attached per seq_combo because + # e2el scales ~linearly with osl. _Forbid: a typo'd key fails load, not a + # silently-dropped SLO. + ttft_ms: float + tpot_ms: float + e2el_ms: float + + +class SeqCombo(_Forbid): + # `name` is the join key the `runs` selector references; required. + name: str + isl: str + osl: str + goodput_slo: Optional[GoodputSlo] = None + + +class Run(_Forbid): + # One sweep cell: a named combo run at a single concurrency. The explicit + # list of Runs replaces the old `sequence_combinations x concurrency_levels` + # cartesian -- you enumerate exactly the cells you want, no NxM explosion. + combo: str + concurrency: int + + +def validate_thresholds_cover_sweep( + *, + expected_cells, + thresholds, + enforce_thresholds: bool, + gated_metrics=None, +) -> None: + """Shared sweep/threshold coverage check for inference variant configs.""" + expected = set(expected_cells) + present = set(thresholds.keys()) + missing = sorted(expected - present) + extra = sorted(present - expected) + problems = [] + if missing: + problems.append(f"sweep cells with no threshold entry: {missing}") + if extra: + problems.append(f"threshold keys matching no sweep cell (typo?): {extra}") + gated = gated_metrics if gated_metrics is not None else GATED_METRICS + gated_keys = [f"client.{m}" for m in sorted(gated)] + gated_gaps = {} + for cell in sorted(expected & present): + specs = thresholds.get(cell) or {} + absent = [k for k in gated_keys if k not in specs] + if absent: + gated_gaps[cell] = absent + if gated_gaps: + problems.append(f"cells missing gated-metric specs: {gated_gaps}") + if problems: + msg = "threshold.json does not match the sweep matrix; " + "; ".join(problems) + if enforce_thresholds: + raise ValueError(msg) + warnings.warn(f"{msg} (enforce_thresholds=false -> record-only)", stacklevel=3) + + +def validate_sweep_selector(combo_names, run_combo_refs): + """The sweep-selector rule: combo names unique, every run.combo names one. + + The single home for this check, shared by the typed `Sweep` validator (load + time) and `pytest_generate_tests` (collection time, which reads raw JSON + before the loader runs) so the two can never drift. Operates on plain lists + of strings so both call sites can feed it. + + Without it a duplicate name silently shadows a combo and a typo'd + `run.combo` is a silently-dropped cell -- either way the sweep runs a + different matrix than the config reads. + """ + counts = Counter(combo_names) + dupes = sorted(name for name, count in counts.items() if count > 1) + if dupes: + raise ValueError(f"duplicate sequence_combination names: {dupes}") + known = set(counts) + unknown = sorted({r for r in run_combo_refs if r not in known}) + if unknown: + raise ValueError(f"run.combo names no sequence_combination: {unknown} (known: {sorted(known)})") + + +class Sweep(_Forbid): + sequence_combinations: List[SeqCombo] + runs: List[Run] + + @model_validator(mode="after") + def _check_runs_reference_known_combos(self): + validate_sweep_selector( + [c.name for c in self.sequence_combinations], + [r.combo for r in self.runs], + ) + return self + + +class Params(_Forbid): + backend: str = "vllm" + base_url: str = "http://0.0.0.0" + port_no: str = "8888" + dataset_name: str = "random" + burstiness: str = "1.0" + seed: str = "0" + request_rate: str = "inf" + random_range_ratio: str = "0.8" + random_prefix_len: str = "0" + tensor_parallelism: str = "1" + tokenizer_mode: str = "auto" + percentile_metrics: str = "ttft,tpot,itl,e2el" + metric_percentiles: str = "50,90,95,99" + num_prompts: str = "3200" + # Completion-poll budget for the bench client = client_poll_count * 60s + # (plus a 120s initial wait). Large-output cells (high osl) need a bigger + # budget; the poll loop exits as soon as the client finishes, so raising + # this never slows down fast cells. See regressions REG-20260609-001. + client_poll_count: str = "20" + + +class VariantConfig(BaseVariantConfig): + framework: Literal["vllm_single"] + gpu_arch: str + roles: Roles = Roles() + params: Params + sweep: Sweep + + def cell_key(self, isl, osl, concurrency): + """The canonical threshold key for one sweep cell. + + Single source of truth shared by the loader's coverage check and the + test's verdict lookup -- so the two can never drift on whitespace, + ordering, or field names. + """ + return f"ISL={isl},OSL={osl},TP={self.params.tensor_parallelism},CONC={concurrency}" + + def expected_cells(self): + """Every (isl, osl, conc) cell the sweep's `runs` selector picks.""" + by_name = {c.name: c for c in self.sweep.sequence_combinations} + return [self.cell_key(by_name[r.combo].isl, by_name[r.combo].osl, r.concurrency) for r in self.sweep.runs] + + @model_validator(mode="after") + def _check_thresholds_cover_sweep(self): + """Fail at load time if any sweep cell lacks a threshold entry.""" + validate_thresholds_cover_sweep( + expected_cells=self.expected_cells(), + thresholds=self.thresholds, + enforce_thresholds=self.enforce_thresholds, + ) + return self + + +# ---------- public API (inference) ---------- + + +def load_variant(config_path, cluster_dict): + """Load and validate a vllm_single variant config + its sibling threshold file. + + Delegates the file read + placeholder substitution + threshold discovery to + the generic `substitute_config`, then attaches the thresholds and builds the + typed `VariantConfig`. + """ + raw, thresholds = substitute_config(config_path, cluster_dict) + raw["thresholds"] = thresholds + return VariantConfig(**raw) diff --git a/cvs/lib/inference/utils/vllm_benchmark_scripts/README.md b/cvs/lib/inference/utils/vllm_benchmark_scripts/README.md new file mode 100644 index 000000000..9ca413a58 --- /dev/null +++ b/cvs/lib/inference/utils/vllm_benchmark_scripts/README.md @@ -0,0 +1,17 @@ +# vLLM benchmark server scripts (shared) + +Shell entrypoints for **`vllm serve`** kept for legacy **InferenceBaseJob** flows. + +- **`vllm_serve_mi300x.sh`** — reference MI300-class server flags (``enforce-eager``, ``gpu-memory-utilization``, etc.). **InferenceX ATOM** and **vllm_single** encode equivalent flags in ``roles.server.serve_args`` instead of running this script. + +**Client benchmarks** use ``vllm bench serve`` (stock results artifact). CVS no longer clones a third-party ``bench_serving`` git repo for InferenceX ATOM. + +If **both** the script path and the bench CLI are unavailable, install bench-capable vLLM in the image (e.g. `pip install 'vllm[bench]'`) or bind-mount a matching `benchmarks/` tree from a vLLM checkout. + +Client invocations also pass ``--temperature 0`` so greedy sampling matches the legacy +``benchmark_serving.py`` default, and clamp ``--random-range-ratio`` when the configured +spread would let ``(ISL+OSL)*(1+r)`` exceed ``max_model_length`` (otherwise vLLM rejects +most random prompts). + +Python API: ``bundled_scripts_dir()``, ``bash_export_bench_script_from_vllm_install()``, +``clamped_bench_random_range_ratio_str()``, ``validated_bench_script_basename()``. diff --git a/cvs/lib/inference/utils/vllm_benchmark_scripts/__init__.py b/cvs/lib/inference/utils/vllm_benchmark_scripts/__init__.py new file mode 100644 index 000000000..3d80431d2 --- /dev/null +++ b/cvs/lib/inference/utils/vllm_benchmark_scripts/__init__.py @@ -0,0 +1,200 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Canonical **vLLM benchmark server** shell helpers retained for legacy +:class:`~cvs.lib.inference.base.InferenceBaseJob` flows. **InferenceX ATOM** and +**vllm_single** (:class:`~cvs.lib.inference.vllm_single.VllmJob`) build +``vllm serve`` in Python via ``roles.server.serve_args``; they do not stage +these scripts at runtime. + +Client load generation uses the **vLLM install’s** ``benchmarks/ + + + + +
+
+
+

__TITLE__

+

__SUBTITLE__

+
+ — +
+ + + +
+

Overview

+
+
+ +
+

Filters

+
+ + + + + + + + + + +
+
Loading…
+
+ + + + + + + +
+

Cells

+
+ + + +
CellISLOSLPolicyHostCThroughputTTFTTPOTStatus
+
+
+
+ +__EMBEDDED_JSON__ + + + + diff --git a/cvs/lib/report/viewer/scaffold.py b/cvs/lib/report/viewer/scaffold.py new file mode 100644 index 000000000..e39996c94 --- /dev/null +++ b/cvs/lib/report/viewer/scaffold.py @@ -0,0 +1,52 @@ +'''Interactive viewer: filterable table, Chart.js concurrency charts, heatmap, gate matrix.''' + +from __future__ import annotations + +import html +import json +from pathlib import Path +from typing import Any, Mapping, Optional + +from cvs.lib.report.artifacts import export_payload + +_TEMPLATE_PATH = Path(__file__).with_name("interactive.html") + + +def _embedded_json_script(payload: Mapping[str, Any]) -> str: + """Inline JSON so the viewer works when opened via file:// (fetch is blocked).""" + raw = json.dumps(export_payload(payload), separators=(",", ":"), default=str) + # Keep serialized JSON from closing the surrounding ' + + +def write_interactive_viewer( + out_html: Path, + *, + json_basename: str, + title: str, + subtitle: str = "Interactive sweep explorer (loads sibling JSON sidecar)", + tier_order: tuple[str, ...] = ("throughput", "record"), + embed_payload: Optional[Mapping[str, Any]] = None, +) -> Path: + """Write a static viewer HTML that loads report data embedded or via sibling JSON.""" + out_html = Path(out_html) + out_html.parent.mkdir(parents=True, exist_ok=True) + template = _TEMPLATE_PATH.read_text(encoding="utf-8") + if not template.strip(): + raise FileNotFoundError(f"Viewer template is empty or missing: {_TEMPLATE_PATH}") + tier_js = "[" + ",".join(json.dumps(t) for t in tier_order) + "]" + embedded = _embedded_json_script(embed_payload) if embed_payload is not None else "" + doc = ( + template.replace("__TITLE__", html.escape(title)) + .replace("__SUBTITLE__", html.escape(subtitle)) + .replace("__JSON_PATH__", json.dumps(json_basename)) + .replace("__TIER_ORDER__", tier_js) + .replace("__EMBEDDED_JSON__", embedded) + ) + out_html.write_text(doc, encoding="utf-8") + return out_html + + +def viewer_basename_for(report_basename: str) -> str: + return f"{report_basename}_viewer.html" diff --git a/cvs/lib/report_plugins.py b/cvs/lib/report_plugins.py index f4a3f2158..75a5e3ab4 100644 --- a/cvs/lib/report_plugins.py +++ b/cvs/lib/report_plugins.py @@ -6,6 +6,7 @@ ''' import datetime +import html import re import shutil import sys @@ -92,12 +93,12 @@ def write_test_log(self, report, test_name=None): log_content = [] for section_name, section_content in report.sections: - log_content.append(f"

{section_name}

{section_content}
") + log_content.append(f"

{html.escape(section_name)}

{html.escape(section_content)}
") if log_content: # Persist a standalone html log page per test. log_path.write_text( - f"

{report.nodeid}

{''.join(log_content)}", + f"

{html.escape(report.nodeid)}

{''.join(log_content)}", encoding="utf-8", ) log.info("Wrote external test log: %s", log_path) @@ -356,17 +357,53 @@ def generate_reports_section(self): if not self._custom_test_reports: return "" - # Simple Reports section - only test reports, no config files html = '

Reports

    ' - - # Add test reports only for report in self._custom_test_reports: - html += f'
  • {report["name"]}
    • ' - - html += '
' + path = html.escape(str(report["path"])) + name = html.escape(str(report["name"])) + html += f'
  • {name}
    • ' + html += "" return html + def generate_suite_reports(self, session): + """Write registered suite report HTML/JSON into the pytest bundle before zip.""" + if not self.is_enabled: + return + + from cvs.lib.report.inference import publish_inference_suite_report + from cvs.lib.report.registry import get_session_results, get_suite_report_config + from cvs.lib.report.types import InferenceReportConfig + + report_config = get_suite_report_config(session.config) + if report_config is None: + suite_name = getattr(session.config, "_suite_name", "unknown") + log.info( + "Skipping suite report generation: no preset registered for suite '%s'", + suite_name, + ) + return + + store = get_session_results() + + if not isinstance(report_config, InferenceReportConfig): + log.warning("Unknown suite report config type: %s", type(report_config).__name__) + return + + inf_res_dict = store.get("inf_res_dict") + if not inf_res_dict: + log.info("Skipping suite report generation: no results in session store") + return + + publish_inference_suite_report( + report_config, + variant_config=store.get("variant_config"), + inf_res_dict=inf_res_dict, + lifecycle_report=store.get("lifecycle_report") or {}, + report_manager=self, + pytest_config=session.config, + ) + @staticmethod def inject_style_overrides(prefix): """Inject CSS to hide show/hide details UI elements.""" diff --git a/cvs/lib/utils/AGENTS.md b/cvs/lib/utils/AGENTS.md new file mode 100644 index 000000000..7eeb70d64 --- /dev/null +++ b/cvs/lib/utils/AGENTS.md @@ -0,0 +1,265 @@ +# cvs/lib/utils — framework-agnostic config machinery + +**Boundary**: if every CVS suite (inference, training, ...) needs it, it belongs here. +Inference-only symbols belong in `cvs/lib/inference/utils/`; single-framework symbols in `cvs/lib//utils/`. + +--- + +## Files + +### `config_loader.py` + +#### Schema classes + +| Class | Models | Extra keys | +|---|---|---| +| `_Forbid` | Base: extra keys forbidden | — | +| `_Allow` | Base: extra keys allowed | — | +| `Paths` | Shared filesystem paths | `_Forbid` | +| `ModelSpec` | Model identity and fetch mode | `_Forbid` | +| `RuntimeSpec` | Orchestrator runtime (name + open-ended args) | `_Allow` | +| `ContainerSpec` | Container lifecycle and image | `_Forbid` | +| `BaseVariantConfig` | Framework-agnostic skeleton all suites share | `_Forbid` | + +**`Paths`** — `shared_fs: str`, `models_dir: str`, `log_dir: str`, `hf_token_file: str` + +**`ModelSpec`** — `id: str`, `remote: Literal[0, 1]` + +**`RuntimeSpec`** (`_Allow`) — `name: str`, `args: Dict[str, Any]` (defaults to `{}`). +`_Allow` because orchestrator runtime options are framework-specific. + +**`ContainerSpec`** (`_Forbid`): + +| Field | Type | Default | Meaning | +|---|---|---|---| +| `lifetime` | `Literal["no_launch", "per_run", "persistent"]` | `"per_run"` | `"no_launch"` — skip container management entirely; `"per_run"` — tear down and re-create each run; `"persistent"` — reuse an already-running container | +| `name` | `str` | required | Container name | +| `image` | `str` | required | Declared once here; no separate top-level image block | +| `runtime` | `RuntimeSpec` | required | Nested `RuntimeSpec`; its serialised form inside `container.model_dump()` is `{name, args}` — the full container dump is `{lifetime, name, image, runtime: {name, args}}` | + +**`BaseVariantConfig`** (`_Forbid`) shared fields: + +| Field | Type | Default | Notes | +|---|---|---|---| +| `schema_version` | `Literal[1]` | required | — | +| `enforce_thresholds` | `bool` | `True` | `False` → coverage failures become warnings; test runs record-only | +| `threshold_json` | `str` | required | Literal absolute path; see contract below | +| `paths` | `Paths` | required | — | +| `model` | `ModelSpec` | required | — | +| `container` | `ContainerSpec` | required | — | +| `thresholds` | `Dict[str, Dict[str, Any]]` | `{}` | Populated by the loader, not the config file | + +`BaseVariantConfig` carries one `@model_validator(mode="after")`: + +**`_check_remote_not_implemented`** — raises `NotImplementedError` when `model.remote == 1`. +Runs first (parent-class validators precede subclass validators), so a remote config fails fast +before any subclass coverage check runs on a config that will be rejected anyway. + +--- + +#### `substitute_config(config_path, cluster_dict) -> (raw_dict, thresholds)` + +- **Accepts**: path to a variant `_config.json` + a resolved cluster dict +- **`threshold_json` handling**: read as a literal absolute path from the raw (un-substituted) config + before any substitution pass runs — **no placeholder substitution of any kind is applied to it** +- **3-pass substitution** (see `docs/placeholder-substitution.md` for worked example): + 1. Cluster placeholders (`{user-id}`, etc.) resolved everywhere in the document + 2. Self-reference within the `paths` block (`{shared_fs}` expanded inside other `paths.*` values) + 3. Cross-block references (`{paths.models_dir}`, etc.) resolved anywhere in the document +- **Strips** `_`-prefixed comment keys from thresholds before returning +- **Returns**: substituted-but-**unvalidated** dict + parsed thresholds +- **Does NOT**: validate, type-coerce, or build a typed config — that is the caller's job +- **Unknown `{token}`**: left verbatim (no error; typo surfaces as a literal brace in a path) + +--- + +#### `_resolve_cluster_mapping(cluster_dict)` + +- Returns `{"user-id": }` +- Falls back to `getpass.getuser()` when `cluster_dict` has no `username` key (or it is falsy) +- This is how `{user-id}` resolves on clusters without an explicit `username` field + +--- + +### `verdict.py` + +**`ThresholdViolation(Exception)`** +- `.violations: list[str]` — all failure strings +- Exception message is the violation strings joined by newlines + +**`evaluate_all(actuals, thresholds)`** + +| Situation | Behaviour | +|---|---| +| Metric in `thresholds` but not in `actuals` | Violation string (not `KeyError`) | +| Metric present in `actuals` with value `None` | Loud violation string (not `float(None)` crash) | +| Metric present in `actuals` with a non-numeric, non-None value | Uncaught `ValueError` from `float()` — callers must ensure actuals values are numeric or `None` | +| `min_ratio` spec | `evaluate_all` injects `_actuals` into the spec dict before calling `_check_one`; callers never set `_actuals` | +| `min_ratio` — **reference** metric value is `None` | Caught inside `_check_one` (not in `evaluate_all`'s per-metric guard); `_check_one` returns a violation string when `actuals[ref_metric] is None`. The `evaluate_all` `None` guard covers only the **primary** metric being checked, not the reference metric for ratio specs. | +| Multiple failures | Raises `ThresholdViolation` listing ALL failures, not just the first | + +- `actuals`: `{metric_name: value_or_None}` +- `thresholds`: `{metric_name: spec_dict}` + +See `docs/threshold-kinds.md` for the full threshold kind reference. + +--- + +### `gpu.py` + +GPU metrics polling library. No side-effects at import time; safe to import in any suite — +inference or training. Shells out to `amd-smi metric --json` via an `Orchestrator`; no +suite-specific logic. See `docs/gpu-metrics.md` for the integration guide. + +**When to use**: add GPU utilisation rows to any suite's HTML report. +Do not copy-paste this logic — import it. + +#### Public API + +| Symbol | Kind | Purpose | +|---|---|---| +| `GPU_METRICS` | `list[tuple[str, str]]` | 5 derived metric keys + units, in display order. Iterate to register `test_gpu_metric` parametrize IDs and threshold keys. | +| `GPU_METRIC_UNITS` | `dict[str, str]` | `{key: unit}` convenience dict built from `GPU_METRICS`. | +| `capture_gpu_metrics(orch, nodes=None)` | function | One `amd-smi metric --json` exec round. Returns `{gpu.*: value_or_None}` merged snapshot. | +| `agg_readings(readings)` | function | Aggregates a list of raw snapshots → `{peak_gpu_memory_mb, gpu_compute_util_pct, gpu_bandwidth_util_pct}`. | +| `poll_gpu_metrics(orch, is_done_fn, ...)` | function | Polling loop. Returns list of raw snapshots. Never raises. | + +#### Single-node vs multi-node + +`capture_gpu_metrics` and `poll_gpu_metrics` both take an optional `nodes` parameter. + +- **`nodes=None` (default, single-node)**: `orch` must implement `.exec_on_head(cmd) -> {host: str}`. + amd-smi runs once, on the orchestrator's head node. +- **`nodes` provided (multi-node)**: `nodes` is a `list[(label, hosts)]`, where `hosts` is a + list of hostnames. `orch` must implement `.exec(cmd, hosts=hosts) -> {host: str}` — every + `Orchestrator` subclass (`BaremetalOrchestrator`, `ContainerOrchestrator`, ...) already + supports this. One `amd-smi` exec runs per `(label, hosts)` pair per poll; all nodes' GPU + entries are merged into a single snapshot before aggregation, and the last successful + per-node VRAM reading is tracked separately for the summary block. + See `cvs/lib/inference/sglang_disagg_lib.py::sglang_disagg_gpu_counts` for a role-based + usage example (prefill/decode/router/benchmark node groups). + +Do not construct raw `Pssh`/ssh handles per node — pass hostnames through `nodes` and let +`orch.exec(cmd, hosts=...)` route the call; this keeps polling orchestrator-agnostic. + +#### `poll_gpu_metrics` parameters + +| Parameter | Default | Notes | +|---|---|---| +| `orch` | — | `Orchestrator`; must have `.exec_on_head(cmd)` and, for multi-node, `.exec(cmd, hosts=...)` | +| `is_done_fn` | — | Callable returning `bool`; polling stops when it returns `True`. Runs outside the amd-smi try/except, so an exception here always propagates and is never misattributed as a polling failure. | +| `poll_interval_s` | `15` | Seconds between polls | +| `label` | `"poll"` | Log-line prefix tag | +| `log_path` | `None` | If given, writes `gpu_poll.log` to this path | +| `max_consecutive_failures` | `3` | Stops early after this many back-to-back `amd-smi` failures | +| `model_load_s` | `None` | Passed through into the summary block of `gpu_poll.log` | +| `model_load_memory_mb` | `None` | Passed through into the summary block of `gpu_poll.log` | +| `nodes` | `None` | Optional `list[(label, hosts)]` for multi-node polling; see above | + +`poll_gpu_metrics` returns the raw readings list. The caller computes the 5 derived +metrics by combining `agg_readings(readings)` with the separately-measured +`model_load_s` and `model_load_memory_mb` scalars. + +#### The 5 derived metrics and how they are computed + +| Key | Source | Aggregation | +|---|---|---| +| `peak_gpu_memory_mb` | `agg_readings` | `max(used_vram)` over polls, each poll summed across GPUs/nodes | +| `model_load_memory_mb` | caller-measured | `post_load_snap["gpu.used_vram"] - pre_load_snap["gpu.used_vram"]` | +| `model_load_s` | caller-measured | wall-clock elapsed while server starts | +| `gpu_bandwidth_util_pct` | `agg_readings` | `mean(umc_activity)` over polls, each poll averaged across GPUs/nodes | +| `gpu_compute_util_pct` | `agg_readings` | `mean(gfx_activity)` over polls, each poll averaged across GPUs/nodes | + +Store as `inf_res_dict[f"gpu.{key}"]` so a `test_gpu_metric`-style test can retrieve them. + +#### Gotchas + +- **`amd-smi` runs on the host, not in the container.** Single-node: use `orch.exec_on_head(...)`, + never `orch.exec_in_container(...)`. Multi-node: use `orch.exec(cmd, hosts=[...])` — same + host-side constraint, just targeted at a specific host subset. +- **`capture_gpu_metrics` can raise**; only `poll_gpu_metrics` guarantees never-raises. + Wrap one-shot snapshot calls in a `try/except` that returns `{}`. +- **`model_load_memory_mb` should be `None` when VRAM data is unavailable**, not `0`. + Use `... or None` after the subtraction so a missing-data case is skipped rather than + gated as a zero value. +- **`agg_readings` only returns 3 of the 5 metrics.** `model_load_memory_mb` and + `model_load_s` come from the caller's timing and snapshot code, not from the poll loop. +- **All poll readings use raw `gpu.*` keys** (e.g. `gpu.used_vram`), not derived metric + keys (e.g. `peak_gpu_memory_mb`). Do not pass raw snapshots to `evaluate_all`. +- **Multi-node degrades per label, not globally.** If `orch.exec` raises for one node in + `nodes`, that node's entries are excluded from the merged snapshot and its per-node VRAM + is `None`; other nodes' data is unaffected. + +--- + +## The boundary rule + +| Question | Answer | +|---|---| +| Does every CVS suite need it? | `cvs/lib/utils/` | +| Do only serving/inference suites need it? | `cvs/lib/inference/utils/` | +| Does only one framework (vllm, megatron, jax) need it? | `cvs/lib//utils/` | + +When in doubt: "does any other suite need this?" → move it up one layer. + +--- + +## Subclassing `BaseVariantConfig` + +Contract for new suite authors: + +**Must add:** +- `framework: Literal["your_name"]` +- `params` — your framework's CLI flags schema +- `sweep` — your sweep schema + +**Must implement:** +- `cell_key(...)` — returns a string key matching threshold.json top-level keys +- `expected_cells()` — returns a list of all cell keys the sweep produces + +**Must add:** +- A `@model_validator(mode="after")` that performs threshold-coverage checking + (equivalent to `_check_thresholds_cover_sweep` in `inferencing_config_loader.py`). + The check must cover **two axes**: + 1. **Cell coverage** — sweep cells with no threshold entry AND threshold keys + that match no sweep cell (both directions; a one-way check silently skips + orphaned threshold entries). + 2. **Gated-metric coverage** — for every cell that is present in both the sweep + and the threshold file, every member of the framework's gated-metric set must + have a spec. Without this axis, a gated metric with no spec falls through the + `spec is None` record-only branch and silently reports a green PASS with zero + assertions even under `enforce_thresholds=True`. For the vllm/inference + framework this set is `GATED_METRICS` imported from + `cvs.lib.inference.utils.vllm_parsing`; a new framework author must define an + equivalent set. + +**Validator ordering:** parent-class validators run before subclass validators. +`_check_remote_not_implemented` always fires first — do not add a base validator that +assumes a valid config before this check passes. + +**`load_variant`:** always delegate to `substitute_config` — never reimplement file-read +or substitution. After calling `substitute_config`, attach `thresholds`, then build +`YourVariantConfig(**raw)`. + +--- + +## Gotchas + +- **`threshold_json` is a literal absolute path** — not a glob, not relative to the config + file. It is read from the raw un-substituted config before Pass 1 runs, so no placeholder + substitution (not even `{user-id}`) applies to it. If your threshold path needs to vary + by user, it must be pre-resolved before being written into the config file. +- **Unknown `{token}` left verbatim** — a typo surfaces as a literal brace in a path at + runtime, not a load failure. Check paths block values after loading if substitution is + suspected to have silently failed. +- **`_Forbid` vs `_Allow`**: never loosen `_Forbid` to silence an "extra key" validation + error — add the field explicitly. +- **Validator ordering**: `BaseVariantConfig` validators run before subclass validators; + `_check_remote_not_implemented` always fires first. Do not add a subclass validator that + assumes `model.remote == 0` without relying on this ordering guarantee. +- **`_resolve_cluster_mapping` fallback**: if the running user differs from the cluster user, + verify `cluster_dict` has a `username` key; omitting it silently resolves `{user-id}` to + the local OS user. +- **`container.model_dump()` is the orchestrator contract** — serialises to + `{lifetime, name, image, runtime: {name, args}}` that `OrchestratorConfig.from_configs` + consumes; do not reshape the dict before passing it. diff --git a/cvs/lib/utils/__init__.py b/cvs/lib/utils/__init__.py new file mode 100644 index 000000000..d3438a6e8 --- /dev/null +++ b/cvs/lib/utils/__init__.py @@ -0,0 +1,4 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. +''' diff --git a/cvs/lib/utils/config_loader.py b/cvs/lib/utils/config_loader.py new file mode 100644 index 000000000..d0215c044 --- /dev/null +++ b/cvs/lib/utils/config_loader.py @@ -0,0 +1,224 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Framework-agnostic config machinery shared by every CVS suite. + +Holds the generic half of what used to be one monolithic loader: the +container/paths/model/image schema, the 3-pass placeholder substitution, the +`enforce_thresholds` gate carried on `BaseVariantConfig`, and the +`substitute_config` helper that reads a variant `config.json` + sibling +`*threshold.json` and resolves placeholders. A per-framework module subclasses +`BaseVariantConfig` and adds its own `Params`/`Sweep`/`cell_key` (see +`cvs.lib.inference.utils.vllm_config_loader` for the vllm flavour). + +3-pass placeholder substitution: + 1. cluster placeholders (`{user-id}`) anywhere + 2. self-reference within `paths` (e.g. `{shared_fs}`) + 3. cross-block (`{paths.models_dir}`, etc.) into the rest of the doc + +A loaded variant's `container` field (`lifetime`, `name`, `image`, `runtime`) +matches the dict shape that `cvs.core.orchestrators.factory.OrchestratorConfig` +already understands. `runtime` is a nested `RuntimeSpec`, not a flat dict; +`container.model_dump()` serialises it to the `runtime: {name, args}` shape the +factory consumes (the vllm conftest does exactly this before building the +orchestrator). The loaded variant also carries a `thresholds` field with the +parsed threshold contents. + +`model.remote=1` raises NotImplementedError -- schema is present, but the +download/resolve logic lives in cvs-dtni-v1's `resource_resolver.py` and is +out of scope for this PoC. +''' + +from __future__ import annotations + +import getpass +import json +import re +from pathlib import Path +from typing import Any, Dict + +from pydantic import BaseModel, ConfigDict, Field, model_validator +from typing_extensions import Literal + + +# ---------- pydantic models (generic) ---------- + + +class _Forbid(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class _Allow(BaseModel): + model_config = ConfigDict(extra="allow") + + +class Paths(_Forbid): + shared_fs: str + models_dir: str + log_dir: str + hf_token_file: str + + +class ModelSpec(_Forbid): + id: str + remote: Literal[0, 1] + precision: str = "" + + +class RuntimeSpec(_Allow): + name: str + args: Dict[str, Any] = Field(default_factory=dict) + + +class ContainerSpec(_Forbid): + lifetime: Literal["no_launch", "per_run", "persistent"] = "per_run" + name: str + image: str + runtime: RuntimeSpec + + +class BaseVariantConfig(_Forbid): + """The framework-agnostic skeleton of a variant config. + + Carries the fields every suite shares (schema/paths/model/image/container/ + thresholds + the enforce gate) and the remote-not-implemented guard. + Per-framework subclasses add their own `framework`/`Params`/`Sweep` and the + `cell_key`/coverage-check pair that depend on them. + """ + + schema_version: Literal[1] + # When false, the threshold-coverage gate warns instead of raising and the + # test records metrics without asserting pass/fail (record-only). Use for + # un-calibrated shapes (e.g. a throughput characterization whose published + # numbers are curves, not tabulated values). Default true keeps the gate + # strict for calibrated configs -- no regression to the remediation work. + enforce_thresholds: bool = True + threshold_json: str = "" + paths: Paths + model: ModelSpec + # The container image is declared once, on container.image (ContainerSpec). + # There is no separate top-level image block. + container: ContainerSpec + thresholds: Dict[str, Dict[str, Any]] = Field(default_factory=dict) + + # pydantic runs @model_validator(mode="after") hooks in definition order, + # parent-class hooks before subclass hooks. This remote check is intentionally + # the first to run: an unimplemented remote config fails fast + # (NotImplementedError) before any subclass's threshold-coverage check runs, + # which is meaningless for a config we are going to reject anyway. + @model_validator(mode="after") + def _check_remote_not_implemented(self): + if self.model.remote == 1: + raise NotImplementedError( + "model.remote=1 (remote model download) is not implemented in the PoC. " + "Port from cvs-dtni-v1/resource_resolver.py before enabling." + ) + return self + + +# ---------- placeholder substitution ---------- + +_PLACEHOLDER_RE = re.compile(r"\{([a-zA-Z0-9_.\-]+)\}") + + +def _walk_substitute(node, mapping): + if isinstance(node, str): + + def repl(m): + key = m.group(1) + if key in mapping: + return str(mapping[key]) + return m.group(0) + + return _PLACEHOLDER_RE.sub(repl, node) + if isinstance(node, list): + return [_walk_substitute(x, mapping) for x in node] + if isinstance(node, dict): + return {k: _walk_substitute(v, mapping) for k, v in node.items()} + return node + + +def _flatten_paths(d, prefix=""): + out = {} + for k, v in d.items(): + key = f"{prefix}.{k}" if prefix else k + if isinstance(v, dict): + out.update(_flatten_paths(v, key)) + elif isinstance(v, (str, int, float)): + out[key] = str(v) + return out + + +def _resolve_cluster_mapping(cluster_dict): + raw = cluster_dict.get("username") or "{user-id}" + user = getpass.getuser() if raw == "{user-id}" else raw + return {"user-id": user} + + +# ---------- public API (generic) ---------- + + +def substitute_config(config_path, cluster_dict): + """Read a variant config + sibling threshold file and resolve placeholders. + + Returns `(raw_dict, thresholds)`: the substituted config dict (NOT yet + validated -- the caller's per-framework `VariantConfig(**raw)` does that) + and the parsed, comment-stripped threshold dict. + + Threshold discovery supports both layouts: + - ``threshold_json`` in the config (literal path; vllm_single style), or + - a sole ``*threshold.json`` sibling next to the config (inferencex_atom style). + + This is the framework-neutral body of the old `load_variant`: file read + + 3-pass substitution + threshold read. Per-framework loaders call it, attach + ``thresholds``, then build their typed config. + """ + config_path = Path(config_path) + if not config_path.is_file(): + raise FileNotFoundError(f"variant config not found: {config_path}") + + raw = json.loads(config_path.read_text()) + + threshold_json = (raw.get("threshold_json") or "").strip() + if threshold_json: + threshold_path = Path(threshold_json) + if not threshold_path.is_absolute(): + threshold_path = (config_path.parent / threshold_path).resolve() + if not threshold_path.is_file(): + raise FileNotFoundError(f"threshold_json not found: {threshold_path}") + else: + threshold_candidates = sorted(config_path.parent.glob("*threshold.json")) + if not threshold_candidates: + raise FileNotFoundError(f"no *threshold.json next to config: {config_path.parent}") + if len(threshold_candidates) > 1: + raise ValueError(f"multiple *threshold.json files next to config (ambiguous): {threshold_candidates}") + threshold_path = threshold_candidates[0] + thresholds = json.loads(threshold_path.read_text()) + + # Pass 1: cluster placeholders ({user-id}) everywhere. + cluster_map = _resolve_cluster_mapping(cluster_dict) + raw = _walk_substitute(raw, cluster_map) + + # Pass 2: self-reference within paths ({shared_fs} inside paths.*). + paths_block = raw.get("paths", {}) + if isinstance(paths_block, dict): + for _ in range(len(paths_block) + 1): + new = { + k: _walk_substitute(v, {pk: pv for pk, pv in paths_block.items() if isinstance(pv, str)}) + for k, v in paths_block.items() + } + if new == paths_block: + break + paths_block = new + raw["paths"] = paths_block + + # Pass 3: cross-block ({paths.models_dir} -> anywhere else). + flat_map = _flatten_paths({"paths": raw.get("paths", {})}) + raw = _walk_substitute(raw, flat_map) + + # Drop comment keys (e.g. "_comment") before framework/threshold validation. + raw = {k: v for k, v in raw.items() if not k.startswith("_")} + thresholds = {k: v for k, v in thresholds.items() if not k.startswith("_")} + + return raw, thresholds diff --git a/cvs/lib/utils/docs/gpu-metrics.md b/cvs/lib/utils/docs/gpu-metrics.md new file mode 100644 index 000000000..93e11de94 --- /dev/null +++ b/cvs/lib/utils/docs/gpu-metrics.md @@ -0,0 +1,374 @@ +# GPU Metrics Polling — Integration Guide + +`cvs/lib/utils/gpu.py` is a shared library that any CVS suite — inference or training — +can use to collect GPU utilisation data during a run and surface it as rows in the +HTML report. It has no suite-specific logic: it shells out to `amd-smi metric --json` +via an `Orchestrator` and parses/aggregates the result. This document explains what the +library measures and how a suite can wire it in; the exact fixture/parametrize/threshold +plumbing shown below is illustrative reference pseudocode drawn from an inference suite — +adapt it to your suite's own lifecycle-as-tests structure. + +> **Prerequisite**: this guide assumes you have completed (or are familiar with) +> the steps in `cvs/lib/inference/ADDING_A_SUITE.md`. Concepts like `cell_key`, +> `GATED_METRICS`, and `inf_res_dict` structure are defined there. + +--- + +## What it measures + +Five derived metrics are produced per run: + +| Metric key | Unit | Description | +|---|---|---| +| `gpu.peak_gpu_memory_mb` | MB | Highest VRAM used across all GPUs at any single poll during inference. Each poll sums VRAM across all GPUs on the node; this value is the max of those sums. | +| `gpu.model_load_memory_mb` | MB | VRAM delta between a snapshot taken before model load and one taken after. Represents the memory cost of loading the model weights. | +| `gpu.model_load_s` | s | Wall-clock time from server start to the post-load snapshot. | +| `gpu.gpu_bandwidth_util_pct` | % | Mean UMC (unified memory controller) activity across all GPUs, averaged over all polls taken during inference. | +| `gpu.gpu_compute_util_pct` | % | Mean GFX (shader/compute) activity across all GPUs, averaged over all polls taken during inference. | + +Each metric appears as its own row in the HTML report, with value, unit, and a +pass/fail result if a threshold is configured. + +--- + +## How polling works + +1. **Pre-load snapshot** — `capture_gpu_metrics(orch)` is called before the server + starts. Records baseline VRAM. +2. **Server start + post-load snapshot** — after the server is ready, + `capture_gpu_metrics(orch)` is called again. The VRAM delta and elapsed time give + `model_load_memory_mb` and `model_load_s`. +3. **Client phase polling** — `poll_gpu_metrics(...)` is called (either synchronously + with a backgrounded client, or from a thread with a synchronous client) and calls + `amd-smi metric --json` on the head node every `poll_interval_s` seconds + (default 15 s) until `is_done_fn()` returns `True`. +4. **Aggregation** — after the client completes, `agg_readings(readings)` reduces the + poll list to `peak_gpu_memory_mb`, `gpu_compute_util_pct`, and + `gpu_bandwidth_util_pct`. +5. **Results stored** — all five derived metrics are written into `inf_res_dict` under + `gpu.` so `test_gpu_metric` can read them. + +`amd-smi` runs on the host node, not inside the container. Single-node suites use +`orch.exec_on_head("amd-smi metric --json")`; multi-node suites pass a `nodes` list and +`gpu.py` calls `orch.exec("amd-smi metric --json", hosts=hosts)` per node instead. This +is intentional — `amd-smi` is a host-side tool and is not available inside the benchmark +container. + +--- + +## Multi-node polling + +Both `capture_gpu_metrics` and `poll_gpu_metrics` accept an optional `nodes` parameter: +a `list[(label, hosts)]`, where `hosts` is a list of hostnames. When provided, `gpu.py` +calls `orch.exec("amd-smi metric --json", hosts=hosts)` once per `(label, hosts)` pair +per poll, merges every node's GPU entries into a single aggregated snapshot (same shape +as the single-node case), and separately tracks the last successful per-node VRAM +reading for the summary block. + +```python +nodes = [ + ("prefill-0", prefill_node_list), + ("decode-0", decode_node_list), +] +poll_readings = poll_gpu_metrics( + orch, + is_done_fn=, + log_path=str(_gpu_log) if _gpu_log else None, + model_load_s=load_s, + model_load_memory_mb=load_mb, + nodes=nodes, +) +``` + +Any `Orchestrator` subclass works here since `.exec(cmd, hosts=...)` is part of the base +`Orchestrator` contract — no need to construct raw `Pssh`/ssh handles per role. See +`cvs/lib/inference/sglang_disagg_lib.py::sglang_disagg_gpu_counts` for a disaggregated +prefill/decode suite that groups nodes by role this way. + +When `nodes` is provided, log lines are tagged with `[label1+label2]` and the summary +block gains a `--- per-node vram (last reading) ---` section listing each label's most +recent successful VRAM reading (or `-` if every poll failed for that node). + +--- + +## Integrating into a suite + +### 1. Add the GPU polling block to `test__inference` + +The function signature must include `gpu_metrics_snap` (see Step 3). Wrap +`capture_gpu_metrics` in a helper that degrades gracefully if `amd-smi` is unavailable +at snapshot time — unlike `poll_gpu_metrics`, it can raise. + +**Pattern A — client is backgrounded by the framework (synchronous poll):** + +```python +import pathlib +import time +from cvs.lib.utils.gpu import GPU_METRICS, GPU_METRIC_UNITS, agg_readings, capture_gpu_metrics, poll_gpu_metrics + +def test__inference(orch, variant_config, inf_res_dict, gpu_metrics_snap, request, ...): + + def _snap(): + try: + return capture_gpu_metrics(orch) + except Exception: + return {} + + pre_snap = _snap() + t0 = time.monotonic() + # ... start server (returns immediately; framework backgrounds the client) ... + post_snap = _snap() + load_s = time.monotonic() - t0 + load_mb = ((post_snap.get("gpu.used_vram") or 0) - (pre_snap.get("gpu.used_vram") or 0)) or None + + # Write the log into the local report dir so it lands in the zip bundle. + _htmlpath = getattr(request.config.option, "htmlpath", None) + _html_dir = getattr(request.config, "_test_html_dir", "test_html") + _gpu_log = ( + pathlib.Path(_htmlpath).parent / _html_dir / "gpu_poll.log" + if _htmlpath else None + ) + + poll_readings = poll_gpu_metrics( + orch, + is_done_fn=, # e.g. job.is_client_done + log_path=str(_gpu_log) if _gpu_log else None, + model_load_s=load_s, + model_load_memory_mb=load_mb, + ) + + agg = agg_readings(poll_readings) + inf_res_dict["gpu.peak_gpu_memory_mb"] = agg.get("peak_gpu_memory_mb") + inf_res_dict["gpu.model_load_memory_mb"] = load_mb + inf_res_dict["gpu.model_load_s"] = load_s + inf_res_dict["gpu.gpu_bandwidth_util_pct"] = agg.get("gpu_bandwidth_util_pct") + inf_res_dict["gpu.gpu_compute_util_pct"] = agg.get("gpu_compute_util_pct") +``` + +**Pattern B — client runs synchronously in the main thread (thread the poll):** + +```python +import threading + + done_flag = threading.Event() + poll_readings = [] + def _poll(): + poll_readings.extend(poll_gpu_metrics( + orch, done_flag.is_set, + log_path=f"{variant_config.paths.log_dir}/gpu_poll.log", + model_load_s=load_s, + model_load_memory_mb=load_mb, + )) + poll_thread = threading.Thread(target=_poll, daemon=True) + poll_thread.start() + # ... run client synchronously ... + done_flag.set() + poll_thread.join() + # then aggregate as in Pattern A +``` + +### 2. Add `test_gpu_metric` + +`test_gpu_metric` is parametrized via `pytest_generate_tests` (see Step 4), not via a +`@pytest.mark.parametrize` decorator. The fixture parameter name is `gpu_metric` +(singular, matching the `pytest_generate_tests` branch). + +Pass the **full** per-cell actuals dict to `evaluate_all` — not just the single metric +— so that `min_ratio` threshold specs can resolve their reference metric: + +```python +from cvs.lib.utils.gpu import GPU_METRIC_UNITS +from cvs.lib.utils.verdict import ThresholdViolation, evaluate_all + +def test_gpu_metric(gpu_metric, inf_res_dict, variant_config, request): + val = inf_res_dict.get(gpu_metric) + unit = GPU_METRIC_UNITS.get(gpu_metric, "") + + request.node.user_properties.append(("metric_value", val)) + request.node.user_properties.append(("metric_unit", unit)) + + if val is None: + pytest.skip(f"{gpu_metric}: no value recorded (amd-smi unavailable or polling failed)") + + if not variant_config.enforce_thresholds: + return + + cell = variant_config.cell_key(isl, osl, concurrency) # same key used for test_metric + spec = (variant_config.thresholds.get(cell) or {}).get(gpu_metric) + if spec is None: + return # no spec → record-only + + # Pass full cell actuals so min_ratio specs can resolve their reference metric + cell_actuals = {k: inf_res_dict.get(k) for k in inf_res_dict} + try: + evaluate_all(cell_actuals, {gpu_metric: spec}) + except ThresholdViolation as exc: + pytest.fail(str(exc)) +``` + +### 3. Add `gpu_metrics_snap` fixture to `conftest.py` + +```python +@pytest.fixture(scope="module") +def gpu_metrics_snap(): + return {} +``` + +This fixture is a forward-declaration that lets `test_gpu_metric` be collected without +errors even if a future version stores intermediate state in it. + +### 4. Register `test_gpu_metric` in `pytest_collection_modifyitems` and `pytest_generate_tests` + +**Collection sort** — add `test_gpu_metric` at rank 4 alongside `test_metric`: + +```python +rank = { + "test_launch_container": 0, + "test_setup_sshd": 1, + "test_model_fetch": 2, + "test__inference": 3, + "test_metric": 4, + "test_gpu_metric": 4, # must be present; omitting → rank 99 → runs after teardown + "test_print_results_table": 5, + "test_teardown": 6, +} +``` + +**Parametrize** — add an `elif` branch to `pytest_generate_tests` in the test module. +The fixture name is `gpu_metric` (singular): + +```python +from cvs.lib.utils.gpu import GPU_METRICS + +def pytest_generate_tests(metafunc): + if "metric" in metafunc.fixturenames: + # ... your existing metric parametrize branch ... + elif "gpu_metric" in metafunc.fixturenames: + metafunc.parametrize( + "gpu_metric", + [k for k, _ in GPU_METRICS], + ids=[k for k, _ in GPU_METRICS], + ) +``` + +Without this branch `test_gpu_metric` collects zero instances and produces no HTML rows. + +### 5. Add threshold entries and update `GATED_METRICS` + +**Threshold JSON** — threshold keys use the `gpu.` prefix. For each sweep cell: + +```json +"isl1000_osl1000_conc16": { + "client.total_token_throughput": { "kind": "min_tok_s", "value": 1000 }, + "gpu.peak_gpu_memory_mb": { "kind": "max", "value": 200000 }, + "gpu.model_load_memory_mb": { "kind": "max", "value": 150000 }, + "gpu.model_load_s": { "kind": "max", "value": 300 }, + "gpu.gpu_bandwidth_util_pct": { "kind": "min", "value": 10 }, + "gpu.gpu_compute_util_pct": { "kind": "min", "value": 5 } +} +``` + +**`GATED_METRICS`** — if your `VariantConfig` subclass validates that every gated +metric has a threshold entry (the two-axis coverage check in `ADDING_A_SUITE.md` +Step 2), add all five `gpu.*` keys to your `GATED_METRICS` set: + +```python +GATED_METRICS = { + "client.total_token_throughput", + ... + "gpu.peak_gpu_memory_mb", + "gpu.model_load_memory_mb", + "gpu.model_load_s", + "gpu.gpu_bandwidth_util_pct", + "gpu.gpu_compute_util_pct", +} +``` + +Omitting them causes a silent green PASS with no assertions when `enforce_thresholds=True` +and the spec is missing. + +**First run / characterisation** — set `enforce_thresholds: false` in the suite config. +All five metrics will be collected and surfaced as HTML rows but will never cause a +test failure. Use the reported values to populate your threshold JSON, then flip +`enforce_thresholds` to `true`. + +See `docs/threshold-kinds.md` for the full threshold kind reference (`min`, `max`, +`max_ms`, `within`, `min_tok_s`, `min_ratio`). + +--- + +## The `gpu_poll.log` file + +Every run writes `gpu_poll.log` into the local HTML report directory (the same folder +as the per-test HTML files, e.g. `_html/`). Because the zip bundle includes +that directory, the log is always available in the run archive. It is also copied to +the suite's NFS `out_dir` on the head node for cluster-side inspection. + +The file contains one line per poll and a summary block: + +``` +[gpu poll 1/?] used_vram=131072 MB gfx=87% umc=74% mm=0% +[gpu poll 2/?] used_vram=132864 MB gfx=91% umc=78% mm=0% +... +[gpu poll 12/?] used_vram=132480 MB gfx=89% umc=76% mm=0% [done] + +--- summary --- +samples: 12 +peak_gpu_memory_mb: 132864 MB +model_load_memory_mb: 127418 MB +model_load_s: 148.3 s +gpu_compute_util_pct: 89.2 % +gpu_bandwidth_util_pct: 76.1 % +``` + +A poll that fails (e.g. `amd-smi` exits non-zero or returns unparseable JSON) is +logged with a `FAILED [N/max consecutive]` tag and excluded from aggregation. After +`max_consecutive_failures` (default 3) consecutive failures the loop stops early and +logs a warning. + +--- + +## Failure handling and None values + +The library never raises from `poll_gpu_metrics`. Every metric can be `None`: + +| Situation | Result | +|---|---| +| `amd-smi` fails or returns unparseable JSON | snapshot excluded from aggregation; metric may be `None` if all polls fail | +| GPU reports `"N/A"` for a field | that field is `None` in the snapshot | +| Zero valid polls | all three `agg_readings` outputs are `None` | +| Caller passes `model_load_memory_mb=None` | stored as `None`; `test_gpu_metric` should `pytest.skip` rather than fail | + +`test_gpu_metric` should always check for `None` before evaluating thresholds. +`pytest.skip` (not `pytest.fail`) is the correct response when a metric is `None` — +the metric was unavailable for this run, not a regression. + +--- + +## Gotchas + +- **`model_load_memory_mb` should be `None` when VRAM data is unavailable, not `0`.** + Use `... or None` after the subtraction (as shown in Step 1). A zero stored as `0` + gets gated against thresholds and displayed as `"0"` in the report; `None` causes + `test_gpu_metric` to skip instead. +- **`capture_gpu_metrics` can raise; `poll_gpu_metrics` never does.** Always wrap + one-shot snapshot calls in a `try/except` that returns `{}` on failure. +- **`agg_readings` returns 3 keys, not 5.** `model_load_memory_mb` and `model_load_s` + are measured by the caller and stored separately. Do not look for them in + `agg_readings` output. +- **Raw snapshot keys differ from derived metric keys.** The poll loop returns dicts + with keys like `gpu.used_vram`; the stored/threshold-gated keys use names like + `gpu.peak_gpu_memory_mb`. Do not pass raw snapshots to `evaluate_all`. +- **Threshold JSON keys use the `gpu.` prefix** (`"gpu.peak_gpu_memory_mb"`, not + `"peak_gpu_memory_mb"`). A missing prefix means the spec is never found and the + metric silently operates as record-only even when `enforce_thresholds=True`. +- **`amd-smi` runs on the host, not in the container.** Single-node polling requires + `orch.exec_on_head`; multi-node polling (via `nodes=`) requires `orch.exec(cmd, + hosts=...)` instead. Every `Orchestrator` subclass supports both — if yours doesn't, + GPU polling is not available for your suite. +- **Multi-node degrades per label, not globally.** If `orch.exec` raises for one node in + `nodes`, that node's entries are excluded from the merged snapshot and its per-node + VRAM is `None` for that poll; other nodes' data is unaffected. +- **Pass the full cell actuals dict to `evaluate_all`.** `min_ratio` threshold specs + need to resolve a reference metric from `actuals`. Passing only the single metric's + value causes a reference-resolution failure. diff --git a/cvs/lib/utils/docs/placeholder-substitution.md b/cvs/lib/utils/docs/placeholder-substitution.md new file mode 100644 index 000000000..61cb211f7 --- /dev/null +++ b/cvs/lib/utils/docs/placeholder-substitution.md @@ -0,0 +1,279 @@ +# Placeholder Substitution — Three-Pass Walkthrough + +`substitute_config` resolves placeholders in a variant config JSON in exactly three +passes. Each pass has a defined scope: which keys it reads from and which part of the +document it writes to. Understanding the order matters because a value produced by an +earlier pass becomes available as input to a later one. + +Source: `cvs/lib/utils/config_loader.py`, function `substitute_config`. + +--- + +## Worked example + +The example uses `llama31_70b_fp8_config.json` (the `w1_llama31_70b_fp8kv` variant). +The cluster dict supplied at runtime is: + +```json +{ "username": "jsmith" } +``` + +### Starting document (before any substitution) + +Only the fields that contain placeholders or that receive substituted values are shown; +the rest of the config (model, sweep, params, roles) is omitted for brevity. + +```json +{ + "threshold_json": "", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "{shared_fs}/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "" + }, + "container": { + "runtime": { + "args": { + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "{paths.models_dir}:/models" + ] + } + } + } +} +``` + +Note: `threshold_json` and `hf_token_file` are shown here with the sentinel value +`""` that appears in the actual file — both fields must be filled in with +real absolute paths before use. The substitution walkthrough below uses illustrative +values (`/shared/cvs/thresholds/llama31_70b_fp8_threshold.json` and +`/shared/cvs/tokens/{user-id}.token` respectively) to keep the example concrete; those +paths do not come from the real config file. + +Note: `threshold_json` must always be a fully-resolved absolute path with no +placeholders. Using `{user-id}` or any other token there causes a +`FileNotFoundError` at the literal unresolved path (see the `threshold_json` +section below). + +--- + +## Pass 1 — Cluster placeholders (`{user-id}`) everywhere + +**Mapping built:** `_resolve_cluster_mapping(cluster_dict)` reads +`cluster_dict["username"]` (`"jsmith"`), producing `{"user-id": "jsmith"}`. When +`cluster_dict` has no `username` key or the value is falsy (empty string, `None`, +etc.), `getpass.getuser()` is used as the fallback. + +**Scope:** `_walk_substitute` is called on the entire document. Every `{user-id}` token +in any string value is replaced. + +**After Pass 1:** + +```json +{ + "threshold_json": "/shared/cvs/thresholds/llama31_70b_fp8_threshold.json", + "paths": { + "shared_fs": "/home/jsmith", + "models_dir": "{shared_fs}/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "/shared/cvs/tokens/jsmith.token" + }, + "container": { + "runtime": { + "args": { + "volumes": [ + "/home/jsmith:/home/jsmith", + "{paths.models_dir}:/models" + ] + } + } + } +} +``` + +Key observations: + +- `threshold_json` contains no placeholders so it is unchanged by Pass 1. The + threshold file was already read off disk before this pass ran (see the + `threshold_json` section below). +- `paths.shared_fs` resolved to `/home/jsmith`, but `paths.models_dir` still shows + `{shared_fs}/models`. That token is a self-reference within the paths block and is + not in the cluster mapping. It is resolved in Pass 2. +- `volumes[1]` still shows `{paths.models_dir}:/models`. That cross-block reference + is resolved in Pass 3. + +--- + +## Pass 2 — Self-reference within `paths` (`{shared_fs}` inside `paths.*`) + +**Mapping built:** the keys and string values of the `paths` block at their current +state after Pass 1: + +``` +shared_fs -> "/home/jsmith" +models_dir -> "{shared_fs}/models" +log_dir -> "{shared_fs}/LOGS" +hf_token_file -> "/shared/cvs/tokens/jsmith.token" +``` + +**Scope:** `_walk_substitute` is called only on the `paths` block. The mapping is the +paths block's own key-value pairs (string values only). The substitution loop repeats +until the block stops changing, which handles chains: if `models_dir` referenced +`{log_dir}` which in turn referenced `{shared_fs}`, multiple iterations would unwind +the chain completely. The loop is capped at `len(paths_block) + 1` iterations. A cycle +in self-references (e.g. `a = "{b}"`, `b = "{a}"`) exits the cap silently, leaving +both tokens unresolved and verbatim — the same no-error-at-load-time behaviour as any +other unknown token. + +**After Pass 2:** + +```json +{ + "paths": { + "shared_fs": "/home/jsmith", + "models_dir": "/home/jsmith/models", + "log_dir": "/home/jsmith/LOGS", + "hf_token_file": "/shared/cvs/tokens/jsmith.token" + } +} +``` + +Every `paths.*` value is now fully concrete. The rest of the document is unchanged at +this point — `volumes[1]` still holds `{paths.models_dir}:/models`. + +--- + +## Pass 3 — Cross-block (`{paths.models_dir}` into the rest of the document) + +**Mapping built:** `_flatten_paths({"paths": raw.get("paths", {})})` produces dotted keys for +every leaf in the paths block: + +``` +paths.shared_fs -> "/home/jsmith" +paths.models_dir -> "/home/jsmith/models" +paths.log_dir -> "/home/jsmith/LOGS" +paths.hf_token_file -> "/shared/cvs/tokens/jsmith.token" +``` + +**Scope:** `_walk_substitute` is called on the entire document with this mapping. Any +`{paths.}` token anywhere in the document is replaced with the fully-resolved +paths value. + +Note: `{paths.*}` tokens inside the paths block itself are also resolved by Pass 3 — +Pass 2 only recognises bare keys (e.g. `{shared_fs}`), not dotted keys (e.g. +`{paths.shared_fs}`). A dotted self-reference in paths survives Pass 2 verbatim and is +then expanded by Pass 3. + +**After Pass 3 (final document):** + +```json +{ + "threshold_json": "/shared/cvs/thresholds/llama31_70b_fp8_threshold.json", + "paths": { + "shared_fs": "/home/jsmith", + "models_dir": "/home/jsmith/models", + "log_dir": "/home/jsmith/LOGS", + "hf_token_file": "/shared/cvs/tokens/jsmith.token" + }, + "container": { + "runtime": { + "args": { + "volumes": [ + "/home/jsmith:/home/jsmith", + "/home/jsmith/models:/models" + ] + } + } + } +} +``` + +`volumes[1]` is now `/home/jsmith/models:/models`. This is the value the container +orchestrator receives for the volume mount. + +--- + +## Special case: `threshold_json` and substitution + +`threshold_json` is a **literal absolute path**. The threshold file is read using the +raw (pre-substitution) string value before any pass runs: + +```python +raw = json.loads(config_path.read_text()) +threshold_path = Path(raw["threshold_json"]) # raw value, before any pass +thresholds = json.loads(threshold_path.read_text()) +# ... passes run after this point +``` + +This has two consequences: + +1. **`{user-id}` in `threshold_json` causes a FileNotFoundError.** If your + `threshold_json` is `/shared/cvs/thresholds/{user-id}/threshold.json`, the file + open is attempted at that literal string — the substitution that would resolve + `{user-id}` has not run yet. Write `threshold_json` as a fully-resolved absolute + path with no placeholders. + + Pass 1 does rewrite the `threshold_json` string value in the in-memory `raw` dict + (so after substitution the string shows the resolved path), but because the + threshold file was already opened before Pass 1 ran, that rewrite has no effect on + which file was loaded. + +2. **`{paths.*}` tokens in `threshold_json` are not substituted.** Pass 3 rewrites the + entire document, so `{paths.log_dir}` in `threshold_json` would technically be + replaced in the in-memory `raw` dict — but because the threshold file was already + read before that pass, the token in the string value is irrelevant to which file + was loaded. Keep `threshold_json` as a plain absolute path. + +--- + +## What a typo'd placeholder looks like + +If a placeholder token does not match any key in the current pass's mapping, +`_walk_substitute` leaves it verbatim — curly braces and all. No error is raised at +substitution time. + +Example: suppose `volumes` contained a typo in the models dir token: + +```json +"volumes": [ + "/home/{user-id}:/home/{user-id}", + "{paths.modles_dir}:/models" +] +``` + +After all three passes (with `user-id` resolved and `paths.models_dir` in the mapping +but `paths.modles_dir` not), the output is: + +```json +"volumes": [ + "/home/jsmith:/home/jsmith", + "{paths.modles_dir}:/models" +] +``` + +The misspelled token `{paths.modles_dir}` survives all three passes unchanged. The +container launch then receives the literal string `{paths.modles_dir}:/models` as a +volume mount argument, which Docker rejects at runtime — not at config-load time. + +The same applies to `paths` self-references. A typo'd `{sahred_fs}` in +`paths.models_dir` survives Pass 2 and propagates as a brace-wrapped string into the +paths block that feeds Pass 3. + +**Diagnostic rule:** if a mount, log path, or model path looks wrong at runtime, +inspect the `raw` dict returned by `substitute_config`. Any string value containing +a literal `{...}` is a placeholder that failed to resolve, which always indicates a +token name mismatch or a missing mapping key. + +--- + +## Summary table + +| Pass | Mapping source | Document scope | Example tokens resolved | +|------|---------------|----------------|------------------------| +| 1 | `cluster_dict["username"]` (falls back to `getpass.getuser()` when the key is absent **or the value is falsy** — empty string, `None`, etc.) | Entire document | `{user-id}` | +| 2 | `paths` block key-value pairs (string values only), iterated until stable | `paths` block only | `{shared_fs}` (any bare key name in paths can be used as a self-reference token) | +| 3 | Flattened `paths` block as dotted keys | Entire document | `{paths.shared_fs}`, `{paths.models_dir}`, `{paths.log_dir}`, `{paths.hf_token_file}` | + +Unknown tokens survive all passes verbatim. No error is raised at substitution time. diff --git a/cvs/lib/utils/docs/threshold-kinds.md b/cvs/lib/utils/docs/threshold-kinds.md new file mode 100644 index 000000000..b3562c104 --- /dev/null +++ b/cvs/lib/utils/docs/threshold-kinds.md @@ -0,0 +1,356 @@ +# Threshold Kinds Reference + +Full reference for every threshold kind understood by `_check_one` and +`evaluate_all` in `cvs/lib/utils/verdict.py`. + +--- + +## How thresholds are evaluated + +`evaluate_all(actuals, thresholds)` iterates over every key in `thresholds`. +Before calling `_check_one` on a spec, it handles two conditions centrally: + +- **Metric missing from actuals** — appends + `"{metric}: missing from actuals"` as a violation and skips `_check_one`. +- **Metric value is `None`** — appends + `"{metric}: value is None (metric unavailable for this run)"` as a violation + and skips `_check_one`. This prevents a `float(None)` TypeError and surfaces + the problem explicitly. + +> **Note:** only `None` and missing keys are handled centrally. Any other +> non-float-convertible value — for example, a string `"N/A"` or `"error"` +> produced by a metric-extraction bug — passes both guards and reaches +> `_check_one`, where `_to_float` raises a raw `ValueError`, not a +> `ThresholdViolation`. Because `evaluate_all` has no try/except around +> `_check_one`, that `ValueError` propagates uncaught and breaks out of the +> violations-collection loop entirely. Callers that may populate `actuals` from +> untrusted or partially-failed data should coerce or validate metric values to +> `float`-or-`None` before passing to `evaluate_all`. + +After these guards, `evaluate_all` calls `_check_one(metric, actual, spec)`. +If `_check_one` returns a truthy (non-empty, non-`None`) string, that string is +added to the violation list. Note: `_check_one` must return `None` (not `""`) +on the passing path — the collector uses `if v:` rather than +`if v is not None`, so an empty string would be silently dropped. All violations are collected before raising; `ThresholdViolation` +carries the full list in `.violations` and its message is the newline-joined +string of all of them. + +--- + +## Kinds + +### `min` + +**JSON shape** + +```json +{ "kind": "min", "value": } +``` + +| Field | Type | Required | +|---------|--------|----------| +| `kind` | string | yes | +| `value` | number | yes | + +**Comparison** + +Passes when `actual >= value`. Fails when `actual < value`. + +**Failure message** + +``` +{metric}: actual {actual} < min {target} +``` + +**When to use** + +Use for dimensionless or mixed-unit lower bounds where the unit is either +implicit from the metric name or irrelevant to the failure message. Examples: +token counts, request counts, dimensionless ratios that do not already have a +dedicated kind. Prefer `min_tok_s` when the metric is a token-per-second +throughput — it produces a more readable failure message with explicit units. + +--- + +### `max` + +**JSON shape** + +```json +{ "kind": "max", "value": } +``` + +| Field | Type | Required | +|---------|--------|----------| +| `kind` | string | yes | +| `value` | number | yes | + +**Comparison** + +Passes when `actual <= value`. Fails when `actual > value`. + +**Failure message** + +``` +{metric}: actual {actual} > max {target} +``` + +**When to use** + +Use for upper bounds on metrics whose unit is not milliseconds. The canonical +case is `failed` (failed request count): a milliseconds suffix in the message +would be a unit lie. Use `max_ms` when the metric is a latency in milliseconds. +The comparison logic is identical to `max_ms`; only the failure message differs. + +--- + +### `max_ms` + +**JSON shape** + +```json +{ "kind": "max_ms", "value": } +``` + +| Field | Type | Required | +|---------|--------|----------| +| `kind` | string | yes | +| `value` | number | yes | + +**Comparison** + +Passes when `actual <= value`. Fails when `actual > value`. + +**Failure message** + +``` +{metric}: actual {actual} ms > max {target} ms +``` + +**When to use** + +Use for latency upper bounds where the metric is expressed in milliseconds (TTFT, +TPOT, E2EL, ITL). The `ms` suffix in both slots of the failure message makes the +unit explicit. Use `max` instead when the metric is not a time measurement. + +--- + +### `within` + +**JSON shape** + +```json +{ "kind": "within", "value": , "tolerance_pct": } +``` + +| Field | Type | Required | +|-----------------|--------|----------| +| `kind` | string | yes | +| `value` | number | yes | +| `tolerance_pct` | number | yes | + +**Comparison** + +Computes an acceptable band: + +``` +lo = value * (1 - tolerance_pct / 100.0) +hi = value * (1 + tolerance_pct / 100.0) +``` + +Passes when `lo <= actual <= hi`. Fails otherwise. + +> **Gotcha — `value` must be positive.** A negative `value` inverts the band: +> for example, `value = -100` with `tolerance_pct = 10` yields +> `lo = -90` and `hi = -110`, so `lo > hi` and the test `lo <= actual <= hi` +> can never pass. Every check fails with the normal outside-band message, +> giving no hint that the spec itself is the problem. There is no guard in +> `_check_one` or `evaluate_all` against this. +> +> A `value` of `0` collapses the band to a single point (`lo = hi = 0`); only +> `actual == 0` passes. + +**Failure message** + +``` +{metric}: actual {actual} outside {target} ±{pct}% +``` + +**When to use** + +Use when the acceptable range is symmetric around a target value and you want +to express the tolerance as a percentage rather than an absolute bound. Useful +for stability metrics or regressions against a known baseline where ±N% drift +is acceptable. Prefer `min` or `max` when the bound is one-sided. + +--- + +### `min_tok_s` + +**JSON shape** + +```json +{ "kind": "min_tok_s", "value": } +``` + +| Field | Type | Required | +|---------|--------|----------| +| `kind` | string | yes | +| `value` | number | yes | + +**Comparison** + +Passes when `actual >= value`. Fails when `actual < value`. + +**Failure message** + +``` +{metric}: actual {actual} tok/s < min {target} tok/s +``` + +**When to use** + +Use for token throughput lower bounds (`client.total_token_throughput`, +`client.output_throughput`, `client.per_gpu_throughput`). Functionally identical +to `min` but annotates `tok/s` in both slots of the failure message. Use `min` +for lower bounds on metrics that are not token-per-second rates. + +--- + +### `min_ratio` + +**JSON shape** + +```json +{ "kind": "min_ratio", "value": , "reference": "" } +``` + +| Field | Type | Required | +|-------------|--------|----------| +| `kind` | string | yes | +| `value` | number | yes | +| `reference` | string | yes | + +**Comparison** + +Computes `observed = actual / actuals[reference]` and passes when +`observed >= value`. Fails when `observed < value`. + +`reference` names another metric that must also appear in `actuals`. The +observed value of that metric is used as the denominator. + +**Failure message (ratio below minimum)** + +``` +{metric}: observed ratio {observed:.3f} < min {ratio} (vs {ref_metric}) +``` + +**When to use** + +Use when the threshold for one metric is expressed as a fraction of another +metric from the same cell run. Example: asserting that `client.output_throughput` +is at least 0.8 of `client.total_token_throughput`. This avoids hard-coding an +absolute number that would need to be recalibrated every time the reference +metric changes with hardware or model configuration. + +Do not use `min_ratio` when an absolute lower bound suffices — the ratio +interpretation adds coupling between two metrics and the failure message is +harder to read than a plain `min` or `min_tok_s` violation. + +--- + +#### The `_actuals` injection mechanism + +Callers of `evaluate_all` never set `_actuals` in a spec dict. `evaluate_all` +injects it automatically for every `min_ratio` spec: + +```python +spec_with_actuals = dict(spec) +if spec.get("kind") == "min_ratio": + spec_with_actuals["_actuals"] = actuals +v = _check_one(metric, actuals[metric], spec_with_actuals) +``` + +Inside `_check_one`, `_actuals` is read with `.get("_actuals", {})` to look up +the reference metric's value at check time. This means: + +- The threshold JSON never contains `_actuals`. +- The reference metric is resolved at evaluation time from the same `actuals` + dict that holds the primary metric's value. +- Passing all cell actuals to `evaluate_all` (not just the single metric being + asserted) is required for `min_ratio` to work. Inference suite `test_metric` + hooks (e.g. `cvs/tests/inference/vllm/vllm_single.py`) call + `evaluate_all(actuals, {full: spec})` with the full per-host actuals dict for + exactly this reason. + +--- + +#### `min_ratio` failure modes + +There are four failure conditions distinct from the ratio comparison itself. +Each produces its own violation string and short-circuits before the ratio is +computed. + +**Reference metric missing from actuals** + +Triggered when the metric named by `reference` is not a key in the `actuals` +dict at all (e.g., the key was never populated, or has a typo in `reference`). + +``` +{metric}: reference metric '{ref_metric}' missing from actuals +``` + +**Reference metric is `None`** + +Triggered when the reference metric key exists in `actuals` but its value is +`None`. This happens when a derived metric could not be computed for the run +(e.g., a zero-divisor upstream in `to_client_metrics`). + +``` +{metric}: reference '{ref_metric}' is None (metric unavailable for this run) +``` + +**Reference metric is zero** + +Triggered when `float(actuals[ref_metric]) == 0`. Division by zero is blocked +explicitly before the ratio is computed. + +``` +{metric}: reference '{ref_metric}' is 0; cannot compute ratio +``` + +**Reference metric is non-numeric and non-`None`** + +Triggered when `actuals[ref_metric]` is neither `None` nor float-convertible +(e.g., the string `"error"` produced by a metric-extraction bug). `_to_float` +raises `ValueError` uncaught; `evaluate_all` has no `try/except` around +`_check_one`, so the exception propagates and breaks out of the +violations-collection loop. Callers must coerce reference metric values to +`float`-or-`None` before calling `evaluate_all`. + +--- + +## Unknown kind + +If `kind` does not match any of the six strings above, `_check_one` returns: + +``` +{metric}: unknown threshold kind '{kind}' +``` + +This surfaces as a violation collected by `evaluate_all`. There is no exception +at parse time — the error only appears when `evaluate_all` runs against a spec +with an unrecognised kind. + +--- + +## Quick reference + +| Kind | Bound | Unit in message | Required fields beyond `kind` | +|-------------|---------------|-----------------|---------------------------------| +| `min` | lower | none | `value` | +| `max` | upper | none | `value` | +| `max_ms` | upper | `ms` | `value` | +| `within` | lower + upper | none | `value`, `tolerance_pct` | +| `min_tok_s` | lower | `tok/s` | `value` | +| `min_ratio` | lower (ratio) | ratio (3 d.p.) | `value`, `reference` | diff --git a/cvs/lib/utils/gpu.py b/cvs/lib/utils/gpu.py new file mode 100644 index 000000000..dbead6f16 --- /dev/null +++ b/cvs/lib/utils/gpu.py @@ -0,0 +1,399 @@ +'''Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. +''' + +from __future__ import annotations + +import json +import logging +import pathlib +import time + +# Human-readable derived metrics exposed as HTML rows (one row per entry per cell). +# These are computed by the calling suite from the raw amd-smi snapshots and stored +# under "gpu." keys in inf_res_dict. +GPU_METRICS: list[tuple[str, str]] = [ + ("peak_gpu_memory_mb", "MB"), + ("model_load_memory_mb", "MB"), + ("model_load_s", "s"), + ("gpu_bandwidth_util_pct", "%"), + ("gpu_compute_util_pct", "%"), +] +GPU_METRIC_UNITS: dict[str, str] = {k: u for k, u in GPU_METRICS} + +# Raw amd-smi field keys emitted by parse_gpu_metrics(). Not used as test rows. +_RAW_GPU_FIELDS: list[tuple[str, str]] = [ + ("gfx_activity", "%"), + ("umc_activity", "%"), + ("mm_activity", "%"), + ("total_vram", "MB"), + ("used_vram", "MB"), + ("free_vram", "MB"), + ("energy_j", "J"), +] +_RAW_GPU_FIELD_UNITS: dict[str, str] = {k: u for k, u in _RAW_GPU_FIELDS} + + +def _safe_get(d, *keys, default=None): + """Navigate nested dicts safely; return default on missing key or 'N/A' value.""" + cur = d + for key in keys: + if not isinstance(cur, dict): + return default + cur = cur.get(key, default) + if cur is default: + return default + if cur == "N/A": + return default + return cur + + +def parse_usage(gpu_entry: dict) -> dict: + """Extract activity metrics from one GPU entry dict. + + Returns: {"gpu.gfx_activity", "gpu.umc_activity", "gpu.mm_activity"} + Values are int or None; never raises. + """ + fields = ("gfx_activity", "umc_activity", "mm_activity") + result = {} + for field in fields: + val = _safe_get(gpu_entry, "usage", field, "value") + result[f"gpu.{field}"] = val + return result + + +def parse_mem_usage(gpu_entry: dict) -> dict: + """Extract memory usage metrics from one GPU entry dict. + + Returns: {"gpu.total_vram", "gpu.used_vram", "gpu.free_vram"} + Values are int or None; never raises. + """ + fields = ("total_vram", "used_vram", "free_vram") + result = {} + for field in fields: + val = _safe_get(gpu_entry, "mem_usage", field, "value") + result[f"gpu.{field}"] = val + return result + + +def parse_energy(gpu_entry: dict) -> dict: + """Extract energy consumption from one GPU entry dict. + + Returns: {"gpu.energy_j"} + Value is float or None; never raises. + """ + val = _safe_get(gpu_entry, "energy", "total_energy_consumption", "value") + if val is not None: + val = float(val) + return {"gpu.energy_j": val} + + +def parse_gpu_metrics(raw: list) -> dict: + """Aggregate all GPU entries from one host's amd-smi --json output. + + raw: the parsed JSON list (one dict per GPU per host). + Activity metrics (%) -> averaged across GPUs (only non-None values counted). + Memory / energy metrics -> summed across GPUs (only non-None values counted). + Empty/missing -> all None. + """ + all_none = {f"gpu.{k}": None for k, _u in _RAW_GPU_FIELDS} + if not raw: + return all_none + + activity_keys = ("gpu.gfx_activity", "gpu.umc_activity", "gpu.mm_activity") + vram_keys = ("gpu.total_vram", "gpu.used_vram", "gpu.free_vram") + energy_key = "gpu.energy_j" + + # Accumulators: sum and count per field (None excluded from both) + activity_sums: dict[str, float] = {k: 0.0 for k in activity_keys} + activity_counts: dict[str, int] = {k: 0 for k in activity_keys} + vram_sums: dict[str, int | None] = {k: None for k in vram_keys} + energy_sum: float | None = None + + for entry in raw: + usage = parse_usage(entry) + mem = parse_mem_usage(entry) + eng = parse_energy(entry) + + for key in activity_keys: + val = usage[key] + if val is not None: + activity_sums[key] += val + activity_counts[key] += 1 + + for key in vram_keys: + val = mem[key] + if val is not None: + if vram_sums[key] is None: + vram_sums[key] = val + else: + vram_sums[key] += val + + e = eng[energy_key] + if e is not None: + if energy_sum is None: + energy_sum = e + else: + energy_sum += e + + result = {} + for key in activity_keys: + count = activity_counts[key] + result[key] = (activity_sums[key] / count) if count > 0 else None + + for key in vram_keys: + result[key] = vram_sums[key] + + result[energy_key] = energy_sum + return result + + +def _try_parse(text: str) -> list: + """Parse JSON text; return [] on empty/None/invalid JSON or non-list result. + + Accepts both bare-list format and the {"gpu_data": [...]} envelope that + amd-smi metric --json emits on ROCm 6.x nodes. + """ + if not text: + return [] + try: + parsed = json.loads(text) + except (json.JSONDecodeError, ValueError, TypeError): + return [] + if isinstance(parsed, dict): + parsed = parsed.get("gpu_data", []) + if not isinstance(parsed, list): + return [] + return parsed + + +def capture_gpu_metrics(orch, nodes=None, timeout_s=None) -> dict: + """One amd-smi exec on the host node(s). Returns flat {gpu.* metrics} dict. + + Single-node (nodes=None): orch must have .exec_on_head(cmd) -> {host: str}. + Multi-node (nodes provided, incl. []): nodes is a list of (label, hosts) + pairs where hosts is a list of hostnames passed to + orch.exec(cmd, hosts=hosts) -> {host: str}. nodes=[] is a valid "zero + nodes" case: no exec call is made and all fields come back None, the same + no-op result an empty raw list produces. All nodes' GPU entries are merged + before aggregation. Return type is identical in both cases. + + timeout_s: optional timeout (seconds) passed through to orch.exec/ + exec_on_head. None means no timeout (blocks until the remote call + returns), matching this function's historical behavior. + + Exceptions from exec calls (including a timeout firing) propagate to the + caller (poll_gpu_metrics handles them). + """ + all_entries = [] + if nodes is None: + kwargs = {"timeout": timeout_s} if timeout_s is not None else {} + out = orch.exec_on_head("amd-smi metric --json", **kwargs) + for _host, text in out.items(): + all_entries.extend(_try_parse(text)) + else: + kwargs = {"timeout": timeout_s} if timeout_s is not None else {} + for _label, hosts in nodes: + out = orch.exec("amd-smi metric --json", hosts=hosts, **kwargs) + for _host, text in out.items(): + all_entries.extend(_try_parse(text)) + return parse_gpu_metrics(all_entries) + + +def _mean(values: list) -> "float | None": + vals = [v for v in values if v is not None] + return sum(vals) / len(vals) if vals else None + + +def agg_readings(readings: list) -> dict: + """Aggregate poll readings into derived metrics. + Returns dict with peak_gpu_memory_mb, gpu_compute_util_pct, gpu_bandwidth_util_pct. + Any metric is None if no valid readings exist for it. + + Readings are raw snapshot dicts from capture_gpu_metrics (keys use gpu.* prefix). + """ + used_vrams = [r.get("gpu.used_vram") for r in readings if r.get("gpu.used_vram") is not None] + gfx_vals = [r.get("gpu.gfx_activity") for r in readings if r.get("gpu.gfx_activity") is not None] + umc_vals = [r.get("gpu.umc_activity") for r in readings if r.get("gpu.umc_activity") is not None] + return { + "peak_gpu_memory_mb": max(used_vrams) if used_vrams else None, + "gpu_compute_util_pct": _mean(gfx_vals), + "gpu_bandwidth_util_pct": _mean(umc_vals), + } + + +def _node_label_tag(nodes) -> str: + """Return '+'-joined node labels for log line tagging, or empty string.""" + if not nodes: + return "" + return "[" + "+".join(lbl for lbl, _hosts in nodes) + "] " + + +def _capture_multi_node(orch, nodes, timeout_s=None) -> "tuple[dict, dict[str, int | None]]": + """One amd-smi exec per (label, hosts) pair. + + Returns (merged_snapshot, per_node_vram) computed from a single exec round: + merged_snapshot is parse_gpu_metrics() over every node's GPU entries combined + (same shape as capture_gpu_metrics), per_node_vram is {label: used_vram_mb}. + + Degrades per label: if orch.exec raises (including a timeout_s firing) for + a node, that label's entries are excluded from the merge and its per-node + VRAM is None. + """ + all_entries = [] + per_node_vram: "dict[str, int | None]" = {} + kwargs = {"timeout": timeout_s} if timeout_s is not None else {} + for label, hosts in nodes: + try: + out = orch.exec("amd-smi metric --json", hosts=hosts, **kwargs) + node_entries = [] + for _host, text in out.items(): + node_entries.extend(_try_parse(text)) + all_entries.extend(node_entries) + snap = parse_gpu_metrics(node_entries) + per_node_vram[label] = snap.get("gpu.used_vram") + except Exception: + per_node_vram[label] = None + return parse_gpu_metrics(all_entries), per_node_vram + + +def poll_gpu_metrics( + orch, + is_done_fn, + poll_interval_s: float = 15, + label: str = "poll", + log_path=None, + max_consecutive_failures: int = 3, + model_load_s=None, + model_load_memory_mb=None, + nodes=None, + timeout_s=None, +) -> list: + """Poll GPU metrics while an inference client is running. + + Calls capture_gpu_metrics repeatedly until is_done_fn() returns True + or max_consecutive_failures consecutive exceptions are raised. + Returns list of raw snapshot dicts (failed polls excluded). + Never raises for amd-smi/parsing failures — those are caught, counted, + and logged. is_done_fn() is called outside that guard and any exception + it raises propagates to the caller (a broken done-predicate is a caller + bug, not a polling failure). Writes per-poll lines + summary to log_path + if given. + + nodes: optional list of (label, hosts) pairs for multi-node polling, where + hosts is a list of hostnames passed to orch.exec(cmd, hosts=hosts). When + provided (including nodes=[], the zero-node case: no exec call is made, + every field comes back None), all nodes are polled once per iteration and + merged into a single reading. In multi-node mode, a round where every + listed node failed is itself counted as one consecutive failure (same as + a raised exception in single-node mode); a partial success/failure round + still counts as success, preserving per-label degradation. Log lines are + tagged with node labels; summary includes per-node VRAM. When nodes=None + (default), uses orch.exec_on_head — single-node behaviour. + + timeout_s: optional timeout (seconds) passed through to orch.exec/ + exec_on_head on every poll. A timeout firing is caught like any other + amd-smi failure and counted toward max_consecutive_failures. None means + no timeout (blocks until the remote call returns). + """ + log = logging.getLogger(__name__) + readings: list = [] + log_lines: list = [] + poll_n = 0 + consecutive_failures = 0 + # Per-node VRAM tracking: {label: last_successful_used_vram} + node_last_vram: "dict[str, int | None]" = {lbl: None for lbl, _ in nodes} if nodes is not None else {} + node_tag = _node_label_tag(nodes) + + while True: + poll_n += 1 + snap = None + try: + if nodes is not None: + snap, per_node = _capture_multi_node(orch, nodes, timeout_s=timeout_s) + for lbl, vram in per_node.items(): + if vram is not None: + node_last_vram[lbl] = vram + if len(nodes) > 0 and not any(v is not None for v in per_node.values()): + raise RuntimeError(f"all nodes failed this round: {list(per_node)}") + else: + snap = capture_gpu_metrics(orch, nodes=None, timeout_s=timeout_s) + except Exception as exc: + consecutive_failures += 1 + line = ( + f"[gpu {label} {poll_n}/?] {node_tag}FAILED" + f" [{consecutive_failures}/{max_consecutive_failures} consecutive]:" + f" {type(exc).__name__}: {exc} (skipped)" + ) + log_lines.append(line) + if consecutive_failures >= max_consecutive_failures: + log.warning( + "poll_gpu_metrics: %d consecutive failures, stopping early", + consecutive_failures, + ) + break + time.sleep(poll_interval_s) + continue + + consecutive_failures = 0 + readings.append(snap) + used = snap.get("gpu.used_vram") + gfx = snap.get("gpu.gfx_activity") + umc = snap.get("gpu.umc_activity") + mm = snap.get("gpu.mm_activity") + # is_done_fn() runs outside the amd-smi try/except so an exception here + # is never misattributed as a polling failure. + done = is_done_fn() + done_tag = " [done]" if done else "" + line = f"[gpu {label} {poll_n}/?] {node_tag}used_vram={used} MB gfx={gfx}% umc={umc}% mm={mm}%{done_tag}" + log_lines.append(line) + if done: + break + + time.sleep(poll_interval_s) + + # Build summary + agg = agg_readings(readings) + n_failed = poll_n - len(readings) + failed_note = f" ({n_failed} failed, excluded)" if n_failed else "" + peak = agg.get("peak_gpu_memory_mb") + compute = agg.get("gpu_compute_util_pct") + bw = agg.get("gpu_bandwidth_util_pct") + ml_mem = f"{model_load_memory_mb:.0f}" if model_load_memory_mb is not None else "-" + ml_s = f"{model_load_s:.1f}" if model_load_s is not None else "-" + compute_s = f"{compute:.1f}" if compute is not None else "-" + bw_s = f"{bw:.1f}" if bw is not None else "-" + peak_s = f"{peak:.0f}" if peak is not None else "-" + + summary_lines = [ + "", + "--- summary ---", + f"samples: {poll_n}{failed_note}", + f"peak_gpu_memory_mb: {peak_s} MB", + f"model_load_memory_mb: {ml_mem} MB", + f"model_load_s: {ml_s} s", + f"gpu_compute_util_pct: {compute_s} %", + f"gpu_bandwidth_util_pct: {bw_s} %", + ] + if node_last_vram: + summary_lines.append("--- per-node vram (last reading) ---") + for lbl, vram in node_last_vram.items(): + vram_s = f"{vram}" if vram is not None else "-" + summary_lines.append(f"node_vram_mb [{lbl}]: {vram_s} MB") + log_lines.extend(summary_lines) + + if log_path is not None: + try: + pathlib.Path(log_path).write_text("\n".join(log_lines) + "\n") + except Exception as exc: + log.warning("poll_gpu_metrics: failed to write log %s: %s", log_path, exc) + + log.info( + "poll_gpu_metrics: %d readings (%d failed) | peak_vram=%s MB compute=%s%% bw=%s%%", + len(readings), + n_failed, + peak_s, + compute_s, + bw_s, + ) + return readings diff --git a/cvs/lib/utils/ib_discovery.py b/cvs/lib/utils/ib_discovery.py new file mode 100644 index 000000000..8f9b298b3 --- /dev/null +++ b/cvs/lib/utils/ib_discovery.py @@ -0,0 +1,231 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +InfiniBand HCA discovery via ibv_devinfo or /sys/class/infiniband fallback. + +Shared across suites (vllm, rccl, inferencemax). Infrastructure step, not +a benchmark step: topology is stable within a run and should be probed once. +''' + +from __future__ import annotations + +import re +import shlex + +from cvs.lib import globals + +log = globals.log + +_IB_HCA_NETDEV_RE = re.compile(r"^mlx5_\d+$", re.I) +_HCA_NAME_RE = re.compile(r"^(mlx5_\d+|rdma\d+|rocep\w+|bnxt_\w+)$", re.I) +_NETDEV_NAME_RE = re.compile(r"^[a-zA-Z0-9_.:-]{1,64}$") +_INVALID_NETDEV_MARKERS = ( + "command not found", + "syntax error", + "bash:", + "no such file", + "cannot open", +) + +_SYSFS_CMD = "ls /sys/class/infiniband/ 2>/dev/null | tr '\\n' ' '" +_IBVDEVINFO_CMD = "ibv_devinfo -l 2>/dev/null" + + +def _topology_exec(orch, cmd, hosts=None): + """Run topology probes on the host OS when the orchestrator is container-backed.""" + host_exec = getattr(orch, "exec_on_host", None) + if callable(host_exec): + return host_exec(cmd, hosts=hosts) + return orch.exec(cmd, hosts=hosts) + + +def _parse_sysfs(output: str) -> list[str]: + return [tok for tok in _parse_ibv_devinfo_list(output)] + + +def _parse_ibv_devinfo_list(output: str) -> list[str]: + """Parse ``ibv_devinfo -l`` output, ignoring banner lines like ``8 HCAs found:``.""" + if not (output or "").strip(): + return [] + tokens: list[str] = [] + for line in (output or "").splitlines(): + line = line.strip() + if not line or line.lower().startswith("warning"): + continue + for tok in line.split(): + if _HCA_NAME_RE.match(tok): + tokens.append(tok) + return tokens + + +def _valid_netdev_name(name: str) -> bool: + name = (name or "").strip() + if not name or not _NETDEV_NAME_RE.match(name): + return False + lower = name.lower() + if any(marker in lower for marker in _INVALID_NETDEV_MARKERS): + return False + if _IB_HCA_NETDEV_RE.match(name): + return False + return True + + +def discover_ib_hca_names(orch) -> dict[str, list[str]]: + """Return {host: [hca_name, ...]} for all hosts in orch. + + Tries ``ibv_devinfo -l`` first; falls back to listing + ``/sys/class/infiniband/`` when ibv_devinfo is absent from the image. + Returns HCA names (e.g. ``rocep28s0``, ``mlx5_0``), correct for + ``NCCL_IB_HCA``. These are NOT Linux netdev names (``ens51f1np1``) -- + those belong in ``ib_netdev`` in the suite config. + + Raises ``RuntimeError`` if: + - any host returns an empty HCA list (indicates missing driver or no IB + hardware on that node), or + - the HCA lists are asymmetric across nodes (a hardware/driver mismatch + must surface loudly, not be silently papered over by intersection). + """ + raw = _topology_exec(orch, _IBVDEVINFO_CMD) + result: dict[str, list[str]] = {} + use_sysfs = False + for host, output in (raw or {}).items(): + hcas = _parse_ibv_devinfo_list(output or "") + if not hcas: + use_sysfs = True + break + result[host] = hcas + + if use_sysfs: + log.info("ib_discovery: ibv_devinfo unavailable or empty; falling back to /sys/class/infiniband") + raw = _topology_exec(orch, _SYSFS_CMD) + result = {} + for host, output in (raw or {}).items(): + hcas = _parse_sysfs(output) + log.info("ib_discovery (sysfs): %s -> %s", host, hcas) + result[host] = hcas + else: + for host, hcas in result.items(): + log.info("ib_discovery: %s -> %s", host, hcas) + + empty = [h for h, devs in result.items() if not devs] + if empty: + raise RuntimeError( + f"ib_discovery: no IB HCA devices found on {empty}. " + "Check that ibv_devinfo is installed and IB drivers are loaded." + ) + + lists = [tuple(sorted(devs)) for devs in result.values()] + if len(set(lists)) > 1: + detail = "; ".join(f"{h}={devs}" for h, devs in result.items()) + raise RuntimeError( + f"ib_discovery: asymmetric HCA device lists across nodes ({detail}). " + "Investigate hardware/driver mismatch before running." + ) + + return result + + +def validate_ib_hca_preflight(discovered: dict[str, list[str]], requested: list[str]) -> None: + """Raise if any requested HCA name is absent from any node's discovered list. + + Called when the config provides an explicit ``ib_hca_devices`` list (not + absent/``"auto"``). Fails loudly naming the missing devices and the node, + so the operator knows exactly which device is wrong rather than getting a + cryptic NCCL error later. + """ + for host, devs in discovered.items(): + missing = [d for d in requested if d not in devs] + if missing: + raise RuntimeError( + f"ib_discovery preflight: requested HCA devices {missing} not found on {host}. Available: {devs}" + ) + + +def _netdev_for_ip_cmd(ip: str) -> str: + inner = ( + f"IF=$((ip -4 -o addr show 2>/dev/null || /sbin/ip -4 -o addr show 2>/dev/null) | " + f"awk -v ip={shlex.quote(ip)} '{{split($4,a,\"/\"); if(a[1]==ip) {{print $2; exit}}}}'); " + 'echo "${IF}"' + ) + return f"bash -c {shlex.quote(inner)}" + + +def _netdev_via_route_cmd(dest_ip: str) -> str: + inner = ( + f"IF=$((ip route get {shlex.quote(dest_ip)} 2>/dev/null || " + f"/sbin/ip route get {shlex.quote(dest_ip)} 2>/dev/null) | awk " + "'{{for(i=1;i<=NF;i++) if($i==\"dev\") {{print $(i+1); exit}}}}'); " + 'echo "${IF}"' + ) + return f"bash -c {shlex.quote(inner)}" + + +def discover_socket_netdev_name(orch, master_addr: str | None = None) -> str: + """Return the Linux netdev for NCCL/GLOO socket traffic on a homogeneous cluster. + + On each host, prefers the interface that owns that host's cluster IP (the key + in ``orch.hosts``). Falls back to the egress interface toward ``master_addr``. + Requires the same netdev **name** on every node because the suite broadcasts + one env script to all ranks. + + These are IP netdevs (``ens51f1np1``), not IB HCA names (``mlx5_0``). + """ + hosts = list(getattr(orch, "hosts", []) or []) + if not hosts: + raise RuntimeError("socket_netdev discovery: orchestrator has no hosts") + + master = (master_addr or "").strip() or hosts[0] + per_host: dict[str, str] = {} + for host in hosts: + host_ip = str(host).strip() + out = _topology_exec(orch, _netdev_for_ip_cmd(host_ip), hosts=[host]) + netdev = (out or {}).get(host, "").strip() + if not _valid_netdev_name(netdev): + out = _topology_exec(orch, _netdev_via_route_cmd(master), hosts=[host]) + netdev = (out or {}).get(host, "").strip() + if not _valid_netdev_name(netdev): + raise RuntimeError( + f"socket_netdev discovery: no IPv4 netdev on {host} " + f"(host_ip={host_ip!r}, master_addr={master!r}, last_output={netdev!r})" + ) + per_host[host] = netdev + log.info("socket_netdev discovery: %s -> %s", host, netdev) + + unique = set(per_host.values()) + if len(unique) > 1: + detail = "; ".join(f"{h}={d}" for h, d in per_host.items()) + raise RuntimeError( + f"socket_netdev discovery: asymmetric netdev names across nodes ({detail}). " + "Set roles.server.ib_netdev explicitly when node interface names differ." + ) + return next(iter(unique)) + + +def resolve_multinode_fabric( + orch, + *, + ib_hca_devices=None, + ib_netdev=None, + master_addr=None, +) -> tuple[list[str], str]: + """Resolve ``NCCL_IB_HCA`` devices and the socket netdev for a multinode run. + + Used by ``test_discover_topology`` (once per lifecycle) and lazily by + ``InferenceXAtomJob.build_server_cmd`` when a partial ``-k`` filter skips + the topology test. + """ + discovered = discover_ib_hca_names(orch) + if ib_hca_devices and ib_hca_devices != "auto": + validate_ib_hca_preflight(discovered, ib_hca_devices) + hcas = list(ib_hca_devices) + else: + hcas = list(next(iter(discovered.values()))) + + configured = (ib_netdev or "").strip() + master = (master_addr or "").strip() or orch.hosts[0] + if configured and configured.lower() != "auto": + netdev = configured + else: + netdev = discover_socket_netdev_name(orch, master_addr=master) + return hcas, netdev diff --git a/cvs/lib/utils/model_query_lib.py b/cvs/lib/utils/model_query_lib.py new file mode 100644 index 000000000..76240b144 --- /dev/null +++ b/cvs/lib/utils/model_query_lib.py @@ -0,0 +1,518 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. +''' + +from __future__ import annotations + +import json +import re +import shlex +from typing import Any, Mapping, Optional + +JsonDict = dict[str, Any] + +# Keys passed from scoring_config into LmEvalBenchmark.check_results(). +LM_EVAL_CHECK_RESULT_KEYS = ( + "task_name", + "parse_metric", + "expected", + "metric_key", + "tasks", + "tolerance_frac", + "log_path", + "label", +) + + +class OpenAIProbe: + """OpenAI-compatible HTTP smoke tests (stdlib probe script + validation).""" + + TIMEOUT_S = 120.0 + CHAT_MAX_TOKENS = 8 + COMPLETION_MAX_TOKENS = 50 + STRUCTURED_BOOK_MAX_TOKENS = 256 + + CHAT_USER = "Reply with exactly one word: OK." + COMPLETION_PROMPT = "The capital of France is" + STRUCTURED_BOOK_SYSTEM = "Respond with a single JSON object only. No markdown or text outside the JSON." + STRUCTURED_BOOK_USER = ( + "Return one book as a JSON object with keys: title (string), author (string), year (integer), genre (string)." + ) + + STEP_TITLES: dict[str, str] = { + "model_endpoint": "Model endpoint — GET /v1/models", + "chat_completion_endpoint": "Chat completion endpoint — POST /v1/chat/completions", + "completion_endpoint": "Completion endpoint — POST /v1/completions", + "structured_output_book": ( + "Structured output (book) — POST /v1/chat/completions (response_format: json_object)" + ), + } + + _FAILURE_MARKER = "OpenAI-compatible probe failed port=" + + @classmethod + def probe_script( + cls, + port: int, + model: str, + *, + timeout_s: float = TIMEOUT_S, + chat_max_tokens: int = CHAT_MAX_TOKENS, + completion_max_tokens: int = COMPLETION_MAX_TOKENS, + structured_book_max_tokens: int = STRUCTURED_BOOK_MAX_TOKENS, + ) -> str: + """Stdlib-only Python source for docker exec / orch.exec.""" + chat_messages = [{"role": "user", "content": cls.CHAT_USER}] + book_messages = [ + {"role": "system", "content": cls.STRUCTURED_BOOK_SYSTEM}, + {"role": "user", "content": cls.STRUCTURED_BOOK_USER}, + ] + return "\n".join( + [ + "import json, urllib.request, urllib.error", + f"PORT = {int(port)}", + f"TIMEOUT = {float(timeout_s)}", + f"CHAT_MAX = {int(chat_max_tokens)}", + f"COMP_MAX = {int(completion_max_tokens)}", + f"BOOK_MAX = {int(structured_book_max_tokens)}", + f"MODEL = {json.dumps(model)}", + 'BASE = "http://0.0.0.0:%d" % PORT', + "", + "def req(method, path, body=None):", + " url = BASE + path", + " data = None if body is None else json.dumps(body).encode('utf-8')", + " hdrs = {}", + " if body is not None:", + ' hdrs["Content-Type"] = "application/json"', + " r = urllib.request.Request(url, data=data, headers=hdrs, method=method)", + " try:", + " with urllib.request.urlopen(r, timeout=TIMEOUT) as resp:", + ' raw = resp.read().decode("utf-8", errors="replace")', + " code = int(getattr(resp, 'status', 200))", + " try:", + " return code, json.loads(raw) if raw else {}", + " except json.JSONDecodeError:", + " return code, raw", + " except urllib.error.HTTPError as e:", + ' raw = e.read().decode("utf-8", errors="replace")', + " try:", + " return int(e.code), json.loads(raw) if raw else {}", + " except json.JSONDecodeError:", + " return int(e.code), raw", + "", + "out = {}", + 'out["model_endpoint"] = req("GET", "/v1/models")', + 'out["chat_completion_endpoint"] = req("POST", "/v1/chat/completions", {', + ' "model": MODEL,', + f' "messages": {json.dumps(chat_messages)},', + ' "max_tokens": CHAT_MAX,', + ' "temperature": 0.0,', + "})", + 'out["completion_endpoint"] = req("POST", "/v1/completions", {', + ' "model": MODEL,', + f' "prompt": {json.dumps(cls.COMPLETION_PROMPT)},', + ' "max_tokens": COMP_MAX,', + ' "temperature": 0.0,', + "})", + 'out["structured_output_book"] = req("POST", "/v1/chat/completions", {', + ' "model": MODEL,', + f' "messages": {json.dumps(book_messages)},', + ' "max_tokens": BOOK_MAX,', + ' "temperature": 0.0,', + ' "response_format": {"type": "json_object"},', + "})", + "print(json.dumps(out, default=str))", + ] + ) + + @classmethod + def log_results(cls, results: dict[str, tuple[int, Any]], logger: Any) -> None: + """One headed block per endpoint for logs / HTML capture.""" + for step, (code, body) in results.items(): + title = cls.STEP_TITLES.get(step, step) + logger.info("#================ %s ================#", title) + logger.info("HTTP status: %s", code) + if isinstance(body, (dict, list)): + try: + logger.info("Body:\n%s", json.dumps(body, indent=2, default=str)) + except (TypeError, ValueError): + logger.info("Body: %s", body) + else: + logger.info("Body: %s", body) + + @classmethod + def check_results( + cls, + results: dict[str, tuple[int, Any]], + *, + port: Any, + logger: Optional[Any] = None, + ) -> tuple[bool, Optional[str]]: + """ + Each step: HTTP 200, JSON object body, and minimal useful content. + """ + failures: list[str] = [] + + def _fail(detail: str) -> None: + failures.append(detail) + if logger is not None: + logger.info( + "OpenAI-compatible probe check FAILED: %s port=%r", + detail, + port, + ) + + for step, (code, body) in results.items(): + title = cls.STEP_TITLES.get(step, step) + if code != 200: + _fail(f"{title} (step={step!r}): HTTP status={code} body={body!r}") + continue + if body is None: + _fail(f"{title}: response body is None") + continue + if not isinstance(body, dict): + _fail(f"{title}: body is not a JSON object ({type(body).__name__})") + continue + + if step == "model_endpoint": + data = body.get("data") + if not isinstance(data, list) or not data: + _fail(f"{title}: missing or empty models list") + continue + first = data[0] + if not isinstance(first, dict) or not str(first.get("id") or first.get("model") or "").strip(): + _fail(f"{title}: no model id in models response") + continue + + if not str(body.get("model") or "").strip(): + _fail(f"{title}: response has no model field") + continue + + choices = body.get("choices") + if not isinstance(choices, list) or not choices: + _fail(f"{title}: missing or empty choices") + continue + ch0 = choices[0] + if not isinstance(ch0, dict): + _fail(f"{title}: choices[0] is not an object") + continue + + if step == "completion_endpoint": + if not str(ch0.get("text") or "").strip(): + _fail(f"{title}: empty completion text") + continue + + if step in ("chat_completion_endpoint", "structured_output_book"): + msg = ch0.get("message") + if not isinstance(msg, dict): + _fail(f"{title}: choices[0].message is not an object") + continue + content = msg.get("content") + if content is None or not str(content).strip(): + _fail(f"{title}: empty assistant content") + continue + if step == "structured_output_book": + try: + obj = json.loads(str(content)) + except json.JSONDecodeError as e: + _fail(f"{title}: assistant content is not JSON ({e!r})") + continue + if not isinstance(obj, dict): + _fail(f"{title}: parsed content is not a JSON object") + continue + for key in ("title", "author", "year", "genre"): + if key not in obj: + _fail(f"{title}: JSON missing {key!r}") + break + v = obj[key] + if key == "year": + if isinstance(v, (int, float)): + continue + if not str(v).strip(): + _fail(f"{title}: JSON year empty") + break + elif not str(v).strip(): + _fail(f"{title}: JSON field {key!r} empty") + break + continue + + if failures: + return False, f"{cls._FAILURE_MARKER}{port!r}: " + " | ".join(failures) + return True, None + + @classmethod + def summarize_results( + cls, + results: dict[str, tuple[int, Any]], + ok: bool, + err: Optional[str], + ) -> list[str]: + """Human-readable per-step pass/fail lines for logs and inf_res_dict.""" + failure_parts: list[str] = [] + if not ok and err and err.startswith(cls._FAILURE_MARKER): + rest = err[len(cls._FAILURE_MARKER) :] + colon_idx = rest.find(": ") + if colon_idx != -1: + failure_parts = [p.strip() for p in rest[colon_idx + 2 :].split("|")] + + summary: list[str] = [] + for step, (status, _content) in results.items(): + title = cls.STEP_TITLES.get(step, step) + if ok: + outcome = "Pass" if status == 200 else "Fail" + elif status != 200: + outcome = "Fail" + elif any(p.startswith(title) or p.startswith(f"{title} (step=") for p in failure_parts): + outcome = "Fail" + else: + outcome = "Pass" + summary.append(f"{title} -> {outcome} ({status})") + return summary + + +class LmEvalBenchmark: + """lm-eval helpers for OpenAI-compatible local-* adapters.""" + + DEFAULT_NUM_CONCURRENT = "4" + DEFAULT_NUM_FEWSHOT = "5" + DEFAULT_BATCH_SIZE = "auto" + DEFAULT_LM_EVAL_MODEL = "local-completions" + DEFAULT_EXEC_TIMEOUT_SEC = 7200 + DEFAULT_TOLERANCE_FRAC = 0.05 + + @staticmethod + def parse_metric_value(text: str, task: str, metric: str) -> float | None: + """Parse a metric value from lm-eval's ASCII results table.""" + m = re.search( + rf"{re.escape(metric)}\s*\|\s*[^|\n]+\|\s*([0-9]+(?:\.[0-9]+)?)", + text, + flags=re.I, + ) + return float(m.group(1)) if m else None + + @staticmethod + def openai_base_url(port: int, lm_eval_model: str) -> str: + """Build base_url for lm-eval's local-completions / local-chat-completions.""" + path = "/v1/chat/completions" if "chat" in lm_eval_model.lower() else "/v1/completions" + return f"http://0.0.0.0:{int(port)}{path}" + + @classmethod + def build_model_args( + cls, + *, + model_id: str, + base_url: str, + num_concurrent: str, + extra_model_args: str = "", + ) -> str: + model_args = f"model={model_id},base_url={base_url},num_concurrent={num_concurrent},tokenized_requests=False" + extra = str(extra_model_args or "").strip() + if extra: + model_args = f"{model_args},{extra}" + return model_args + + @classmethod + def build_command( + cls, + *, + lm_eval_model: str, + model_id: str, + base_url: str, + tasks: str, + num_fewshot: str, + batch_size: str, + num_concurrent: str, + limit: str = "", + extra_model_args: str = "", + log_path: str, + pip_install: bool = True, + ) -> str: + """ + Inner shell fragment to run lm-eval (no docker/ssh). + Caller handles mkdir, source env, docker exec, etc. + """ + limit_s = str(limit or "").strip() + limit_arg = f" --limit {limit_s}" if limit_s else "" + model_args_q = shlex.quote( + cls.build_model_args( + model_id=model_id, + base_url=base_url, + num_concurrent=num_concurrent, + extra_model_args=extra_model_args, + ) + ) + parts: list[str] = [] + if pip_install: + parts.append("pip install -q 'lm-eval[api]'") + parts.append( + "python3 -m lm_eval " + f"--model {lm_eval_model} " + f"--model_args {model_args_q} " + f"--tasks {tasks} " + f"--num_fewshot {num_fewshot} " + f"--batch_size {batch_size}" + f"{limit_arg} " + f"2>&1 | tee {shlex.quote(log_path)}" + ) + return " && ".join(parts) + + @classmethod + def check_results( + cls, + text: str, + *, + task_name: str, + parse_metric: str, + expected: float, + metric_key: str, + tasks: str = "", + tolerance_frac: float = DEFAULT_TOLERANCE_FRAC, + log_path: str = "", + label: str = "", + ) -> tuple[bool, dict[str, Any] | None, str | None]: + """ + Validate lm-eval stdout after exec. + + Returns ``(ok, summary, err)`` where summary is:: + + {"task", "metric_key", "actual", "expected"} + """ + display = label or task_name + log_hint = f" (see {log_path})" if log_path else "" + + if not text or not str(text).strip(): + return False, None, f"lm-eval {display} produced no output" + if re.search(r"Traceback \(most recent call last\)", text): + return False, None, f"lm-eval {display} failed: Python traceback in output" + if not re.search(re.escape(task_name), text, re.I): + return ( + False, + None, + f"lm-eval {display} output missing {task_name!r} results{log_hint}", + ) + + actual = cls.parse_metric_value(text, task_name, parse_metric) + if actual is None: + return ( + False, + None, + f"could not parse lm-eval table score for {task_name} / {parse_metric}", + ) + + expected_f = float(expected) + if abs(actual - expected_f) > tolerance_frac * abs(expected_f): + short_metric = "flexible-extract" if "flexible" in metric_key.lower() else parse_metric + err = ( + f"{task_name} {short_metric} {actual:.4f} not within " + f"{tolerance_frac * 100:.0f}% of expected {expected_f:.4f}" + ) + summary = { + "task": str(tasks or task_name), + "metric_key": metric_key, + "actual": float(actual), + "expected": expected_f, + "passed": False, + } + return False, summary, err + summary = { + "task": str(tasks or task_name), + "metric_key": metric_key, + "actual": float(actual), + "expected": expected_f, + "passed": True, + } + return True, summary, None + + @classmethod + def prepare( + cls, + i_dict: Mapping[str, Any], + *, + port: int, + model_id: str, + task_name: str, + default_tasks: str, + default_metric: str, + default_metric_key: str, + log_dir: str, + log_basename: str, + default_num_concurrent: str = DEFAULT_NUM_CONCURRENT, + ) -> tuple[str, dict[str, Any]]: + """ + Resolve config and build the inner lm-eval shell command. + + Returns ``(inner_cmd, scoring_config)``. + """ + lm_eval_model = str(i_dict.get("lm_eval_model", cls.DEFAULT_LM_EVAL_MODEL)) + base_url = cls.openai_base_url(port, lm_eval_model) + num_concurrent = str(i_dict.get("num_concurrent", default_num_concurrent)) + tasks = str(i_dict.get("tasks", default_tasks)) + num_fewshot = str(i_dict.get("num_fewshot", cls.DEFAULT_NUM_FEWSHOT)) + batch_size = str(i_dict.get("batch_size", cls.DEFAULT_BATCH_SIZE)) + limit = str(i_dict.get("limit", "")).strip() + extra_model_args = str(i_dict.get("extra_model_args", "")).strip() + exec_timeout_sec = int(i_dict.get("exec_timeout_sec", cls.DEFAULT_EXEC_TIMEOUT_SEC)) + tolerance_frac = float(i_dict.get("tolerance_frac", cls.DEFAULT_TOLERANCE_FRAC)) + log_path = f"{log_dir.rstrip('/')}/benchmark_node/{log_basename}" + + expected_block = i_dict.get("expected_results") or {} + if not isinstance(expected_block, Mapping): + raise ValueError("expected_results must be a mapping") + task_expected = expected_block.get(task_name) or {} + if not isinstance(task_expected, Mapping): + raise ValueError(f"expected_results[{task_name!r}] must be a mapping") + if default_metric_key not in task_expected: + raise KeyError(f"expected_results[{task_name!r}][{default_metric_key!r}] missing") + expected = float(task_expected[default_metric_key]) + + inner_cmd = cls.build_command( + lm_eval_model=lm_eval_model, + model_id=model_id, + base_url=base_url, + tasks=tasks, + num_fewshot=num_fewshot, + batch_size=batch_size, + num_concurrent=num_concurrent, + limit=limit, + extra_model_args=extra_model_args, + log_path=log_path, + pip_install=bool(i_dict.get("pip_install", True)), + ) + + scoring_config: dict[str, Any] = { + "task_name": task_name, + "parse_metric": default_metric, + "metric_key": default_metric_key, + "expected": expected, + "tasks": tasks, + "tolerance_frac": tolerance_frac, + "log_path": log_path, + "exec_timeout_sec": exec_timeout_sec, + "label": str(i_dict.get("label", task_name)), + "lm_eval_model": lm_eval_model, + "base_url": base_url, + } + return inner_cmd, scoring_config + + @classmethod + def check_kwargs_from_scoring(cls, scoring: Mapping[str, Any]) -> dict[str, Any]: + """Extract kwargs for check_results() from a scoring_config dict.""" + return {k: scoring[k] for k in LM_EVAL_CHECK_RESULT_KEYS} + + @classmethod + def fallback_summary( + cls, + scoring: Mapping[str, Any], + *, + actual: float | None = None, + error: str | None = None, + ) -> dict[str, Any]: + return { + "task": str(scoring.get("tasks") or scoring["task_name"]), + "metric_key": scoring["metric_key"], + "actual": actual, + "expected": float(scoring["expected"]), + "passed": False, + "error": error, + } diff --git a/cvs/lib/utils/unittests/__init__.py b/cvs/lib/utils/unittests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cvs/lib/utils/unittests/test_config_loader.py b/cvs/lib/utils/unittests/test_config_loader.py new file mode 100644 index 000000000..e769c1873 --- /dev/null +++ b/cvs/lib/utils/unittests/test_config_loader.py @@ -0,0 +1,459 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for cvs.lib.utils.config_loader. + +Covers the public contract described in the spec: + - ModelSpec accepts an optional ``precision`` field (defaults to empty string). + - BaseVariantConfig accepts an optional ``threshold_json`` field (defaults to ""). + - substitute_config reads the threshold file from ``threshold_json`` when set + (literal absolute path), or discovers a sole sibling ``*threshold.json``. + - substitute_config raises FileNotFoundError when ``threshold_json`` names a + non-existent file, or when no sibling threshold exists and the field is empty. + - Multiple sibling ``*threshold.json`` files raise ValueError (ambiguous). + +Framework: unittest.TestCase + self.subTest + unittest.mock (no pytest). +''' + +import json +import tempfile +import unittest +from pathlib import Path +from pydantic import ValidationError + +from cvs.lib.utils.config_loader import ( + BaseVariantConfig, + ContainerSpec, + ModelSpec, + Paths, + substitute_config, +) + +# --------------------------------------------------------------------------- +# Shared helpers +# --------------------------------------------------------------------------- + + +def _paths_dict(): + return { + "shared_fs": "/data/shared", + "models_dir": "/data/models", + "log_dir": "/data/logs", + "hf_token_file": "/data/.hf_token", + } + + +def _model_dict(): + """A valid ModelSpec payload.""" + return {"id": "amd/Llama-3.1-70B-Instruct", "remote": 0} + + +def _container_dict(): + return { + "name": "my-container", + "image": "rocm/vllm-dev:nightly", + "runtime": {"name": "docker"}, + } + + +def _base_config_dict(threshold_json: str = ""): + """Minimal dict that satisfies BaseVariantConfig.""" + d = { + "schema_version": 1, + "paths": _paths_dict(), + "model": _model_dict(), + "container": _container_dict(), + } + if threshold_json: + d["threshold_json"] = threshold_json + return d + + +# --------------------------------------------------------------------------- +# ModelSpec — stateful pydantic model +# --------------------------------------------------------------------------- + + +class TestModelSpecLifecycle(unittest.TestCase): + """ModelSpec is a _Forbid model: valid construction, unknown fields rejected.""" + + # --- legal transitions --- + + def test_valid_construction_remote_zero(self): + spec = ModelSpec(id="some/model", remote=0) + self.assertEqual(spec.id, "some/model") + self.assertEqual(spec.remote, 0) + + def test_valid_construction_remote_one(self): + # remote=1 is accepted by the schema (validator rejects it at + # BaseVariantConfig level, not here) + spec = ModelSpec(id="some/model", remote=1) + self.assertEqual(spec.remote, 1) + + # --- illegal transitions (extra fields forbidden) --- + + def test_precision_field_is_optional(self): + """ModelSpec may carry an optional precision label (e.g. fp8 in filenames).""" + spec = ModelSpec(id="amd/llama", remote=0, precision="fp8") + self.assertEqual(spec.precision, "fp8") + + def test_precision_defaults_empty(self): + spec = ModelSpec(id="amd/llama", remote=0) + self.assertEqual(spec.precision, "") + + def test_arbitrary_extra_field_is_rejected(self): + with self.assertRaises(ValidationError): + ModelSpec(id="amd/llama", remote=0, unknown_key="value") + + def test_missing_id_is_rejected(self): + with self.assertRaises(ValidationError): + ModelSpec(remote=0) + + def test_missing_remote_is_rejected(self): + with self.assertRaises(ValidationError): + ModelSpec(id="amd/llama") + + def test_invalid_remote_value_is_rejected(self): + """remote must be Literal[0, 1] — other ints are invalid.""" + with self.assertRaises(ValidationError): + ModelSpec(id="amd/llama", remote=2) + + # --- idempotent re-entry --- + + def test_model_dict_roundtrip(self): + """model_dump / model_validate are idempotent.""" + spec = ModelSpec(id="amd/llama", remote=0) + dumped = spec.model_dump() + spec2 = ModelSpec.model_validate(dumped) + self.assertEqual(spec.id, spec2.id) + self.assertEqual(spec.remote, spec2.remote) + + +# --------------------------------------------------------------------------- +# BaseVariantConfig — stateful pydantic model +# --------------------------------------------------------------------------- + + +class TestBaseVariantConfigLifecycle(unittest.TestCase): + """BaseVariantConfig optional threshold_json and forbid extra fields.""" + + # --- legal transitions --- + + def test_valid_construction_with_threshold_json(self): + cfg = BaseVariantConfig(**_base_config_dict("/abs/path/threshold.json")) + self.assertEqual(cfg.threshold_json, "/abs/path/threshold.json") + + def test_threshold_json_defaults_empty(self): + cfg = BaseVariantConfig(**_base_config_dict()) + self.assertEqual(cfg.threshold_json, "") + + def test_threshold_json_preserved_verbatim(self): + """threshold_json is stored as-is; no substitution or normalization.""" + path = "/some/deeply/nested/threshold.json" + cfg = BaseVariantConfig(**_base_config_dict(path)) + self.assertEqual(cfg.threshold_json, path) + + def test_enforce_thresholds_defaults_true(self): + cfg = BaseVariantConfig(**_base_config_dict()) + self.assertTrue(cfg.enforce_thresholds) + + def test_enforce_thresholds_can_be_false(self): + d = _base_config_dict() + d["enforce_thresholds"] = False + cfg = BaseVariantConfig(**d) + self.assertFalse(cfg.enforce_thresholds) + + def test_thresholds_defaults_to_empty_dict(self): + cfg = BaseVariantConfig(**_base_config_dict()) + self.assertEqual(cfg.thresholds, {}) + + # --- illegal transitions --- + + def test_missing_threshold_json_is_optional(self): + """threshold_json defaults to empty when omitted.""" + d = _base_config_dict() + self.assertNotIn("threshold_json", d) + cfg = BaseVariantConfig(**d) + self.assertEqual(cfg.threshold_json, "") + + def test_extra_field_is_rejected(self): + d = _base_config_dict() + d["unexpected_key"] = "oops" + with self.assertRaises(ValidationError): + BaseVariantConfig(**d) + + def test_missing_schema_version_raises(self): + d = _base_config_dict() + del d["schema_version"] + with self.assertRaises(ValidationError): + BaseVariantConfig(**d) + + def test_invalid_schema_version_raises(self): + d = _base_config_dict() + d["schema_version"] = 2 + with self.assertRaises(ValidationError): + BaseVariantConfig(**d) + + def test_remote_one_raises_not_implemented(self): + """model.remote==1 triggers the _check_remote_not_implemented guard.""" + d = _base_config_dict() + d["model"] = {"id": "some/model", "remote": 1} + with self.assertRaises((ValidationError, NotImplementedError)): + BaseVariantConfig(**d) + + # --- idempotent re-entry --- + + def test_model_validate_roundtrip(self): + cfg = BaseVariantConfig(**_base_config_dict("/t.json")) + dumped = cfg.model_dump() + cfg2 = BaseVariantConfig.model_validate(dumped) + self.assertEqual(cfg.threshold_json, cfg2.threshold_json) + self.assertEqual(cfg.schema_version, cfg2.schema_version) + + +# --------------------------------------------------------------------------- +# Paths — stateful pydantic model +# --------------------------------------------------------------------------- + + +class TestPathsLifecycle(unittest.TestCase): + """Paths is _Forbid — required fields must be present; extras rejected.""" + + def test_valid_construction(self): + p = Paths(**_paths_dict()) + self.assertEqual(p.shared_fs, "/data/shared") + + def test_extra_field_rejected(self): + d = dict(_paths_dict()) + d["extra"] = "x" + with self.assertRaises(ValidationError): + Paths(**d) + + def test_missing_field_rejected(self): + for key in ["shared_fs", "models_dir", "log_dir", "hf_token_file"]: + with self.subTest(missing=key): + d = dict(_paths_dict()) + del d[key] + with self.assertRaises(ValidationError): + Paths(**d) + + +# --------------------------------------------------------------------------- +# ContainerSpec — stateful pydantic model +# --------------------------------------------------------------------------- + + +class TestContainerSpecLifecycle(unittest.TestCase): + """ContainerSpec is _Forbid; lifetime has a default.""" + + def test_valid_construction_defaults(self): + spec = ContainerSpec(**_container_dict()) + self.assertEqual(spec.lifetime, "per_run") + + def test_valid_lifetime_values(self): + for lifetime in ["no_launch", "per_run", "persistent"]: + with self.subTest(lifetime=lifetime): + d = dict(_container_dict()) + d["lifetime"] = lifetime + spec = ContainerSpec(**d) + self.assertEqual(spec.lifetime, lifetime) + + def test_invalid_lifetime_rejected(self): + d = dict(_container_dict()) + d["lifetime"] = "never" + with self.assertRaises(ValidationError): + ContainerSpec(**d) + + def test_extra_field_rejected(self): + d = dict(_container_dict()) + d["bogus"] = "x" + with self.assertRaises(ValidationError): + ContainerSpec(**d) + + def test_missing_image_rejected(self): + d = dict(_container_dict()) + del d["image"] + with self.assertRaises(ValidationError): + ContainerSpec(**d) + + +# --------------------------------------------------------------------------- +# substitute_config — pure function (with I/O side-effects via filesystem) +# --------------------------------------------------------------------------- + + +class TestSubstituteConfigThresholdJsonField(unittest.TestCase): + """substitute_config threshold discovery: explicit path or sibling glob.""" + + def _write_config(self, tmp_dir: Path, config_dict: dict) -> Path: + config_path = tmp_dir / "variant_config.json" + config_path.write_text(json.dumps(config_dict)) + return config_path + + def _write_threshold(self, tmp_dir: Path, name: str = "threshold.json") -> Path: + threshold_path = tmp_dir / name + threshold_path.write_text(json.dumps({"ISL=128,OSL=2048,TP=8,CONC=16": {}})) + return threshold_path + + def test_happy_path_reads_from_threshold_json_field(self): + """Config with valid threshold_json pointing to a real file must load.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_dir = Path(tmp) + threshold_path = self._write_threshold(tmp_dir) + cfg = _base_config_dict(str(threshold_path)) + config_path = self._write_config(tmp_dir, cfg) + cluster_dict = {} + raw, thresholds = substitute_config(config_path, cluster_dict) + self.assertEqual(raw["schema_version"], 1) + + def test_sibling_threshold_discovered_when_threshold_json_empty(self): + """When threshold_json is omitted, a sole sibling *threshold.json is used.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_dir = Path(tmp) + self._write_threshold(tmp_dir, "mi300x_variant_threshold.json") + cfg = _base_config_dict() + config_path = self._write_config(tmp_dir, cfg) + raw, thresholds = substitute_config(config_path, {}) + self.assertIsInstance(thresholds, dict) + self.assertIn("ISL=128,OSL=2048,TP=8,CONC=16", thresholds) + + def test_sibling_threshold_ambiguous_raises_value_error(self): + with tempfile.TemporaryDirectory() as tmp: + tmp_dir = Path(tmp) + self._write_threshold(tmp_dir, "a_threshold.json") + self._write_threshold(tmp_dir, "b_threshold.json") + config_path = self._write_config(tmp_dir, _base_config_dict()) + with self.assertRaises(ValueError): + substitute_config(config_path, {}) + + def test_no_sibling_and_empty_threshold_json_raises_file_not_found(self): + with tempfile.TemporaryDirectory() as tmp: + config_path = self._write_config(Path(tmp), _base_config_dict()) + with self.assertRaises(FileNotFoundError): + substitute_config(config_path, {}) + + def test_threshold_json_as_absolute_path_no_sibling_glob(self): + """The threshold file must be read from the explicit path, not discovered + via glob. Even when no sibling *threshold.json exists, a valid explicit + path must succeed.""" + with tempfile.TemporaryDirectory() as config_dir, tempfile.TemporaryDirectory() as threshold_dir: + # Put threshold file in a DIFFERENT directory than the config + threshold_path = Path(threshold_dir) / "my_threshold.json" + threshold_path.write_text(json.dumps({"cell": {}})) + + cfg = dict(_base_config_dict(str(threshold_path))) + config_path = self._write_config(Path(config_dir), cfg) + cluster_dict = {} + # Must succeed even though no *threshold.json exists in config_dir + raw, thresholds = substitute_config(config_path, cluster_dict) + self.assertIsNotNone(raw) + + def test_missing_threshold_file_raises_file_not_found(self): + """If threshold_json names a non-existent file, FileNotFoundError is raised.""" + with tempfile.TemporaryDirectory() as tmp: + cfg = _base_config_dict("/nonexistent/path/threshold.json") + config_path = self._write_config(Path(tmp), cfg) + cluster_dict = {} + with self.assertRaises(FileNotFoundError): + substitute_config(config_path, cluster_dict) + + def test_explicit_threshold_json_ignores_sibling_when_path_missing(self): + """An explicit but missing threshold_json path fails even if a sibling exists.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_dir = Path(tmp) + self._write_threshold(tmp_dir, "sibling_threshold.json") + cfg = _base_config_dict("/does/not/exist/threshold.json") + config_path = self._write_config(tmp_dir, cfg) + with self.assertRaises(FileNotFoundError): + substitute_config(config_path, {}) + + def test_threshold_json_value_not_placeholder_substituted(self): + """The threshold_json value is a literal absolute path; placeholders in it + must NOT be substituted against the cluster dict.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_dir = Path(tmp) + # If placeholder substitution happened, {user-id} would be replaced + # and the path would change, potentially finding a different file. + # Write the file at the literal path (no substitution expected). + literal_path = tmp_dir / "threshold.json" + literal_path.write_text(json.dumps({})) + # Use a path that contains no placeholders — the real contract is + # the path is used verbatim. The test confirms successful load. + cfg = _base_config_dict(str(literal_path)) + config_path = self._write_config(tmp_dir, cfg) + raw, thresholds = substitute_config(config_path, {}) + self.assertIsInstance(thresholds, dict) + + def test_returns_tuple_of_raw_and_thresholds(self): + """substitute_config must return a (raw_dict, thresholds_dict) tuple.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_dir = Path(tmp) + threshold_data = {"ISL=128,OSL=2048,TP=8,CONC=16": {"metric": "v"}} + threshold_path = tmp_dir / "threshold.json" + threshold_path.write_text(json.dumps(threshold_data)) + cfg = dict(_base_config_dict(str(threshold_path))) + config_path = self._write_config(tmp_dir, cfg) + result = substitute_config(config_path, {}) + self.assertIsInstance(result, tuple) + self.assertEqual(len(result), 2) + raw, thresholds = result + self.assertIsInstance(raw, dict) + self.assertIsInstance(thresholds, dict) + + def test_threshold_comment_keys_stripped(self): + """Keys starting with '_' (comment keys) in the threshold file must be + stripped from the returned thresholds dict.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_dir = Path(tmp) + threshold_data = { + "_comment": "this is documentation", + "cell1": {"metric": "v"}, + } + threshold_path = tmp_dir / "threshold.json" + threshold_path.write_text(json.dumps(threshold_data)) + cfg = dict(_base_config_dict(str(threshold_path))) + config_path = self._write_config(tmp_dir, cfg) + _, thresholds = substitute_config(config_path, {}) + self.assertNotIn("_comment", thresholds) + self.assertIn("cell1", thresholds) + + +class TestSubstituteConfigPlaceholders(unittest.TestCase): + """Placeholder substitution behavior preserved from old behavior.""" + + def _write_files(self, tmp_dir: Path, config_dict: dict, threshold_dict: dict = None): + if threshold_dict is None: + threshold_dict = {} + threshold_path = tmp_dir / "threshold.json" + threshold_path.write_text(json.dumps(threshold_dict)) + cfg = dict(config_dict) + cfg["threshold_json"] = str(threshold_path) + config_path = tmp_dir / "variant_config.json" + config_path.write_text(json.dumps(cfg)) + return config_path + + def test_cluster_placeholder_substituted_in_paths(self): + """Cluster dict values are substituted for {key} placeholders.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_dir = Path(tmp) + cfg_base = _base_config_dict() + cfg_base["paths"]["log_dir"] = "/logs/{user-id}/run" + config_path = self._write_files(tmp_dir, cfg_base) + cluster = {"username": "jdoe"} + raw, _ = substitute_config(config_path, cluster) + self.assertEqual(raw["paths"]["log_dir"], "/logs/jdoe/run") + + def test_unknown_placeholder_left_verbatim(self): + """An unknown {token} that has no cluster mapping is left as-is (no error).""" + with tempfile.TemporaryDirectory() as tmp: + tmp_dir = Path(tmp) + cfg_base = _base_config_dict() + cfg_base["paths"]["log_dir"] = "/logs/{unknown-token}/run" + config_path = self._write_files(tmp_dir, cfg_base) + raw, _ = substitute_config(config_path, {}) + self.assertEqual(raw["paths"]["log_dir"], "/logs/{unknown-token}/run") + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/utils/unittests/test_gpu.py b/cvs/lib/utils/unittests/test_gpu.py new file mode 100644 index 000000000..cf8c157e8 --- /dev/null +++ b/cvs/lib/utils/unittests/test_gpu.py @@ -0,0 +1,1519 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for cvs.lib.utils.gpu. + +Black-box tests authored from the behavioral spec only (impl-blind). The module +contains pure parsers for `amd-smi metric --json` output: no I/O, no hardware, +pure dict transformations. + +Contract under test (from spec): + parse_usage(gpu_entry) -> {"gpu.gfx_activity", "gpu.umc_activity", + "gpu.mm_activity"}; int|None each. Degrades to + None for any missing key or "N/A" value; never raises. + parse_mem_usage(gpu_entry) -> {"gpu.total_vram", "gpu.used_vram", + "gpu.free_vram"}; int|None each. Degrades; never raises. + parse_energy(gpu_entry) -> {"gpu.energy_j"}; float|None. Degrades; never raises. + parse_gpu_metrics(raw) -> single dict with all 7 gpu.* keys. + activity fields averaged across GPUs; + vram + energy_j summed across GPUs. + [] -> all 7 keys present, all None. Never raises. + GPU_METRICS / GPU_METRIC_UNITS: every metric short_name has a matching unit; + parse_gpu_metrics([full]) emits "gpu." for every k. + +Framework: unittest.TestCase + self.subTest + unittest.mock (no pytest). +''' + +import unittest +from unittest.mock import MagicMock, patch + +from cvs.lib.utils.gpu import ( + GPU_METRICS, + GPU_METRIC_UNITS, + _RAW_GPU_FIELDS, + _RAW_GPU_FIELD_UNITS, + _mean, + agg_readings, + poll_gpu_metrics, + capture_gpu_metrics, + parse_usage, + parse_mem_usage, + parse_energy, + parse_gpu_metrics, +) + +# --------------------------------------------------------------------------- +# Shared fixtures — amd-smi JSON schema (one GPU entry) +# --------------------------------------------------------------------------- + +# The seven spec'd metrics, each as the bare "gpu." key produced by +# the parsers / aggregator. +ACTIVITY_KEYS = ["gpu.gfx_activity", "gpu.umc_activity", "gpu.mm_activity"] +VRAM_KEYS = ["gpu.total_vram", "gpu.used_vram", "gpu.free_vram"] +ENERGY_KEY = "gpu.energy_j" +ALL_KEYS = ACTIVITY_KEYS + VRAM_KEYS + [ENERGY_KEY] + + +def _full_gpu_entry(gfx=30, umc=20, mm=10, total=196608, used=4096, free=192512, energy=12345.5): + """A complete amd-smi entry for one GPU with all seven fields present.""" + return { + "usage": { + "gfx_activity": {"value": gfx}, + "umc_activity": {"value": umc}, + "mm_activity": {"value": mm}, + }, + "mem_usage": { + "total_vram": {"value": total}, + "used_vram": {"value": used}, + "free_vram": {"value": free}, + }, + "energy": { + "total_energy_consumption": {"value": energy}, + }, + } + + +# --------------------------------------------------------------------------- +# parse_usage — pure function (dict -> dict) +# --------------------------------------------------------------------------- + + +class TestParseUsage(unittest.TestCase): + """parse_usage extracts ["usage"]; degrades to None; never raises.""" + + def test_full_entry_extracts_all_three(self): + out = parse_usage(_full_gpu_entry(gfx=55, umc=44, mm=33)) + self.assertEqual( + out, + { + "gpu.gfx_activity": 55, + "gpu.umc_activity": 44, + "gpu.mm_activity": 33, + }, + ) + + def test_returns_exactly_the_three_activity_keys(self): + out = parse_usage(_full_gpu_entry()) + self.assertEqual(set(out.keys()), set(ACTIVITY_KEYS)) + + def test_value_types_are_int(self): + out = parse_usage(_full_gpu_entry(gfx=1, umc=2, mm=3)) + for k in ACTIVITY_KEYS: + with self.subTest(key=k): + self.assertIsInstance(out[k], int) + + def test_degradation_table(self): + """Each degraded-input shape maps every activity field to None. + + Boundary classes: empty dict, missing "usage", "N/A" string value. + """ + na = "N/A" + cases = [ + # (description, gpu_entry) + ("empty entry", {}), + ("missing usage key", {"mem_usage": {}}), + ( + "all fields N/A", + { + "usage": { + "gfx_activity": {"value": na}, + "umc_activity": {"value": na}, + "mm_activity": {"value": na}, + } + }, + ), + ] + expected = {k: None for k in ACTIVITY_KEYS} + for desc, entry in cases: + with self.subTest(case=desc): + self.assertEqual(parse_usage(entry), expected) + + def test_partial_entry_degrades_only_missing_field(self): + """One field missing/N/A -> None for that field; others extracted.""" + entry = { + "usage": { + "gfx_activity": {"value": 77}, + "umc_activity": {"value": "N/A"}, + # mm_activity entirely absent + } + } + out = parse_usage(entry) + self.assertEqual(out["gpu.gfx_activity"], 77) + self.assertIsNone(out["gpu.umc_activity"]) + self.assertIsNone(out["gpu.mm_activity"]) + + def test_zero_values_not_coerced_to_none(self): + """0 is a valid reading (fully idle GPU); must not degrade to None.""" + out = parse_usage(_full_gpu_entry(gfx=0, umc=0, mm=0)) + self.assertEqual(out["gpu.gfx_activity"], 0) + self.assertEqual(out["gpu.umc_activity"], 0) + self.assertEqual(out["gpu.mm_activity"], 0) + + def test_never_raises_on_malformed_shapes(self): + """Contract: degrades, never raises. Always returns all three keys as None.""" + malformed = [ + {}, + {"usage": {}}, + {"usage": {"gfx_activity": {}}}, + ] + for entry in malformed: + with self.subTest(entry=entry): + out = parse_usage(entry) + self.assertEqual(set(out.keys()), set(ACTIVITY_KEYS)) + for k in ACTIVITY_KEYS: + self.assertIsNone(out[k]) + + +# --------------------------------------------------------------------------- +# parse_mem_usage — pure function (dict -> dict) +# --------------------------------------------------------------------------- + + +class TestParseMemUsage(unittest.TestCase): + """parse_mem_usage extracts ["mem_usage"]; degrades; never raises.""" + + def test_full_entry_extracts_all_three(self): + out = parse_mem_usage(_full_gpu_entry(total=196608, used=4096, free=192512)) + self.assertEqual( + out, + { + "gpu.total_vram": 196608, + "gpu.used_vram": 4096, + "gpu.free_vram": 192512, + }, + ) + + def test_returns_exactly_the_three_vram_keys(self): + out = parse_mem_usage(_full_gpu_entry()) + self.assertEqual(set(out.keys()), set(VRAM_KEYS)) + + def test_value_types_are_int(self): + out = parse_mem_usage(_full_gpu_entry(total=10, used=3, free=7)) + for k in VRAM_KEYS: + with self.subTest(key=k): + self.assertIsInstance(out[k], int) + + def test_degradation_table(self): + na = "N/A" + cases = [ + ("empty entry", {}), + ("missing mem_usage", {"usage": {}}), + ( + "all N/A", + { + "mem_usage": { + "total_vram": {"value": na}, + "used_vram": {"value": na}, + "free_vram": {"value": na}, + } + }, + ), + ] + expected = {k: None for k in VRAM_KEYS} + for desc, entry in cases: + with self.subTest(case=desc): + self.assertEqual(parse_mem_usage(entry), expected) + + def test_partial_entry_degrades_only_missing_field(self): + entry = { + "mem_usage": { + "total_vram": {"value": 1000}, + "used_vram": {"value": "N/A"}, + # free_vram absent + } + } + out = parse_mem_usage(entry) + self.assertEqual(out["gpu.total_vram"], 1000) + self.assertIsNone(out["gpu.used_vram"]) + self.assertIsNone(out["gpu.free_vram"]) + + def test_zero_values_not_coerced_to_none(self): + """0 is a valid reading (idle GPU); must not degrade to None.""" + out = parse_mem_usage(_full_gpu_entry(total=0, used=0, free=0)) + self.assertEqual(out["gpu.total_vram"], 0) + self.assertEqual(out["gpu.used_vram"], 0) + self.assertEqual(out["gpu.free_vram"], 0) + + def test_never_raises_on_malformed_shapes(self): + malformed = [ + {}, + {"mem_usage": {}}, + {"mem_usage": {"used_vram": {}}}, + ] + for entry in malformed: + with self.subTest(entry=entry): + out = parse_mem_usage(entry) + self.assertEqual(set(out.keys()), set(VRAM_KEYS)) + for k in VRAM_KEYS: + self.assertIsNone(out[k]) + + +# --------------------------------------------------------------------------- +# parse_energy — pure function (dict -> dict) +# --------------------------------------------------------------------------- + + +class TestParseEnergy(unittest.TestCase): + """parse_energy extracts total_energy_consumption; degrades; never raises.""" + + def test_full_entry_extracts_energy(self): + out = parse_energy(_full_gpu_entry(energy=99999.25)) + self.assertEqual(out, {"gpu.energy_j": 99999.25}) + + def test_returns_exactly_the_energy_key(self): + out = parse_energy(_full_gpu_entry()) + self.assertEqual(set(out.keys()), {ENERGY_KEY}) + + def test_value_type_is_float(self): + out = parse_energy(_full_gpu_entry(energy=1.5)) + self.assertIsInstance(out[ENERGY_KEY], float) + + def test_degradation_table(self): + na = "N/A" + cases = [ + ("empty entry", {}), + ("missing energy", {"usage": {}}), + ("missing total_energy_consumption", {"energy": {}}), + ( + "N/A value", + {"energy": {"total_energy_consumption": {"value": na}}}, + ), + ] + for desc, entry in cases: + with self.subTest(case=desc): + self.assertEqual(parse_energy(entry), {ENERGY_KEY: None}) + + def test_never_raises_on_malformed_shapes(self): + malformed = [ + {}, + {"energy": {}}, + {"energy": {"total_energy_consumption": {}}}, + ] + for entry in malformed: + with self.subTest(entry=entry): + out = parse_energy(entry) + self.assertEqual(set(out.keys()), {ENERGY_KEY}) + self.assertIsNone(out[ENERGY_KEY]) + + def test_zero_energy_not_coerced_to_none(self): + """0.0 is a valid reading (GPU powered but idle); must not degrade to None.""" + out = parse_energy(_full_gpu_entry(energy=0.0)) + self.assertEqual(out[ENERGY_KEY], 0.0) + self.assertIsInstance(out[ENERGY_KEY], float) + + def test_int_energy_coerced_to_float(self): + """parse_energy must return float even when the raw value is a Python int.""" + out = parse_energy(_full_gpu_entry(energy=100)) + self.assertIsInstance(out[ENERGY_KEY], float) + + +# --------------------------------------------------------------------------- +# parse_gpu_metrics — pure aggregator (list -> dict) +# --------------------------------------------------------------------------- + + +class TestParseGpuMetrics(unittest.TestCase): + """Aggregates per-GPU entries: activity averaged, vram + energy summed.""" + + # --- key-presence contract --- + + def test_all_seven_keys_present_for_full_entry(self): + out = parse_gpu_metrics([_full_gpu_entry()]) + self.assertIsInstance(out, dict) + self.assertEqual(set(out.keys()), set(ALL_KEYS)) + + def test_empty_list_yields_all_keys_none(self): + """[] -> all 7 keys present, every value None. Never raises.""" + out = parse_gpu_metrics([]) + self.assertEqual(set(out.keys()), set(ALL_KEYS)) + for k in ALL_KEYS: + with self.subTest(key=k): + self.assertIsNone(out[k]) + + # --- single-GPU identity invariant --- + + def test_single_gpu_equals_that_gpus_values(self): + """Single GPU: averaged/summed result equals that GPU's values exactly.""" + entry = _full_gpu_entry(gfx=30, umc=20, mm=10, total=196608, used=4096, free=192512, energy=500.0) + out = parse_gpu_metrics([entry]) + self.assertEqual(out["gpu.gfx_activity"], 30) + self.assertEqual(out["gpu.umc_activity"], 20) + self.assertEqual(out["gpu.mm_activity"], 10) + self.assertEqual(out["gpu.total_vram"], 196608) + self.assertEqual(out["gpu.used_vram"], 4096) + self.assertEqual(out["gpu.free_vram"], 192512) + self.assertEqual(out["gpu.energy_j"], 500.0) + + # --- aggregation semantics: average vs sum --- + + def test_activity_fields_averaged_across_gpus(self): + """gfx/umc/mm averaged. Odd-sum pair verifies true division, not floor.""" + g0 = _full_gpu_entry(gfx=10, umc=40, mm=60) + g1 = _full_gpu_entry(gfx=21, umc=80, mm=20) + out = parse_gpu_metrics([g0, g1]) + self.assertEqual(out["gpu.gfx_activity"], 15.5) # (10+21)/2 — not 15 + self.assertEqual(out["gpu.umc_activity"], 60) # (40+80)/2 + self.assertEqual(out["gpu.mm_activity"], 40) # (60+20)/2 + + def test_vram_and_energy_summed_across_gpus(self): + """total/used/free_vram and energy_j summed across GPUs.""" + g0 = _full_gpu_entry(total=100, used=30, free=70, energy=1.5) + g1 = _full_gpu_entry(total=200, used=50, free=150, energy=2.5) + out = parse_gpu_metrics([g0, g1]) + self.assertEqual(out["gpu.total_vram"], 300) + self.assertEqual(out["gpu.used_vram"], 80) + self.assertEqual(out["gpu.free_vram"], 220) + self.assertEqual(out["gpu.energy_j"], 4.0) + + def test_activity_aggregation_is_average_not_sum(self): + """Guards against an impl that sums activity instead of averaging: + two equal nonzero GPUs must yield the per-GPU value, not double it.""" + g = _full_gpu_entry(gfx=50, umc=50, mm=50) + out = parse_gpu_metrics([g, _full_gpu_entry(gfx=50, umc=50, mm=50)]) + self.assertEqual(out["gpu.gfx_activity"], 50) + self.assertNotEqual(out["gpu.gfx_activity"], 100) + + def test_vram_aggregation_is_sum_not_average(self): + """Guards against an impl that averages vram/energy instead of summing: + two equal GPUs must total double, not stay equal.""" + g0 = _full_gpu_entry(total=100, used=40, free=60, energy=10.0) + g1 = _full_gpu_entry(total=100, used=40, free=60, energy=10.0) + out = parse_gpu_metrics([g0, g1]) + self.assertEqual(out["gpu.total_vram"], 200) + self.assertEqual(out["gpu.energy_j"], 20.0) + self.assertNotEqual(out["gpu.total_vram"], 100) + + # --- partial-entry aggregation --- + + def test_partial_entry_field_excluded_others_aggregated(self): + """A field missing on one GPU -> aggregate the remaining GPUs for it; + other fields still aggregate across all GPUs that have them.""" + g0 = _full_gpu_entry(gfx=20, total=100, used=40, free=60, energy=5.0) + # g1 has no usage block at all -> gfx None for g1 + g1 = { + "mem_usage": { + "total_vram": {"value": 200}, + "used_vram": {"value": 60}, + "free_vram": {"value": 140}, + }, + "energy": {"total_energy_consumption": {"value": 7.0}}, + } + out = parse_gpu_metrics([g0, g1]) + # activity only present on g0 -> aggregate is just g0's values + self.assertEqual(out["gpu.gfx_activity"], 20) + self.assertEqual(out["gpu.umc_activity"], 20) # g0 fixture default + self.assertEqual(out["gpu.mm_activity"], 10) # g0 fixture default + # vram present on both -> summed + self.assertEqual(out["gpu.total_vram"], 300) + self.assertEqual(out["gpu.used_vram"], 100) + self.assertEqual(out["gpu.free_vram"], 200) + # energy present on both -> summed + self.assertEqual(out["gpu.energy_j"], 12.0) + + def test_field_absent_on_all_gpus_yields_none(self): + """If no GPU supplies a field, the aggregate for that field is None, + while present fields still aggregate.""" + no_energy = { + "usage": { + "gfx_activity": {"value": 10}, + "umc_activity": {"value": 10}, + "mm_activity": {"value": 10}, + }, + "mem_usage": { + "total_vram": {"value": 100}, + "used_vram": {"value": 50}, + "free_vram": {"value": 50}, + }, + } + out = parse_gpu_metrics([no_energy, dict(no_energy)]) + self.assertIsNone(out["gpu.energy_j"]) + self.assertEqual(out["gpu.gfx_activity"], 10) + self.assertEqual(out["gpu.total_vram"], 200) + + def test_single_gpu_aggregated_field_types(self): + """Activity and vram fields from a single full entry must be int (or float for energy).""" + out = parse_gpu_metrics([_full_gpu_entry(gfx=10, umc=20, mm=30, total=1000, used=200, free=800, energy=5.0)]) + for k in ACTIVITY_KEYS: + with self.subTest(key=k): + self.assertIsInstance(out[k], (int, float)) + for k in VRAM_KEYS: + with self.subTest(key=k): + self.assertIsInstance(out[k], int) + self.assertIsInstance(out[ENERGY_KEY], float) + + def test_partial_vram_none_excluded_from_sum(self): + """GPU with no mem_usage block: its vram fields are None and excluded; + only the GPU that has vram contributes to the sum.""" + g0 = _full_gpu_entry(total=100, used=40, free=60, energy=2.0) + g1 = { + "usage": {"gfx_activity": {"value": 10}, "umc_activity": {"value": 10}, "mm_activity": {"value": 10}}, + "energy": {"total_energy_consumption": {"value": 3.0}}, + } + out = parse_gpu_metrics([g0, g1]) + self.assertEqual(out["gpu.total_vram"], 100) + self.assertEqual(out["gpu.used_vram"], 40) + self.assertEqual(out["gpu.free_vram"], 60) + self.assertEqual(out["gpu.energy_j"], 5.0) + + def test_partial_energy_none_excluded_from_sum(self): + """GPU with no energy block: its energy is None and excluded; + only the GPU that has energy contributes to the sum.""" + g0 = _full_gpu_entry(energy=500.0) + g1 = { + "usage": {"gfx_activity": {"value": 5}, "umc_activity": {"value": 5}, "mm_activity": {"value": 5}}, + "mem_usage": {"total_vram": {"value": 50}, "used_vram": {"value": 10}, "free_vram": {"value": 40}}, + } + out = parse_gpu_metrics([g0, g1]) + self.assertEqual(out["gpu.energy_j"], 500.0) + + def test_zero_vram_not_excluded_from_aggregation(self): + """total_vram=0 is valid; a falsy-zero aggregation bug (if val: acc += val) + would skip 0 and return None instead of 0. Single-GPU with all-zero VRAM.""" + out = parse_gpu_metrics([_full_gpu_entry(total=0, used=0, free=0)]) + self.assertEqual(out["gpu.total_vram"], 0) + self.assertEqual(out["gpu.used_vram"], 0) + self.assertEqual(out["gpu.free_vram"], 0) + + def test_zero_energy_not_excluded_from_aggregation(self): + """energy=0.0 is valid; a falsy-zero aggregation bug (if energy: skip) would + incorrectly exclude it. Two GPUs each with energy=0.0 must sum to 0.0.""" + g0 = _full_gpu_entry(energy=0.0) + g1 = _full_gpu_entry(energy=0.0) + out = parse_gpu_metrics([g0, g1]) + self.assertEqual(out["gpu.energy_j"], 0.0) + self.assertIsInstance(out["gpu.energy_j"], float) + + def test_zero_activity_not_excluded_from_average(self): + """gfx_activity=0 is valid (GPU idle). A falsy-zero bug in the aggregator + would exclude it from the average, giving wrong denominator+numerator.""" + g0 = _full_gpu_entry(gfx=0) + g1 = _full_gpu_entry(gfx=20) + out = parse_gpu_metrics([g0, g1]) + self.assertEqual(out["gpu.gfx_activity"], 10.0) # (0+20)/2, not 20/1=20 + + def test_partial_none_activity_averaging_three_gpus(self): + """With N=3 where one has no activity, mean of non-None values is: + (30 + 60) / 2 = 45.0 — not sum=90, not divide-by-3=30.""" + g_no_usage = { + "mem_usage": {"total_vram": {"value": 50}, "used_vram": {"value": 20}, "free_vram": {"value": 30}}, + "energy": {"total_energy_consumption": {"value": 1.0}}, + } + g0 = _full_gpu_entry(gfx=30) + g1 = _full_gpu_entry(gfx=60) + out = parse_gpu_metrics([g0, g_no_usage, g1]) + self.assertEqual(out["gpu.gfx_activity"], 45.0) + + def test_activity_averaging_three_full_gpus(self): + """N=3 averaging: (10+20+30)/3=20.0. Guards against hardcoded denominator=2.""" + g0 = _full_gpu_entry(gfx=10, umc=0, mm=5) + g1 = _full_gpu_entry(gfx=20, umc=60, mm=5) + g2 = _full_gpu_entry(gfx=30, umc=120, mm=5) + out = parse_gpu_metrics([g0, g1, g2]) + self.assertEqual(out["gpu.gfx_activity"], 20.0) # (10+20+30)/3 + self.assertEqual(out["gpu.umc_activity"], 60.0) # (0+60+120)/3 + self.assertEqual(out["gpu.mm_activity"], 5.0) # (5+5+5)/3 + + def test_vram_and_energy_summed_three_gpus(self): + """N=3 sum: guards against loop body that caps at 2 entries or re-inits acc.""" + g0 = _full_gpu_entry(total=100, used=10, free=90, energy=1.0) + g1 = _full_gpu_entry(total=200, used=20, free=180, energy=2.0) + g2 = _full_gpu_entry(total=300, used=30, free=270, energy=3.0) + out = parse_gpu_metrics([g0, g1, g2]) + self.assertEqual(out["gpu.total_vram"], 600) + self.assertEqual(out["gpu.used_vram"], 60) + self.assertEqual(out["gpu.free_vram"], 540) + self.assertEqual(out["gpu.energy_j"], 6.0) + + def test_never_raises_on_list_of_empty_entries(self): + """Contract: never raises. All-empty entries -> all keys present, None.""" + out = parse_gpu_metrics([{}, {}, {}]) + self.assertEqual(set(out.keys()), set(ALL_KEYS)) + for k in ALL_KEYS: + with self.subTest(key=k): + self.assertIsNone(out[k]) + + +# --------------------------------------------------------------------------- +# capture_gpu_metrics — I/O subsystem (orch-delegating, not a pure parser) +# Classification: integration boundary; tested only at the mock seam. +# Contract: calls orch to run amd-smi, passes the JSON list to parse_gpu_metrics, +# returns whatever parse_gpu_metrics returns. Never raises on malformed output. +# --------------------------------------------------------------------------- + + +class TestCaptureGpuMetrics(unittest.TestCase): + """capture_gpu_metrics delegates to parse_gpu_metrics and wraps the orch call. + + The function requires a live ContainerOrchestrator to invoke amd-smi, so + unit tests mock the orch dependency and verify delegation semantics only. + They never assert on hardware-specific values. + """ + + def _make_orch(self, raw_gpu_list): + """Return a mock orchestrator whose exec_on_head result decodes to raw_gpu_list. + + amd-smi is a host-side tool; capture_gpu_metrics uses exec_on_head so + the command runs on the bare-metal node, not inside the container. + The real ContainerOrchestrator.exec_on_head(cmd) returns {host: str}; + we mock the same shape so tests are grounded in the actual interface contract. + """ + import json + + orch = MagicMock() + orch.exec_on_head.return_value = {"node0": json.dumps(raw_gpu_list)} + return orch + + def test_happy_path_key_set_matches_all_keys(self): + """Given a valid amd-smi JSON list, capture_gpu_metrics returns all 7 keys, + delegates to parse_gpu_metrics, and passes the parsed values through.""" + orch = self._make_orch([_full_gpu_entry()]) + with patch("cvs.lib.utils.gpu.parse_gpu_metrics", wraps=parse_gpu_metrics) as mock_parse: + out = capture_gpu_metrics(orch) + self.assertIsInstance(out, dict) + self.assertEqual(set(out.keys()), set(ALL_KEYS)) + mock_parse.assert_called_once_with([_full_gpu_entry()]) + # Pin the exact command string sent to amd-smi (host-side, no sudo needed). + orch.exec_on_head.assert_called_once_with("amd-smi metric --json") + # Verify parse result is actually returned, not silently discarded. + self.assertEqual(out["gpu.gfx_activity"], 30) + self.assertIsNotNone(out["gpu.total_vram"]) + + def test_multi_host_entries_aggregated_together(self): + """All hosts' GPU entries must be pooled before aggregation. + + A mutant that reads only the first host's data would yield gfx=10 + (average of one entry), not 15.0 (average across both hosts' entries). + """ + import json + + orch = MagicMock() + orch.exec_on_head.return_value = { + "node0": json.dumps([_full_gpu_entry(gfx=10)]), + "node1": json.dumps([_full_gpu_entry(gfx=20)]), + } + out = capture_gpu_metrics(orch) + self.assertEqual(set(out.keys()), set(ALL_KEYS)) + self.assertAlmostEqual(out["gpu.gfx_activity"], 15.0) + + def test_no_raise_on_empty_gpu_list(self): + """Empty GPU list -> all 7 keys, all None. Must not raise.""" + orch = self._make_orch([]) + out = capture_gpu_metrics(orch) + self.assertEqual(set(out.keys()), set(ALL_KEYS)) + for k in ALL_KEYS: + with self.subTest(key=k): + self.assertIsNone(out[k]) + + def test_no_raise_on_malformed_orch_output(self): + """If orch returns non-JSON text, capture_gpu_metrics degrades; never raises.""" + orch = MagicMock() + orch.exec_on_head.return_value = {"node0": "not valid json at all"} + try: + out = capture_gpu_metrics(orch) + except Exception as exc: # noqa: BLE001 + self.fail(f"capture_gpu_metrics raised unexpectedly: {exc!r}") + else: + self.assertEqual(set(out.keys()), set(ALL_KEYS)) + for k in ALL_KEYS: + with self.subTest(key=k): + self.assertIsNone(out[k]) + + def test_gpu_data_envelope_unwrapped(self): + """ROCm 6.x amd-smi wraps the GPU list as {"gpu_data": [...]}; must be unwrapped.""" + import json + + orch = MagicMock() + orch.exec_on_head.return_value = {"node0": json.dumps({"gpu_data": [_full_gpu_entry(gfx=42)]})} + out = capture_gpu_metrics(orch) + self.assertEqual(set(out.keys()), set(ALL_KEYS)) + self.assertEqual(out["gpu.gfx_activity"], 42) + + def test_no_raise_on_valid_json_wrong_type(self): + """Valid JSON that decodes to a non-list (dict, null, scalar, string) + must degrade gracefully — never raises, returns all-None.""" + import json + + non_list_values = [{}, None, 42, "string"] + for val in non_list_values: + with self.subTest(decoded_type=type(val).__name__): + orch = MagicMock() + orch.exec_on_head.return_value = {"node0": json.dumps(val)} + try: + out = capture_gpu_metrics(orch) + except Exception as exc: # noqa: BLE001 + self.fail(f"capture_gpu_metrics raised on decoded {val!r}: {exc!r}") + else: + self.assertEqual(set(out.keys()), set(ALL_KEYS)) + for k in ALL_KEYS: + self.assertIsNone(out[k]) + + +# --------------------------------------------------------------------------- +# GPU_METRICS / GPU_METRIC_UNITS — module constants (invariants) +# --------------------------------------------------------------------------- + + +class TestGpuMetricsConstants(unittest.TestCase): + """Invariants tying GPU_METRICS, GPU_METRIC_UNITS, _RAW_GPU_FIELDS, and parser output keys.""" + + # Raw amd-smi parser output fields (internal; not surfaced as HTML rows). + EXPECTED_RAW_NAMES = { + "gfx_activity", + "umc_activity", + "mm_activity", + "total_vram", + "used_vram", + "free_vram", + "energy_j", + } + + # --- GPU_METRICS (derived, human-readable) --- + + def test_derived_unit_strings_match_spec(self): + """Unit strings pinned to spec values.""" + EXPECTED_UNITS = { + "peak_gpu_memory_mb": "MB", + "model_load_memory_mb": "MB", + "model_load_s": "s", + "gpu_bandwidth_util_pct": "%", + "gpu_compute_util_pct": "%", + } + self.assertEqual(GPU_METRIC_UNITS, EXPECTED_UNITS) + + # --- _RAW_GPU_FIELDS (amd-smi parser output) --- + + def test_raw_fields_covers_all_seven_amd_smi_fields(self): + raw_names = {short for short, _unit in _RAW_GPU_FIELDS} + self.assertEqual(raw_names, self.EXPECTED_RAW_NAMES) + + def test_raw_unit_strings_match_spec(self): + EXPECTED_RAW_UNITS = { + "gfx_activity": "%", + "umc_activity": "%", + "mm_activity": "%", + "total_vram": "MB", + "used_vram": "MB", + "free_vram": "MB", + "energy_j": "J", + } + self.assertEqual(_RAW_GPU_FIELD_UNITS, EXPECTED_RAW_UNITS) + + def test_parse_gpu_metrics_emits_key_for_every_raw_field(self): + """parse_gpu_metrics([full]) produces "gpu." for every k in _RAW_GPU_FIELDS.""" + self.assertGreater(len(_RAW_GPU_FIELDS), 0, "_RAW_GPU_FIELDS must not be empty") + out = parse_gpu_metrics([_full_gpu_entry()]) + for short, _unit in _RAW_GPU_FIELDS: + with self.subTest(metric=short): + self.assertIn(f"gpu.{short}", out) + + def test_derived_metrics_not_emitted_by_parser(self): + """GPU_METRICS (derived) are computed by the calling suite, not by the + parser. parse_gpu_metrics must NOT emit keys for derived short names.""" + out = parse_gpu_metrics([_full_gpu_entry()]) + for short, _unit in GPU_METRICS: + with self.subTest(metric=short): + self.assertNotIn(f"gpu.{short}", out) + + +class TestMean(unittest.TestCase): + def test_empty(self): + self.assertIsNone(_mean([])) + + def test_all_none(self): + self.assertIsNone(_mean([None, None])) + + def test_normal(self): + self.assertAlmostEqual(_mean([1.0, 3.0]), 2.0) + + def test_skips_none(self): + self.assertAlmostEqual(_mean([None, 4.0, None, 2.0]), 3.0) + + +class TestAggReadings(unittest.TestCase): + def test_empty(self): + result = agg_readings([]) + self.assertIsNone(result["peak_gpu_memory_mb"]) + self.assertIsNone(result["gpu_compute_util_pct"]) + self.assertIsNone(result["gpu_bandwidth_util_pct"]) + + def test_all_none_values(self): + readings = [{"gpu.used_vram": None, "gpu.gfx_activity": None, "gpu.umc_activity": None}] + result = agg_readings(readings) + self.assertIsNone(result["peak_gpu_memory_mb"]) + + def test_normal(self): + readings = [ + {"gpu.used_vram": 1000, "gpu.gfx_activity": 80.0, "gpu.umc_activity": 60.0}, + {"gpu.used_vram": 2000, "gpu.gfx_activity": 90.0, "gpu.umc_activity": 70.0}, + ] + result = agg_readings(readings) + self.assertEqual(result["peak_gpu_memory_mb"], 2000) + self.assertAlmostEqual(result["gpu_compute_util_pct"], 85.0) + self.assertAlmostEqual(result["gpu_bandwidth_util_pct"], 65.0) + + +class TestPollGpuMetrics(unittest.TestCase): + def _make_orch(self): + return unittest.mock.MagicMock() + + def test_happy_path_stops_when_done(self): + orch = self._make_orch() + snap = { + "gpu.used_vram": 1000, + "gpu.gfx_activity": 80.0, + "gpu.umc_activity": 60.0, + "gpu.mm_activity": 1.0, + "gpu.free_vram": 5000, + "gpu.total_vram": 6000, + "gpu.energy_j": 100.0, + } + call_count = [0] + + def is_done(): + call_count[0] += 1 + return call_count[0] >= 2 # done after 2nd poll + + with ( + unittest.mock.patch("cvs.lib.utils.gpu.capture_gpu_metrics", return_value=snap), + unittest.mock.patch("time.sleep"), + ): + readings = poll_gpu_metrics(orch, is_done_fn=is_done, poll_interval_s=0) + + self.assertEqual(len(readings), 2) + + def test_node_death_stops_after_max_consecutive_failures(self): + orch = self._make_orch() + + def is_done(): + return False + + with ( + unittest.mock.patch("cvs.lib.utils.gpu.capture_gpu_metrics", side_effect=RuntimeError("SSH timeout")), + unittest.mock.patch("time.sleep"), + ): + readings = poll_gpu_metrics( + orch, + is_done_fn=is_done, + poll_interval_s=0, + max_consecutive_failures=3, + ) + + self.assertEqual(readings, []) + + def test_writes_log_file(self): + import tempfile + import os + + orch = self._make_orch() + snap = { + "gpu.used_vram": 1000, + "gpu.gfx_activity": 80.0, + "gpu.umc_activity": 60.0, + "gpu.mm_activity": 1.0, + "gpu.free_vram": 5000, + "gpu.total_vram": 6000, + "gpu.energy_j": 100.0, + } + done_calls = [0] + + def is_done(): + done_calls[0] += 1 + return done_calls[0] >= 1 + + with tempfile.NamedTemporaryFile(delete=False, suffix=".log") as f: + log_path = f.name + try: + with ( + unittest.mock.patch("cvs.lib.utils.gpu.capture_gpu_metrics", return_value=snap), + unittest.mock.patch("time.sleep"), + ): + poll_gpu_metrics(orch, is_done_fn=is_done, poll_interval_s=0, log_path=log_path) + content = open(log_path).read() + self.assertIn("summary", content) + finally: + os.unlink(log_path) + + def test_failure_then_recovery_resets_counter(self): + orch = self._make_orch() + snap = { + "gpu.used_vram": 1000, + "gpu.gfx_activity": 80.0, + "gpu.umc_activity": 60.0, + "gpu.mm_activity": 1.0, + "gpu.free_vram": 5000, + "gpu.total_vram": 6000, + "gpu.energy_j": 100.0, + } + call_seq = [RuntimeError("fail"), RuntimeError("fail"), snap, snap] + call_iter = iter(call_seq) + done_calls = [0] + + def capture(*a, **kw): + v = next(call_iter) + if isinstance(v, Exception): + raise v + return v + + def is_done(): + done_calls[0] += 1 + return done_calls[0] >= 2 + + with ( + unittest.mock.patch("cvs.lib.utils.gpu.capture_gpu_metrics", side_effect=capture), + unittest.mock.patch("time.sleep"), + ): + readings = poll_gpu_metrics( + orch, + is_done_fn=is_done, + poll_interval_s=0, + max_consecutive_failures=3, + ) + + self.assertEqual(len(readings), 2) + + +class TestCaptureGpuMetricsMultiNode(unittest.TestCase): + """Tests for capture_gpu_metrics with the nodes= parameter (orch.exec(hosts=...)).""" + + def _make_gpu_json(self, used_vram: int, gfx: float = 80.0) -> str: + import json + + return json.dumps( + [ + { + "usage": { + "gfx_activity": {"value": gfx}, + "umc_activity": {"value": 10.0}, + "mm_activity": {"value": "N/A"}, + }, + "mem_usage": { + "used_vram": {"value": used_vram}, + "total_vram": {"value": used_vram + 1000}, + "free_vram": {"value": 1000}, + }, + "energy": {"total_energy_consumption": {"value": 50.0}}, + } + ] + ) + + def _make_exec_by_hosts(self, host_to_vram: dict, gfx: float = 80.0): + """Build an orch.exec side_effect keyed by the hosts= kwarg.""" + + def _exec(cmd, hosts=None): + return {h: self._make_gpu_json(host_to_vram[h], gfx) for h in hosts} + + return _exec + + def test_nodes_none_calls_exec_on_head(self): + """nodes=None must call orch.exec_on_head (regression guard).""" + orch = MagicMock() + orch.exec_on_head.return_value = {"host0": self._make_gpu_json(1000)} + from cvs.lib.utils.gpu import capture_gpu_metrics + + result = capture_gpu_metrics(orch, nodes=None) + orch.exec_on_head.assert_called_once_with("amd-smi metric --json") + self.assertEqual(result["gpu.used_vram"], 1000) + + def test_nodes_provided_calls_orch_exec_with_hosts_not_exec_on_head(self): + """nodes provided: orch.exec(cmd, hosts=...) is called, orch.exec_on_head is NOT.""" + orch = MagicMock() + orch.exec.side_effect = self._make_exec_by_hosts({"prefill-host": 2000, "decode-host": 3000}) + from cvs.lib.utils.gpu import capture_gpu_metrics + + capture_gpu_metrics( + orch, + nodes=[("prefill-0", ["prefill-host"]), ("decode-0", ["decode-host"])], + ) + orch.exec_on_head.assert_not_called() + orch.exec.assert_any_call("amd-smi metric --json", hosts=["prefill-host"]) + orch.exec.assert_any_call("amd-smi metric --json", hosts=["decode-host"]) + + def test_nodes_vram_summed_across_nodes(self): + """VRAM from all nodes is summed in the merged result.""" + orch = MagicMock() + orch.exec.side_effect = self._make_exec_by_hosts({"p": 2000, "d": 3000}) + from cvs.lib.utils.gpu import capture_gpu_metrics + + result = capture_gpu_metrics(orch, nodes=[("prefill-0", ["p"]), ("decode-0", ["d"])]) + self.assertEqual(result["gpu.used_vram"], 5000) + + def test_nodes_activity_averaged_across_nodes(self): + """GFX activity from all nodes is averaged.""" + orch = MagicMock() + + def _exec(cmd, hosts=None): + gfx = 60.0 if hosts == ["p"] else 100.0 + return {hosts[0]: self._make_gpu_json(1000, gfx)} + + orch.exec.side_effect = _exec + from cvs.lib.utils.gpu import capture_gpu_metrics + + result = capture_gpu_metrics(orch, nodes=[("prefill-0", ["p"]), ("decode-0", ["d"])]) + self.assertAlmostEqual(result["gpu.gfx_activity"], 80.0) + + def test_nodes_exception_propagates(self): + """Exception from orch.exec propagates (not swallowed).""" + orch = MagicMock() + orch.exec.side_effect = RuntimeError("ssh failed") + from cvs.lib.utils.gpu import capture_gpu_metrics + + with self.assertRaises(RuntimeError): + capture_gpu_metrics(orch, nodes=[("prefill-0", ["p"])]) + + def test_nodes_gpu_data_envelope_unwrapped_and_merged(self): + """Each node may independently use the {"gpu_data": [...]} envelope.""" + import json + + orch = MagicMock() + + def _exec(cmd, hosts=None): + vram = {"p": 1000, "d": 2000}[hosts[0]] + return { + hosts[0]: json.dumps( + { + "gpu_data": [ + { + "usage": { + "gfx_activity": {"value": 50.0}, + "umc_activity": {"value": 10.0}, + "mm_activity": {"value": "N/A"}, + }, + "mem_usage": { + "used_vram": {"value": vram}, + "total_vram": {"value": vram + 100}, + "free_vram": {"value": 100}, + }, + "energy": {"total_energy_consumption": {"value": 1.0}}, + } + ] + } + ) + } + + orch.exec.side_effect = _exec + from cvs.lib.utils.gpu import capture_gpu_metrics + + result = capture_gpu_metrics(orch, nodes=[("prefill-0", ["p"]), ("decode-0", ["d"])]) + self.assertEqual(result["gpu.used_vram"], 3000) + + +class TestPollGpuMetricsMultiNode(unittest.TestCase): + """Tests for poll_gpu_metrics with the nodes= parameter (orch.exec(hosts=...)).""" + + def _make_snap(self, used_vram: int = 1000): + return { + "gpu.used_vram": used_vram, + "gpu.gfx_activity": 90.0, + "gpu.umc_activity": 20.0, + "gpu.mm_activity": None, + "gpu.free_vram": 500, + "gpu.total_vram": 1500, + "gpu.energy_j": 50.0, + } + + def test_log_line_tagged_with_node_labels(self): + """When nodes provided, log lines include '[label1+label2] ' tag.""" + import tempfile + import os + + snap = self._make_snap() + per_node = {"prefill-0": 2000, "decode-0": 3000} + nodes = [("prefill-0", ["p"]), ("decode-0", ["d"])] + + with tempfile.NamedTemporaryFile(delete=False, suffix=".log") as f: + log_path = f.name + try: + with ( + patch( + "cvs.lib.utils.gpu._capture_multi_node", + return_value=(snap, per_node), + ), + patch("time.sleep"), + ): + from cvs.lib.utils.gpu import poll_gpu_metrics + + poll_gpu_metrics( + MagicMock(), + is_done_fn=lambda: True, + poll_interval_s=0, + log_path=log_path, + nodes=nodes, + ) + with open(log_path) as _f: + content = _f.read() + self.assertIn("[prefill-0+decode-0]", content) + finally: + os.unlink(log_path) + + def test_summary_contains_per_node_vram(self): + """Summary block includes node_vram_mb lines for each label.""" + import tempfile + import os + + snap = self._make_snap(1000) + per_node = {"prefill-0": 2000, "decode-0": 3000} + nodes = [("prefill-0", ["p"]), ("decode-0", ["d"])] + + with tempfile.NamedTemporaryFile(delete=False, suffix=".log") as f: + log_path = f.name + try: + with ( + patch( + "cvs.lib.utils.gpu._capture_multi_node", + return_value=(snap, per_node), + ), + patch("time.sleep"), + ): + from cvs.lib.utils.gpu import poll_gpu_metrics + + poll_gpu_metrics( + MagicMock(), + is_done_fn=lambda: True, + poll_interval_s=0, + log_path=log_path, + nodes=nodes, + ) + content = open(log_path).read() + self.assertIn("node_vram_mb [prefill-0]", content) + self.assertIn("node_vram_mb [decode-0]", content) + self.assertIn("per-node vram", content) + finally: + os.unlink(log_path) + + def test_no_node_tag_when_nodes_none(self): + """Without nodes, log lines have no '[...]' node tag.""" + import tempfile + import os + + snap = self._make_snap() + with tempfile.NamedTemporaryFile(delete=False, suffix=".log") as f: + log_path = f.name + try: + with ( + patch("cvs.lib.utils.gpu.capture_gpu_metrics", return_value=snap), + patch("time.sleep"), + ): + from cvs.lib.utils.gpu import poll_gpu_metrics + + poll_gpu_metrics( + MagicMock(), + is_done_fn=lambda: True, + poll_interval_s=0, + log_path=log_path, + nodes=None, + ) + content = open(log_path).read() + self.assertNotIn("per-node vram", content) + finally: + os.unlink(log_path) + + def test_inline_vram_failure_degrades_gracefully(self): + """If per-label orch.exec raises for one node, that label gets None; aggregate unaffected.""" + import tempfile + import os + + snap = self._make_snap(5000) + per_node = {"prefill-0": None, "decode-0": 3000} + nodes = [("prefill-0", ["p"]), ("decode-0", ["d"])] + + with tempfile.NamedTemporaryFile(delete=False, suffix=".log") as f: + log_path = f.name + try: + with ( + patch( + "cvs.lib.utils.gpu._capture_multi_node", + return_value=(snap, per_node), + ), + patch("time.sleep"), + ): + from cvs.lib.utils.gpu import poll_gpu_metrics + + readings = poll_gpu_metrics( + MagicMock(), + is_done_fn=lambda: True, + poll_interval_s=0, + log_path=log_path, + nodes=nodes, + ) + # Aggregate reading was not aborted + self.assertEqual(len(readings), 1) + self.assertEqual(readings[0]["gpu.used_vram"], 5000) + content = open(log_path).read() + # decode-0 has vram, prefill-0 is "-" (None) + self.assertIn("node_vram_mb [decode-0]: 3000 MB", content) + self.assertIn("node_vram_mb [prefill-0]: - MB", content) + finally: + os.unlink(log_path) + + def test_is_done_fn_exception_not_misattributed_as_poll_failure(self): + """is_done_fn raising must NOT be counted as an amd-smi/exec failure.""" + import tempfile + import os + + snap = self._make_snap() + calls = {"n": 0} + + def _is_done(): + calls["n"] += 1 + if calls["n"] == 1: + raise RuntimeError("client status check failed") + return True + + with tempfile.NamedTemporaryFile(delete=False, suffix=".log") as f: + log_path = f.name + try: + with ( + patch("cvs.lib.utils.gpu.capture_gpu_metrics", return_value=snap), + patch("time.sleep"), + ): + from cvs.lib.utils.gpu import poll_gpu_metrics + + with self.assertRaises(RuntimeError): + poll_gpu_metrics( + MagicMock(), + is_done_fn=_is_done, + poll_interval_s=0, + log_path=log_path, + nodes=None, + ) + content = open(log_path).read() + self.assertNotIn("FAILED", content) + finally: + os.unlink(log_path) + + +# =========================================================================== +# Hardening spec (plans/gpu-py-polling-reliability.md) — NEW behaviors. +# +# These tests are authored GREENFIELD against the three reliability fixes that +# are NOT yet implemented. They are expected to be RED until the fixes land, +# and must not disturb the 73 characterization tests above. +# +# Classification of the units they exercise: +# poll_gpu_metrics -> subsystem / stateful loop. State carried across +# rounds is `consecutive_failures`; the failure cap is +# a liveness guard (Fan-out Deadline, taxonomy #11). +# capture_gpu_metrics -> I/O subsystem at the orch.exec / orch.exec_on_head +# seam; timeout is threaded to that boundary. +# +# poll_gpu_metrics failure-accounting transition table (multi-node): +# | round outcome | consecutive_failures | round result | +# |----------------------------------|----------------------|--------------| +# | all nodes produced entries | reset to 0 | reading kept | +# | SOME nodes up, some down (mixed) | reset to 0 (success) | reading kept | <- Issue 1: must NOT count +# | ZERO nodes produced entries | += 1 | no reading | <- Issue 1: must count +# | consecutive_failures == cap | -> loop terminates | stop polling | +# =========================================================================== + + +def _gpu_json(used_vram=1000, gfx=80.0): + """Serialize one amd-smi GPU entry as the JSON string orch.exec returns.""" + import json + + return json.dumps([_full_gpu_entry(gfx=gfx, total=used_vram + 1000, used=used_vram, free=1000)]) + + +class TestPollGpuMetricsFailureAccounting(unittest.TestCase): + """Issue 1 — multi-node total failure must count toward the failure cap, + while partial (per-label) degradation must NOT. + + These are lifecycle tests over the `consecutive_failures` state carried + across poll rounds: a legal transition (partial failure stays a success and + the loop keeps running to is_done), the previously-missing illegal one + (every node down -> the round is a failure and the cap eventually fires), + and the liveness guarantee that the loop terminates instead of spinning + forever on a wholly-dead fan-out. + """ + + def _make_snap(self, used_vram=1000): + return { + "gpu.used_vram": used_vram, + "gpu.gfx_activity": 90.0, + "gpu.umc_activity": 20.0, + "gpu.mm_activity": None, + "gpu.free_vram": 500, + "gpu.total_vram": 1500, + "gpu.energy_j": 50.0, + } + + def test_all_nodes_fail_round_trips_failure_cap(self): + """Illegal transition (was silently a success): every node fails every + round. Driven through the REAL _capture_multi_node via an orch.exec that + raises for all hosts, so this holds regardless of where the fix places + the raise. With is_done_fn never truly done, the loop MUST stop on the + cap and return zero readings — not accumulate all-None 'successes'. + + is_done_fn returns True only after 20 calls purely as a safety valve so + a broken (pre-fix) implementation cannot hang the test; the real signal + is `readings == []`. + """ + orch = MagicMock() + orch.exec.side_effect = RuntimeError("ssh failed for every host") + nodes = [("prefill-0", ["p"]), ("decode-0", ["d"])] + done_calls = {"n": 0} + + def _is_done(): + done_calls["n"] += 1 + return done_calls["n"] >= 20 # safety valve, not the assertion + + with patch("time.sleep"): + readings = poll_gpu_metrics( + orch, + is_done_fn=_is_done, + poll_interval_s=0, + max_consecutive_failures=2, + nodes=nodes, + ) + + self.assertEqual(readings, []) + + def test_all_nodes_fail_via_capture_multi_node_seam(self): + """Same illegal transition, asserted at the seam the spec names: when + _capture_multi_node reports every label as None (per_node all-None), + poll_gpu_metrics must treat the round as a failure and stop on the cap. + """ + snap = self._make_snap() + all_none = {"prefill-0": None, "decode-0": None} + nodes = [("prefill-0", ["p"]), ("decode-0", ["d"])] + done_calls = {"n": 0} + + def _is_done(): + done_calls["n"] += 1 + return done_calls["n"] >= 20 # safety valve + + with ( + patch( + "cvs.lib.utils.gpu._capture_multi_node", + return_value=(snap, all_none), + ), + patch("time.sleep"), + ): + readings = poll_gpu_metrics( + MagicMock(), + is_done_fn=_is_done, + poll_interval_s=0, + max_consecutive_failures=2, + nodes=nodes, + ) + + self.assertEqual(readings, []) + + def test_partial_node_failure_does_not_trip_failure_cap(self): + """Legal transition preserved: one node up, one node down every round is + a SUCCESS (per-label degradation). Over 5 rounds with + max_consecutive_failures=2 the loop must NOT stop early — it runs until + is_done_fn, producing one reading per round. Regression guard that the + Issue 1 fix does not start counting partial failures. + """ + orch = MagicMock() + + def _exec(cmd, hosts=None, **kw): + if hosts == ["good"]: + return {"good": _gpu_json(used_vram=1000)} + raise RuntimeError("bad node down") + + orch.exec.side_effect = _exec + nodes = [("good-0", ["good"]), ("bad-0", ["bad"])] + done_calls = {"n": 0} + + def _is_done(): + done_calls["n"] += 1 + return done_calls["n"] >= 5 + + with patch("time.sleep"): + readings = poll_gpu_metrics( + orch, + is_done_fn=_is_done, + poll_interval_s=0, + max_consecutive_failures=2, + nodes=nodes, + ) + + self.assertEqual(len(readings), 5) + + def test_partial_failure_seam_keeps_reading(self): + """Same legal transition at the _capture_multi_node seam: a mixed + per_node (one None, one live) is a success — the aggregate reading is + kept and the loop is not aborted. + """ + snap = self._make_snap(5000) + mixed = {"prefill-0": None, "decode-0": 3000} + nodes = [("prefill-0", ["p"]), ("decode-0", ["d"])] + + with ( + patch( + "cvs.lib.utils.gpu._capture_multi_node", + return_value=(snap, mixed), + ), + patch("time.sleep"), + ): + readings = poll_gpu_metrics( + MagicMock(), + is_done_fn=lambda: True, + poll_interval_s=0, + max_consecutive_failures=2, + nodes=nodes, + ) + + self.assertEqual(len(readings), 1) + self.assertEqual(readings[0]["gpu.used_vram"], 5000) + + +class TestGpuMetricsTimeout(unittest.TestCase): + """Issue 3 — a caller-supplied timeout must be threaded down to the orch + transport (orch.exec / orch.exec_on_head), and a timeout firing must be + counted like any other failure. + + Assertions pin the *explicit* timeout the caller passes rather than the + default value: the spec leaves the default timeout deliberately unresolved + (open question / possibly a required parameter), so pinning a specific + default here would encode a decision the spec has not made. + """ + + def test_capture_single_node_passes_timeout_to_exec_on_head(self): + orch = MagicMock() + orch.exec_on_head.return_value = {"node0": _gpu_json()} + capture_gpu_metrics(orch, timeout_s=7) + _args, kwargs = orch.exec_on_head.call_args + self.assertEqual(kwargs.get("timeout"), 7) + + def test_capture_multi_node_passes_timeout_to_exec(self): + orch = MagicMock() + + def _exec(cmd, hosts=None, **kw): + return {hosts[0]: _gpu_json()} + + orch.exec.side_effect = _exec + capture_gpu_metrics( + orch, + nodes=[("prefill-0", ["p"]), ("decode-0", ["d"])], + timeout_s=5, + ) + self.assertTrue(orch.exec.called) + for call in orch.exec.call_args_list: + _args, kwargs = call + with self.subTest(call=call): + self.assertEqual(kwargs.get("timeout"), 5) + + def test_poll_threads_timeout_to_capture_single_node(self): + """poll_gpu_metrics forwards its timeout_s down to capture_gpu_metrics + in single-node mode (nodes=None).""" + seen = {} + + def _cap(o, nodes=None, timeout_s=None, **kw): + seen["timeout_s"] = timeout_s + return { + "gpu.used_vram": 1000, + "gpu.gfx_activity": 80.0, + "gpu.umc_activity": 60.0, + "gpu.mm_activity": 1.0, + "gpu.free_vram": 5000, + "gpu.total_vram": 6000, + "gpu.energy_j": 100.0, + } + + with ( + patch("cvs.lib.utils.gpu.capture_gpu_metrics", side_effect=_cap), + patch("time.sleep"), + ): + poll_gpu_metrics( + MagicMock(), + is_done_fn=lambda: True, + poll_interval_s=0, + timeout_s=8, + ) + self.assertEqual(seen.get("timeout_s"), 8) + + def test_poll_multinode_threads_timeout_to_orch_exec(self): + """In multi-node mode, poll_gpu_metrics' timeout_s must reach the orch + transport as timeout= on every per-label exec call (driven through the + real _capture_multi_node).""" + orch = MagicMock() + + def _exec(cmd, hosts=None, **kw): + return {hosts[0]: _gpu_json()} + + orch.exec.side_effect = _exec + nodes = [("prefill-0", ["p"]), ("decode-0", ["d"])] + with patch("time.sleep"): + poll_gpu_metrics( + orch, + is_done_fn=lambda: True, + poll_interval_s=0, + timeout_s=9, + nodes=nodes, + ) + self.assertTrue(orch.exec.called) + for call in orch.exec.call_args_list: + _args, kwargs = call + with self.subTest(call=call): + self.assertEqual(kwargs.get("timeout"), 9) + + def test_timeout_s_is_optional_for_capture(self): + """Backward-compat: existing callers pass no timeout_s. Adding the + parameter must keep it OPTIONAL (a default), never required — otherwise + every existing caller (and the 73 characterization tests) breaks.""" + orch = MagicMock() + orch.exec_on_head.return_value = {"node0": _gpu_json()} + try: + out = capture_gpu_metrics(orch) # no timeout_s + except TypeError as exc: # noqa: BLE001 + self.fail(f"timeout_s must be optional, not required: {exc!r}") + self.assertEqual(set(out.keys()), set(ALL_KEYS)) + + def test_single_node_timeout_exception_counted_as_failure(self): + """A timeout raised by the transport is counted like any other failure: + in single-node mode a persistently-timing-out capture must trip the cap + and stop the loop (returning no readings), exactly as a RuntimeError + does today.""" + + class _FakeTimeout(Exception): + pass + + with ( + patch( + "cvs.lib.utils.gpu.capture_gpu_metrics", + side_effect=_FakeTimeout("amd-smi timed out"), + ), + patch("time.sleep"), + ): + readings = poll_gpu_metrics( + MagicMock(), + is_done_fn=lambda: False, + poll_interval_s=0, + max_consecutive_failures=3, + ) + self.assertEqual(readings, []) + + +class TestEmptyNodesListConsistency(unittest.TestCase): + """nodes=[] (zero labeled nodes, distinct from nodes=None) must behave + identically in capture_gpu_metrics and poll_gpu_metrics: no exec call at + all, all-None result. Found live: capture_gpu_metrics used `nodes is None` + to pick single- vs multi-node mode while poll_gpu_metrics used a truthy + check (`if nodes:`), so nodes=[] took the multi-node (no-op) branch in + capture_gpu_metrics but silently fell back to the single-node + exec_on_head branch in poll_gpu_metrics. + """ + + def test_capture_gpu_metrics_empty_nodes_calls_neither_transport(self): + orch = MagicMock() + result = capture_gpu_metrics(orch, nodes=[]) + orch.exec.assert_not_called() + orch.exec_on_head.assert_not_called() + self.assertEqual(set(result.keys()), set(ALL_KEYS)) + self.assertTrue(all(v is None for v in result.values())) + + def test_poll_gpu_metrics_empty_nodes_calls_neither_transport(self): + orch = MagicMock() + with patch("time.sleep"): + readings = poll_gpu_metrics(orch, is_done_fn=lambda: True, poll_interval_s=0, nodes=[]) + orch.exec.assert_not_called() + orch.exec_on_head.assert_not_called() + self.assertEqual(len(readings), 1) + self.assertTrue(all(v is None for v in readings[0].values())) + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/utils/unittests/test_ib_discovery.py b/cvs/lib/utils/unittests/test_ib_discovery.py new file mode 100644 index 000000000..b1ac1fe6e --- /dev/null +++ b/cvs/lib/utils/unittests/test_ib_discovery.py @@ -0,0 +1,167 @@ +''' +Copyright 2025 Advanced Micro Devices Inc. +All rights reserved. +''' + +import unittest + +from cvs.lib.utils.ib_discovery import ( + _parse_ibv_devinfo_list, + discover_ib_hca_names, + discover_socket_netdev_name, + resolve_multinode_fabric, +) + + +class _NetdevOrch: + def __init__(self, hosts, responses): + self.hosts = list(hosts) + self._responses = dict(responses) + + def exec(self, cmd, hosts=None, **kwargs): + target_hosts = list(hosts) if hosts is not None else self.hosts + return {h: self._responses.get((h, cmd), "") for h in target_hosts} + + def exec_on_host(self, cmd, hosts=None, **kwargs): + return self.exec(cmd, hosts=hosts, **kwargs) + + +class _ContainerOrch(_NetdevOrch): + """Simulates container exec (broken/minimal) vs host exec (full OS tools).""" + + def exec(self, cmd, hosts=None, **kwargs): + target_hosts = list(hosts) if hosts is not None else self.hosts + return {h: "bash: line 1: ip: command not found\n" for h in target_hosts} + + def exec_on_host(self, cmd, hosts=None, **kwargs): + return _NetdevOrch.exec(self, cmd, hosts=hosts, **kwargs) + + +class TestParseIbvDevinfoList(unittest.TestCase): + def test_parses_newline_and_space_separated_names(self): + self.assertEqual(_parse_ibv_devinfo_list("mlx5_0\nmlx5_1\n"), ["mlx5_0", "mlx5_1"]) + self.assertEqual(_parse_ibv_devinfo_list("mlx5_0 mlx5_1"), ["mlx5_0", "mlx5_1"]) + + def test_ignores_ibv_banner_lines(self): + raw = """8 HCAs found: + rdma3 + rdma0 + rdma2 + rdma1 +""" + self.assertEqual(_parse_ibv_devinfo_list(raw), ["rdma3", "rdma0", "rdma2", "rdma1"]) + + +class TestDiscoverSocketNetdev(unittest.TestCase): + def test_resolves_common_netdev_from_cluster_ips(self): + h0, h1 = "10.32.80.112", "10.32.80.113" + orch = _NetdevOrch( + [h0, h1], + { + (h0, _cmd_for_ip(h0)): "ens51f1np1\n", + (h1, _cmd_for_ip(h1)): "ens51f1np1\n", + }, + ) + self.assertEqual(discover_socket_netdev_name(orch, master_addr=h0), "ens51f1np1") + + def test_raises_on_asymmetric_netdev_names(self): + h0, h1 = "10.32.80.112", "10.32.80.113" + orch = _NetdevOrch( + [h0, h1], + { + (h0, _cmd_for_ip(h0)): "ens51f1np1\n", + (h1, _cmd_for_ip(h1)): "ens51f1np2\n", + }, + ) + with self.assertRaisesRegex(RuntimeError, "asymmetric netdev"): + discover_socket_netdev_name(orch, master_addr=h0) + + def test_rejects_mlx5_hca_name(self): + h0 = "10.32.80.112" + orch = _NetdevOrch([h0], {(h0, _cmd_for_ip(h0)): "mlx5_0\n"}) + with self.assertRaisesRegex(RuntimeError, "no IPv4 netdev"): + discover_socket_netdev_name(orch, master_addr=h0) + + def test_rejects_shell_error_output(self): + h0 = "10.32.80.112" + orch = _NetdevOrch( + [h0], + { + (h0, _cmd_for_ip(h0)): "bash: line 1: ip: command not found\n", + }, + ) + with self.assertRaisesRegex(RuntimeError, "no IPv4 netdev"): + discover_socket_netdev_name(orch, master_addr=h0) + + def test_container_orch_uses_host_exec_not_container_exec(self): + from cvs.lib.utils.ib_discovery import _IBVDEVINFO_CMD + + h0, h1 = "10.32.80.112", "10.32.80.113" + orch = _ContainerOrch( + [h0, h1], + { + (h0, _IBVDEVINFO_CMD): "mlx5_0\nmlx5_1\n", + (h1, _IBVDEVINFO_CMD): "mlx5_0\nmlx5_1\n", + (h0, _cmd_for_ip(h0)): "ens51f1np1\n", + (h1, _cmd_for_ip(h1)): "ens51f1np1\n", + }, + ) + hcas, netdev = resolve_multinode_fabric( + orch, + ib_hca_devices="auto", + ib_netdev="auto", + master_addr=h0, + ) + self.assertEqual(hcas, ["mlx5_0", "mlx5_1"]) + self.assertEqual(netdev, "ens51f1np1") + + +class TestDiscoverIbHcaNames(unittest.TestCase): + def test_parses_ibv_devinfo_list_output(self): + from cvs.lib.utils.ib_discovery import _IBVDEVINFO_CMD + + h0, h1 = "10.32.80.112", "10.32.80.113" + orch = _NetdevOrch( + [h0, h1], + { + (h0, _IBVDEVINFO_CMD): "mlx5_0\nmlx5_1\n", + (h1, _IBVDEVINFO_CMD): "mlx5_0\nmlx5_1\n", + }, + ) + discovered = discover_ib_hca_names(orch) + self.assertEqual(discovered[h0], ["mlx5_0", "mlx5_1"]) + + +class TestResolveMultinodeFabric(unittest.TestCase): + def test_resolves_hcas_and_netdev_from_auto_config(self): + from cvs.lib.utils.ib_discovery import _IBVDEVINFO_CMD + + h0, h1 = "10.32.80.112", "10.32.80.113" + ibv_out = "8 HCAs found:\n rdma0\n rdma1\n" + orch = _NetdevOrch( + [h0, h1], + { + (h0, _IBVDEVINFO_CMD): ibv_out, + (h1, _IBVDEVINFO_CMD): ibv_out, + (h0, _cmd_for_ip(h0)): "ens51f1np1\n", + (h1, _cmd_for_ip(h1)): "ens51f1np1\n", + }, + ) + hcas, netdev = resolve_multinode_fabric( + orch, + ib_hca_devices="auto", + ib_netdev="auto", + master_addr=h0, + ) + self.assertEqual(hcas, ["rdma0", "rdma1"]) + self.assertEqual(netdev, "ens51f1np1") + + +def _cmd_for_ip(ip: str) -> str: + from cvs.lib.utils.ib_discovery import _netdev_for_ip_cmd + + return _netdev_for_ip_cmd(ip) + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/utils/verdict.py b/cvs/lib/utils/verdict.py new file mode 100644 index 000000000..3f421e171 --- /dev/null +++ b/cvs/lib/utils/verdict.py @@ -0,0 +1,84 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. +''' + +from __future__ import annotations + + +class ThresholdViolation(Exception): + def __init__(self, violations): + self.violations = list(violations) + super().__init__("\n".join(self.violations)) + + +def _to_float(x): + return float(x) + + +def _check_one(metric, actual_raw, spec): + kind = spec["kind"] + actual = _to_float(actual_raw) + if kind == "min": + target = _to_float(spec["value"]) + if actual < target: + return f"{metric}: actual {actual} < min {target}" + elif kind == "max_ms": + target = _to_float(spec["value"]) + if actual > target: + return f"{metric}: actual {actual} ms > max {target} ms" + elif kind == "max": + # Unit-agnostic upper bound, for counts like `failed` where `max_ms` + # would be a unit lie. Same comparison, honest message. + target = _to_float(spec["value"]) + if actual > target: + return f"{metric}: actual {actual} > max {target}" + elif kind == "within": + target = _to_float(spec["value"]) + pct = _to_float(spec["tolerance_pct"]) + lo, hi = target * (1 - pct / 100.0), target * (1 + pct / 100.0) + if not (lo <= actual <= hi): + return f"{metric}: actual {actual} outside {target} ±{pct}%" + elif kind == "min_tok_s": + target = _to_float(spec["value"]) + if actual < target: + return f"{metric}: actual {actual} tok/s < min {target} tok/s" + elif kind == "min_ratio": + ref_metric = spec["reference"] + ratio = _to_float(spec["value"]) + actuals = spec.get("_actuals", {}) + if ref_metric not in actuals: + return f"{metric}: reference metric '{ref_metric}' missing from actuals" + if actuals[ref_metric] is None: + return f"{metric}: reference '{ref_metric}' is None (metric unavailable for this run)" + ref_actual = _to_float(actuals[ref_metric]) + if ref_actual == 0: + return f"{metric}: reference '{ref_metric}' is 0; cannot compute ratio" + observed = actual / ref_actual + if observed < ratio: + return f"{metric}: observed ratio {observed:.3f} < min {ratio} (vs {ref_metric})" + else: + return f"{metric}: unknown threshold kind '{kind}'" + return None + + +def evaluate_all(actuals, thresholds): + violations = [] + for metric, spec in thresholds.items(): + if metric not in actuals: + violations.append(f"{metric}: missing from actuals") + continue + if actuals[metric] is None: + # A derived/goodput metric that could not be computed (e.g. + # decode_latency_ratio with no p50, or goodput with no SLO run). + # Loud violation, not a float(None) TypeError. + violations.append(f"{metric}: value is None (metric unavailable for this run)") + continue + spec_with_actuals = dict(spec) + if spec.get("kind") == "min_ratio": + spec_with_actuals["_actuals"] = actuals + v = _check_one(metric, actuals[metric], spec_with_actuals) + if v: + violations.append(v) + if violations: + raise ThresholdViolation(violations) diff --git a/cvs/lib/utils_lib.py b/cvs/lib/utils_lib.py index c6414b795..8208a07f2 100644 --- a/cvs/lib/utils_lib.py +++ b/cvs/lib/utils_lib.py @@ -364,14 +364,12 @@ def resolve_cluster_config_placeholders(cluster_dict): # Get username from environment (fallback chain: USER -> LOGNAME -> USERNAME -> 'root') username = os.getenv('USER') or os.getenv('LOGNAME') or os.getenv('USERNAME') or 'root' - log.info(f'Resolving cluster path placeholders with system username: {username}') - # Define replacement mapping - only resolve {user-id} in cluster config replacements = { '{user-id}': username, } - resolved_cluster = _resolve_placeholders_in_dict(cluster_dict, replacements, context_name="cluster config") + resolved_cluster = _resolve_placeholders_in_dict(cluster_dict, replacements) return resolved_cluster diff --git a/cvs/parsers/schemas.py b/cvs/parsers/schemas.py index 495f92f8f..b3f30300a 100644 --- a/cvs/parsers/schemas.py +++ b/cvs/parsers/schemas.py @@ -896,6 +896,112 @@ class PreflightConnectivityCheckConfig(BaseModel): rdma: PreflightRdmaConfig = Field(default_factory=PreflightRdmaConfig, description="RDMA connectivity settings") +class PreflightNodeSmokeConfig(BaseModel): + """Primus node_smoke settings (primus-cli direct -- node_smoke).""" + + model_config = ConfigDict(extra="allow") + + connectivity_mode: str = Field( + default="skip", + description="Primus node_smoke mode: 'run' (host/GPU/RDMA roll-call) or 'skip' (default)", + ) + auto_setup: bool = Field( + default=True, + description="Clone/update Primus and prepare venv on each node before node_smoke", + ) + setup_timeout: int = Field(default=600, ge=60, description="SSH timeout in seconds for Primus auto_setup") + force_reclone: bool = Field( + default=False, + description="Remove primus_dir and clone fresh on every run (destructive)", + ) + shared_install: bool = Field( + default=True, + description=( + "When true (default), clone and venv setup run only on the first reachable node; " + "other nodes wait for the shared NFS home install. Set false only if each node has " + "a local primus_dir/venv_activate path." + ), + ) + pip_install_mode: str = Field( + default="minimal", + description="Venv deps: minimal (torch only), requirements, or skip", + ) + torch_pip_index_url: str = Field( + default="https://download.pytorch.org/whl/rocm6.2", + description="PyTorch ROCm wheel index URL for minimal pip_install_mode", + ) + primus_git_url: str = Field( + default="https://github.com/AMD-AIG-AIMA/Primus.git", + description="Primus repository URL for auto_setup clone", + ) + primus_git_branch: str = Field( + default="dev/preflight-direct-test", + description="Git branch to checkout during auto_setup", + ) + primus_git_recurse_submodules: bool = Field( + default=False, + description="Clone git submodules during auto_setup (not required for node_smoke)", + ) + primus_dir: str = Field( + default="/home/{user-id}/INSTALL/Primus", + description="Path to cloned Primus repo under the user's home directory (required when connectivity_mode is 'run')", + ) + venv_activate: str = Field( + default="/home/{user-id}/envs/preflight/.venv/bin/activate", + description="Path to Python venv activate script on each node (required when connectivity_mode is 'run')", + ) + gpus_per_node: int = Field(default=8, ge=1, description="GPUs per node for node_smoke") + master_port: int = Field(default=1234, ge=1024, le=65535, description="Distributed master port for node_smoke") + dump_path: str = Field( + default="", + description="Per-node dump directory for smoke JSON (default: /node_smoke)", + ) + expected_rdma_nics: Optional[int] = Field( + default=None, + ge=1, + description="Hard-fail when training RDMA NIC count differs (default: len(node_check.rdma_interfaces))", + ) + ulimit_l_min_gb: float = Field(default=32.0, ge=0, description="Minimum RLIMIT_MEMLOCK in GiB (0 disables)") + shm_min_gb: float = Field(default=8.0, ge=0, description="Minimum /dev/shm size in GiB (0 disables)") + skip_dmesg: bool = Field(default=False, description="Skip dmesg error scan (e.g. unprivileged containers)") + allow_foreign_procs: bool = Field( + default=False, + description="Do not FAIL nodes with foreign GPU processes (still reported)", + ) + allowed_procs: str = Field( + default="gpuagent,rocm-smi-daemon,amd-smi,dcgm-exporter", + description="Comma-separated process names allowed to hold GPUs", + ) + require_tools: str = Field( + default="", + description="Comma-separated CLI tools that must exist in PATH (empty = warn only)", + ) + nccl_socket_ifname: str = Field(default="", description="NCCL_SOCKET_IFNAME override for node_smoke") + gloo_socket_ifname: str = Field(default="", description="GLOO_SOCKET_IFNAME override (defaults to nccl_socket_ifname)") + nccl_ib_hca: str = Field(default="", description="NCCL_IB_HCA override (defaults to node_check.rdma_interfaces)") + nccl_ib_gid_index: Optional[int] = Field( + default=None, + description="NCCL_IB_GID_INDEX override (defaults to node_check.gid_index)", + ) + rdma_nic_allowlist: str = Field( + default="", + description="Training NIC allowlist for node_smoke (defaults to node_check.rdma_interfaces)", + ) + ssh_timeout: int = Field(default=300, ge=30, description="SSH timeout in seconds for each node_smoke run") + extra_args: List[str] = Field( + default_factory=list, + description="Additional node_smoke CLI flags forwarded to primus-cli", + ) + + @field_validator("connectivity_mode") + @classmethod + def validate_node_smoke_mode(cls, v: str) -> str: + valid_modes = ["run", "skip"] + if v not in valid_modes: + raise ValueError(f"node_smoke.connectivity_mode must be one of: {', '.join(valid_modes)}") + return v + + class PreflightReportingConfig(BaseModel): """Report generation and output settings.""" @@ -936,6 +1042,9 @@ class PreflightConfigFile(BaseModel): connectivity_check: PreflightConnectivityCheckConfig = Field( default_factory=PreflightConnectivityCheckConfig, description="Inter-node connectivity tests" ) + node_smoke: PreflightNodeSmokeConfig = Field( + default_factory=PreflightNodeSmokeConfig, description="Primus node_smoke checks" + ) reporting: PreflightReportingConfig = Field( default_factory=PreflightReportingConfig, description="Report generation and output settings" ) diff --git a/cvs/tests/inference/inferencemax/inferencemax_gpt_oss_120b_single.py b/cvs/tests/inference/inferencemax/inferencemax_gpt_oss_120b_single.py deleted file mode 100644 index a922b1a48..000000000 --- a/cvs/tests/inference/inferencemax/inferencemax_gpt_oss_120b_single.py +++ /dev/null @@ -1,294 +0,0 @@ -''' -Copyright 2025 Advanced Micro Devices, Inc. -All rights reserved. This notice is intended as a precaution against inadvertent publication and does not imply publication or any waiver of confidentiality. -The year included in the foregoing notice is the year of creation of the work. -All code contained here is Property of Advanced Micro Devices, Inc. -''' - -import pytest - -import re -import time -import json - -from cvs.lib.parallel_ssh_lib import * -from cvs.lib.utils_lib import * -from cvs.lib import docker_lib -from cvs.lib.inference.inference_max import InferenceMaxJob -from cvs.lib import globals - -log = globals.log - - -# Importing additional cmd line args to script .. -@pytest.fixture(scope="module") -def cluster_file(pytestconfig): - """ - Retrieve the --cluster_file CLI option provided to pytest. - - Args: - pytestconfig: Built-in pytest fixture exposing command-line options. - - Returns: - str: Path to the cluster JSON file specified via --cluster_file. - - Notes: - - Ensure your pytest.ini or CLI includes: --cluster_file=/path/to/cluster.json - - Use module scope so the value is resolved once per test module. - """ - return pytestconfig.getoption("cluster_file") - - -@pytest.fixture(scope="module") -def training_config_file(pytestconfig): - """ - Retrieve the --config_file CLI option provided to pytest. - - Args: - pytestconfig: Built-in pytest fixture exposing command-line options. - - Returns: - str: Path to the training config JSON file specified via --config_file. - - Notes: - - Ensure your pytest.ini or CLI includes: --config_file=/path/to/training_config.json - - Module scope avoids re-fetching the option across tests in this module. - """ - return pytestconfig.getoption("config_file") - - -# Importing the cluster and cofig files to script to access node, switch, test config params -@pytest.fixture(scope="module") -def cluster_dict(cluster_file): - """ - Load the entire cluster configuration from the provided JSON file. - - Args: - cluster_file (str): Path to the cluster JSON file. - - Returns: - dict: Parsed JSON representing the cluster (nodes, credentials, etc.). - - Notes: - - Logs the loaded structure for visibility; consider using log.debug if verbose. - """ - with open(cluster_file) as json_file: - cluster_dict = json.load(json_file) - - # Resolve path placeholders like {user-id} in cluster config - cluster_dict = resolve_cluster_config_placeholders(cluster_dict) - log.info("%s", cluster_dict) - return cluster_dict - - -@pytest.fixture(scope="module") -def inference_dict(training_config_file, cluster_dict): - with open(training_config_file) as json_file: - inference_dict_t = json.load(json_file) - inference_dict = inference_dict_t['config'] - - # Resolve path placeholders like {user-id}, {home-mount-dir}, etc. - inference_dict = resolve_test_config_placeholders(inference_dict, cluster_dict) - return inference_dict - - -@pytest.fixture(scope="module") -def benchmark_params_dict(training_config_file, cluster_dict): - with open(training_config_file) as json_file: - inference_dict_t = json.load(json_file) - benchmark_params_dict = inference_dict_t['benchmark_params'] - - # Resolve path placeholders like {user-id}, {home-mount-dir}, etc. - benchmark_params_dict = resolve_test_config_placeholders(benchmark_params_dict, cluster_dict) - - log.info("%s", benchmark_params_dict) - return benchmark_params_dict - - -@pytest.fixture(scope="module") -def hf_token(inference_dict): - """ - Load the Hugging Face access token from the file path specified in the training config. - - Args: - inference_dict (dict): Training configuration dict that includes: - - 'hf_token_file': Path to the file containing the HF token. - - Returns: - str: The HF token string read from the file. - - Behavior: - - Reads the token from inference_dict['hf_token_file'] (already resolved for placeholders). - - Strips the trailing newline from the token. - """ - hf_token_file = inference_dict['hf_token_file'] - try: - with open(hf_token_file, 'r') as fp: - hf_token = fp.read().rstrip("\n") - except FileNotFoundError: - log.error(f"Error: The file '{hf_token_file}' was not found.") - except Exception as e: - log.error(f"An error occurred: {e}") - return hf_token - - -@pytest.fixture(scope="module") -def s_phdl(cluster_dict): - """ - Create and return a parallel SSH handle for all cluster nodes. - - Args: - cluster_dict (dict): Cluster configuration loaded by another fixture. Expected keys: - - 'node_dict': dict of node_name -> node_details (used to derive the node list) - - 'username': SSH username for connecting to nodes - - 'priv_key_file': path to the SSH private key file - - Returns: - Pssh: An initialized Pssh handle for issuing commands across all nodes. - - Behavior: - - Prints the full cluster_dict for quick debugging (consider switching to log.debug to reduce noise). - - Collects all node names from cluster_dict['node_dict'] and constructs a Pssh handle. - - Notes: - - This fixture has module scope, so a single connection handle is reused for all tests in the module. - """ - log.info("%s", cluster_dict) - env_vars = cluster_dict.get("env_vars") - node_list = list(cluster_dict['node_dict'].keys()) - s_phdl = Pssh(log, node_list, user=cluster_dict['username'], pkey=cluster_dict['priv_key_file'], env_vars=env_vars) - return s_phdl - - -@pytest.fixture(scope="module") -def c_phdl(cluster_dict): - """ - Create and return a parallel SSH handle for all cluster nodes. - - Args: - cluster_dict (dict): Cluster configuration loaded by another fixture. Expected keys: - - 'node_dict': dict of node_name -> node_details (used to derive the node list) - - 'username': SSH username for connecting to nodes - - 'priv_key_file': path to the SSH private key file - - Returns: - Pssh: An initialized Pssh handle for issuing commands across all nodes. - - Behavior: - - Prints the full cluster_dict for quick debugging (consider switching to log.debug to reduce noise). - - Collects all node names from cluster_dict['node_dict'] and constructs a Pssh handle. - - Notes: - - This fixture has module scope, so a single connection handle is reused for all tests in the module. - """ - log.info("%s", cluster_dict) - env_vars = cluster_dict.get("env_vars") - node_list = list(cluster_dict['node_dict'].keys()) - c_phdl = Pssh(log, node_list, user=cluster_dict['username'], pkey=cluster_dict['priv_key_file'], env_vars=env_vars) - return c_phdl - - -@pytest.fixture(scope="module") -def gpu_type(s_phdl, cluster_dict): - """ - Provide the GPU type string for the test module. - - Args: - cluster_dict (dict): Cluster configuration that includes the GPU type. - - Returns: - str: The GPU type (e.g., 'mi300', 'mi300x') used to select model parameters and logic. - - Notes: - - Module scope ensures this is evaluated once per test module. - - Consider validating this value against an expected set of GPU types to catch typos early. - """ - - log.info("%s", s_phdl) - log.info("%s", list(dir(s_phdl))) - head_node = s_phdl.host_list[0] - smi_out_dict = s_phdl.exec('rocm-smi -a | head -30') - smi_out = smi_out_dict[head_node] - gpu_type = get_model_from_rocm_smi_output(smi_out) - return gpu_type - - -def test_cleanup_stale_containers(s_phdl, inference_dict): - """ - Pytest: Clean up potentially stale Docker containers and volumes before tests. - - Args: - s_phdl: Parallel SSH/process handle used by docker_lib to run commands on nodes. - inference_dict (dict): Training configuration dict that includes: - - 'container_name': Name of the container to be killed if running. - - Behavior: - - Kills the specific container identified by inference_dict['container_name']. - - Deletes all containers and volumes on the target nodes (broad cleanup). - - Notes: - - This performs a broad cleanup via delete_all_containers_and_volumes; ensure the - test environment is isolated so this doesn?t remove unrelated containers/volumes. - - Consider narrowing cleanup scope if other workloads may be present on the hosts. - """ - - container_name = inference_dict['container_name'] - docker_lib.kill_docker_container(s_phdl, container_name) - docker_lib.delete_all_containers_and_volumes(s_phdl) - - -def test_launch_inference_containers(s_phdl, inference_dict, benchmark_params_dict): - """ - Launch InferenceMAX inference containers on all nodes. - - Note: Container image can be model-specific or use global default. - """ - - log.info('Testcase launch InferenceMax containers') - globals.error_list = [] - container_name = inference_dict['container_name'] - - # Get model-specific container image or use global default - container_image = benchmark_params_dict.get('gpt-oss-120b', {}).get( - 'container_image', inference_dict['container_image'] - ) - - # Launch the containers .. - docker_lib.launch_docker_container( - s_phdl, - container_name, - container_image, - inference_dict['container_config']['device_list'], - inference_dict['container_config']['volume_dict'], - inference_dict['container_config']['env_dict'], - shm_size='48G', - timeout=60 * 20, - ) - # ADD verifications .. - time.sleep(30) - log.info('Verify if the containers have been launched properly') - out_dict = s_phdl.exec('docker ps') - for node in out_dict.keys(): - if not re.search(f'{container_name}', out_dict[node], re.I): - fail_test(f'Failed to launch container on node {node}') - update_test_result() - - -def test_gpt_oss_120_single_node(c_phdl, s_phdl, gpu_type, inference_dict, benchmark_params_dict, hf_token): - globals.error_list = [] - im_obj = InferenceMaxJob( - c_phdl=c_phdl, - s_phdl=s_phdl, - model_name='gpt-oss-120b', - inference_config_dict=inference_dict, - benchmark_params_dict=benchmark_params_dict, - hf_token=hf_token, - gpu_type=gpu_type, - distributed_inference=False, - ) - im_obj.build_server_inference_job_cmd() - im_obj.start_inference_server_job() - im_obj.start_inference_client_job() - im_obj.poll_for_inference_completion() - im_obj.verify_inference_results() - update_test_result() diff --git a/cvs/tests/inference/inferencex_atom/_shared.py b/cvs/tests/inference/inferencex_atom/_shared.py new file mode 100644 index 000000000..b598b3b4e --- /dev/null +++ b/cvs/tests/inference/inferencex_atom/_shared.py @@ -0,0 +1,13 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. +''' + +from cvs.lib.inference.utils.inference_suite_results_table import ( + INFERENCEX_ATOM_RESULTS_COLUMNS, + make_print_results_table, +) + +test_print_results_table = make_print_results_table(INFERENCEX_ATOM_RESULTS_COLUMNS) + +__all__ = ["test_print_results_table"] diff --git a/cvs/tests/inference/inferencex_atom/conftest.py b/cvs/tests/inference/inferencex_atom/conftest.py new file mode 100644 index 000000000..776f7b3b2 --- /dev/null +++ b/cvs/tests/inference/inferencex_atom/conftest.py @@ -0,0 +1,149 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. +''' + +import json +import os + +import pytest + +from cvs.core.orchestrators.factory import OrchestratorConfig, OrchestratorFactory +from cvs.lib import globals +from cvs.lib.inference.utils.inference_suite_lifecycle import ( + InferenceLifecycle, + html_metric_table_header, + html_metric_table_row, + sort_lifecycle_items, +) +from cvs.lib.inference.inferencex_atom.inferencex_atom_config_loader import ( + load_variant, + orchestrator_container_from_variant, +) +from cvs.lib.utils_lib import resolve_cluster_config_placeholders + +log = globals.log + + +def _log_variant_run_card(variant_config): + rc = variant_config.run_card + parts = [ + f"gpu_arch={variant_config.gpu_arch}", + f"driver={variant_config.params.driver}", + f"model={variant_config.model.id}", + ] + atom_args = variant_config.roles.server.atom_args + if atom_args: + parts.append(f"atom_args={len(atom_args)} tokens") + if rc.atom_image_pin: + parts.append(f"image_pin={rc.atom_image_pin}") + if rc.upstream_run_url: + parts.append(f"upstream_run={rc.upstream_run_url}") + if rc.notes: + parts.append(f"notes={rc.notes}") + if int(variant_config.params.nnodes) > 1: + parts.append(f"nnodes={variant_config.params.nnodes}") + parts.append(f"pp={variant_config.params.pipeline_parallel_size}") + log.info("InferenceX ATOM run card: %s", "; ".join(parts)) + + +@pytest.fixture(scope="module", autouse=True) +def _emit_variant_run_card(variant_config): + """Log the variant run card once per module (not once per sweep cell).""" + _log_variant_run_card(variant_config) + + +LIFECYCLE_RANK = { + "test_launch_container": 0, + "test_setup_sshd": 1, + "test_discover_topology": 2, + "test_model_fetch": 3, + "test_inferencex_atom_inference": 4, + "test_cell_metrics": 5, + "test_print_results_table": 6, + "test_teardown": 7, +} + + +def _deep_merge(base, override): + """Recursively merge ``override`` onto ``base`` (dicts merged key-wise; scalars/lists replaced).""" + if not (isinstance(base, dict) and isinstance(override, dict)): + return override + out = dict(base) + for k, v in override.items(): + out[k] = _deep_merge(base[k], v) if k in base else v + return out + + +@pytest.fixture(scope="module") +def cluster_dict(pytestconfig): + cluster_file = pytestconfig.getoption("cluster_file") + if not cluster_file: + pytest.fail("--cluster_file is required") + with open(cluster_file) as fp: + d = json.load(fp) + return resolve_cluster_config_placeholders(d) + + +@pytest.fixture(scope="module") +def variant_config(pytestconfig, cluster_dict): + config_file = pytestconfig.getoption("config_file") + if not config_file: + pytest.fail("--config_file is required") + return load_variant(config_file, cluster_dict) + + +@pytest.fixture(scope="module") +def lifecycle(): + return InferenceLifecycle() + + +@pytest.fixture(scope="module") +def orch(cluster_dict, variant_config, lifecycle): + container_block = _deep_merge( + cluster_dict.get("container", {}), + # also injects roles.server.env — do not replace with .container.model_dump() + orchestrator_container_from_variant(variant_config), + ) + testsuite_config = { + "orchestrator": "container", + "container": container_block, + } + cfg = OrchestratorConfig.from_configs(cluster_dict, testsuite_config) + o = OrchestratorFactory.create_orchestrator(log, cfg) + yield o + if not lifecycle.torn_down: + log.info("orch fixture leak-guard: tearing down InferenceX ATOM containers") + o.teardown_containers() + + +@pytest.fixture(scope="module") +def hf_token(variant_config): + path = variant_config.paths.hf_token_file + if not os.path.isfile(path): + pytest.skip(f"hf_token file missing: {path}") + with open(path) as fp: + return fp.read().strip() + + +@pytest.fixture(scope="module") +def server_session(): + """Tracks the active server session key to allow reuse across sweep cells when reuse_server_across_sweep=true.""" + return {"key": None} + + +@pytest.fixture(scope="module") +def inf_res_dict(): + return {} + + +def pytest_collection_modifyitems(items): + sort_lifecycle_items(items, LIFECYCLE_RANK) + + +def pytest_html_results_table_header(cells): + html_metric_table_header(cells) + + +def pytest_html_results_table_row(report, cells): + html_metric_table_row(report, cells) diff --git a/cvs/tests/inference/inferencex_atom/inferencex_atom.py b/cvs/tests/inference/inferencex_atom/inferencex_atom.py new file mode 100644 index 000000000..a6848d286 --- /dev/null +++ b/cvs/tests/inference/inferencex_atom/inferencex_atom.py @@ -0,0 +1,205 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. +''' + +import json +import os +import time + +import pytest + +from cvs.lib import globals +from cvs.lib.inference.utils.inference_suite_lifecycle import ( + sweep_cell_result_key, + test_launch_container, # noqa: F401 + test_model_fetch, # noqa: F401 + test_setup_sshd, # noqa: F401 + test_teardown, # noqa: F401 +) +from cvs.lib.inference.inferencex_atom.inferencex_atom_orch import InferenceXAtomJob +from cvs.lib.inference.inferencex_atom.inferencex_atom_config_loader import ( + expand_sweep_parametrize, + reuse_server_flag, + server_session_key, +) +from cvs.lib.inference.inferencex_atom.inferencex_atom_parsing import ( + CLIENT_METRIC_UNITS as _METRIC_UNITS, + METRIC_TIERS, + RECORD_METRICS, + SCALING_METRIC_UNITS, + tier_metric_specs, +) +from cvs.lib.utils.verdict import evaluate_all +from cvs.tests.inference.inferencex_atom._shared import test_print_results_table # noqa: F401 + +log = globals.log + + +def _tier_display_metric(tier): + if tier == "record": + return RECORD_METRICS[0] if RECORD_METRICS else "output_throughput" + names = METRIC_TIERS.get(tier, ()) + return names[0] if names else tier + + +def test_discover_topology(orch, variant_config, lifecycle, request): + """Discover IB HCAs and socket netdev on all nodes before the benchmark sweep.""" + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + + nn = int(variant_config.params.nnodes) + if nn == 1: + lifecycle.ib_hcas = [] + lifecycle.ib_netdev = "" + return + + from cvs.lib.utils.ib_discovery import resolve_multinode_fabric + + t = time.monotonic() + master_addr = (variant_config.params.master_addr or "").strip() or orch.hosts[0] + try: + resolved_hcas, resolved_netdev = resolve_multinode_fabric( + orch, + ib_hca_devices=variant_config.roles.server.ib_hca_devices, + ib_netdev=variant_config.roles.server.ib_netdev, + master_addr=master_addr, + ) + except RuntimeError as e: + lifecycle.failed = True + lifecycle.record(request.node.nodeid, "topology_discovery", time.monotonic() - t) + pytest.fail(str(e)) + + lifecycle.ib_hcas = resolved_hcas + lifecycle.ib_netdev = resolved_netdev + lifecycle.record(request.node.nodeid, "topology_discovery", time.monotonic() - t) + log.info( + "test_discover_topology: resolved netdev=%s HCAs=%s", + resolved_netdev, + resolved_hcas, + ) + + +def pytest_generate_tests(metafunc): + config_file = metafunc.config.getoption("config_file") + if not config_file or not os.path.isfile(config_file): + raise pytest.UsageError(f"--config_file not found or not specified: {config_file!r}") + with open(config_file) as fp: + raw = json.load(fp) + spec = expand_sweep_parametrize(raw.get("sweep", {}), metafunc.fixturenames) + if spec: + argnames, argvalues, ids = spec + metafunc.parametrize(argnames, argvalues, ids=ids) + + +def test_inferencex_atom_inference( + orch, + variant_config, + hf_token, + seq_combo, + concurrency, + inf_res_dict, + server_session, + lifecycle, + request, +): + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + + isl = seq_combo["isl"] + osl = seq_combo["osl"] + p = variant_config.params + job = InferenceXAtomJob.from_variant( + orch=orch, + variant=variant_config, + hf_token=hf_token, + isl=isl, + osl=osl, + concurrency=concurrency, + ib_hcas=getattr(lifecycle, "ib_hcas", []), + ib_netdev=getattr(lifecycle, "ib_netdev", None), + ) + + session_key = server_session_key(variant_config, isl, osl) + reuse = reuse_server_flag(p) and server_session.get("key") == session_key + + try: + if not reuse: + job.stop_server() + job.build_server_cmd() + t = time.monotonic() + job.start_server() + job.wait_ready() + lifecycle.record(request.node.nodeid, "server_ready", time.monotonic() - t) + if reuse_server_flag(p): + server_session["key"] = session_key + else: + log.info("reusing ATOM server across sweep cell (key=%s)", session_key) + job.prepare_cell_out_dir() + t_client = time.monotonic() + job.run_client() + job.wait_client_complete() + results = job.parse_results() + except Exception: + lifecycle.failed = True + raise + + inf_res_dict[sweep_cell_result_key(variant_config, seq_combo, isl, osl, concurrency)] = results + lifecycle.record(request.node.nodeid, "client_complete", time.monotonic() - t_client) + + +def test_cell_metrics( + seq_combo, + concurrency, + metric_tier, + inf_res_dict, + variant_config, + lifecycle, + request, +): + """One pytest row per metric tier per sweep cell (W1 gate batches). + + Fails when ``enforce_thresholds`` is on and either (1) the cell has no + threshold specs for the requested tier, or (2) specs exist but every gated + metric for that tier is missing from the benchmark artifact (ATOM may omit + tail percentiles even when ``metric_percentiles`` requests them). + """ + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + isl = seq_combo["isl"] + osl = seq_combo["osl"] + key = sweep_cell_result_key(variant_config, seq_combo, isl, osl, concurrency) + if key not in inf_res_dict: + pytest.skip(f"no recorded results for cell {key!r} (inference did not run)") + host_dict = inf_res_dict[key] + _host, actuals = next(iter(host_dict.items())) + cell = variant_config.cell_key(isl, osl, concurrency) + thresholds_cell = variant_config.thresholds.get(cell) or {} + specs = tier_metric_specs(thresholds_cell, metric_tier) + + display = _tier_display_metric(metric_tier) + if metric_tier == "scaling": + full = f"scaling.{display}" + unit = SCALING_METRIC_UNITS.get(display, "%") + else: + full = f"client.{display}" + unit = _METRIC_UNITS.get(display, metric_tier) + value = actuals.get(full) + request.node.user_properties.append(("metric_value", value)) + request.node.user_properties.append(("metric_unit", unit)) + + if not variant_config.enforce_thresholds or metric_tier == "record": + return + if not specs: + if metric_tier == "scaling" and int(variant_config.params.nnodes) <= 1: + pytest.skip("scaling tier not configured for single-node runs") + pytest.fail(f"no threshold specs for tier {metric_tier!r} in cell {cell!r}") + # ATOM benchmark_serving may omit some tail percentiles even when + # metric_percentiles requests them; only gate metrics present in actuals. + specs = {k: v for k, v in specs.items() if k in actuals and actuals[k] is not None} + if not specs: + pytest.fail( + f"no assertable threshold specs for tier {metric_tier!r} in cell {cell!r} " + f"(metrics missing from benchmark artifact)" + ) + evaluate_all(actuals, specs) diff --git a/cvs/tests/inference/sglang/_shared.py b/cvs/tests/inference/sglang/_shared.py new file mode 100644 index 000000000..7a5938d4e --- /dev/null +++ b/cvs/tests/inference/sglang/_shared.py @@ -0,0 +1,193 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Shared helpers and ordering for the SGLang disaggregated inference suite. +''' + +from __future__ import annotations + +import os +import re +from tabulate import tabulate +from typing import Any, Mapping + +from cvs.lib import globals + + +log = globals.log + +__all__ = [ + "resolve_benchmark_variant_key", + "SGLANG_DISAGG_TEST_ORDER", + "test_print_results_table", +] + +_SMOKE_LINE_RE = re.compile(r"^(.+) -> (Pass|Fail) \((\d+)\)$") + + +def resolve_benchmark_variant_key(root: Mapping[str, Any], config_path: str) -> str: + """Pick which ``benchmark_params`` entry to run. + + Resolution order: + 1. Environment ``SGLANG_BENCHMARK_KEY`` (override for CI matrices). + 2. Top-level JSON ``active_benchmark`` (string key into ``benchmark_params``). + 3. If ``benchmark_params`` has exactly one key, use it. + + ``root`` is the full JSON object loaded from ``--config_file`` (not only ``config``). + """ + env_key = (os.environ.get("SGLANG_BENCHMARK_KEY") or "").strip() + bp = root.get("benchmark_params") or {} + if not isinstance(bp, dict) or not bp: + raise ValueError(f"benchmark_params missing or empty in {config_path!r}") + + if env_key: + if env_key not in bp: + raise ValueError( + f"SGLANG_BENCHMARK_KEY={env_key!r} not found in benchmark_params ({config_path}); valid: {sorted(bp)!r}" + ) + log.info("Using benchmark variant from env SGLANG_BENCHMARK_KEY=%r", env_key) + return env_key + + explicit = root.get("active_benchmark") + if explicit is not None: + if explicit not in bp: + raise ValueError( + f"active_benchmark={explicit!r} not found in benchmark_params ({config_path}); valid: {sorted(bp)!r}" + ) + log.info("Using benchmark variant from active_benchmark=%r", explicit) + return str(explicit) + + if len(bp) == 1: + only = next(iter(bp)) + log.info("Single benchmark_params entry; using %r", only) + return str(only) + + raise ValueError( + f"Multiple benchmark_params keys in {config_path!r}: {sorted(bp)!r}. " + "Set top-level \"active_benchmark\" to one of them, or export SGLANG_BENCHMARK_KEY." + ) + + +# Stable test order (definition order already matches; this guards against drift / imports). +SGLANG_DISAGG_TEST_ORDER = { + "test_cleanup_stale_containers": 0, + "test_launch_inference_containers": 1, + "test_setup_ibv_devices": 2, + "test_rms_norm": 3, + "test_launch_prefill_servers": 4, + "test_launch_decode_servers": 5, + "test_poll_for_server_ready": 6, + "test_launch_proxy_router": 7, + "test_openai_compatible_http_endpoints": 8, + "test_run_lm_eval_hellaswag_benchmark_test": 9, + "test_run_lm_eval_gsm8k_benchmark_test": 10, + "test_run_lm_eval_mmlu_benchmark_test": 11, + "test_run_performance_benchmark_test": 12, + "test_disagg_gpu_topology": 13, + "test_print_results_table": 14, +} + + +def test_print_results_table(inf_res_dict): + phase_labels = inf_res_dict.pop("__phase_labels__", None) or {} + + smoke_results = inf_res_dict.pop("__smoke_probe_results__", None) + if smoke_results: + smoke_rows = [] + for line in smoke_results: + m = _SMOKE_LINE_RE.match(str(line).strip()) + if m: + smoke_rows.append([m.group(1), m.group(2).upper(), m.group(3)]) + else: + smoke_rows.append([str(line), "-", "-"]) + if smoke_rows: + log.info( + "\n\n\n\n======== OpenAI-compatible smoke results ========\n%s", + tabulate( + smoke_rows, + headers=["Check", "Result", "HTTP status"], + tablefmt="github", + ), + ) + + acc_rows = [] + for label, key in (("HellaSwag", "accuracy_hellaswag"), ("GSM8K", "accuracy_gsm8k"), ("MMLU", "accuracy_mmlu")): + e = phase_labels.get(key) + if isinstance(e, dict) and "task" in e: + passed = e.get("passed") + acc_rows.append( + [ + label, + e.get("task", "-"), + e.get("metric_key", "-"), + f"{float(e['actual']):.4f}" if e.get("actual") is not None else "-", + f"{float(e['expected']):.4f}", + "PASS" if passed is True else "FAIL" if passed is False else "-", + ] + ) + if acc_rows: + log.info( + "\n\n\n\n======== LM-eval accuracy results ========\n%s", + tabulate( + acc_rows, + headers=["Suite", "Task", "Metric", "Actual", "Expected", "Result"], + tablefmt="github", + ), + ) + + perf_expected = phase_labels.get("performance_expected") or {} + + PERF_METRICS = [ + ("Mean TTFT (ms)", "mean_ttft_ms"), + ("Mean TPOT (ms)", "mean_tpot_ms"), + ("P99 ITL (ms)", "p99_itl_ms"), + ("Mean E2E latency (ms)", "mean_e2e_latency_ms"), + ("Req/s", "request_throughput_per_sec"), + ("Output tok/s", "output_throughput_per_sec"), + ("Output tok/s/GPU", "output_throughput_per_gpu_per_sec"), + ("Goodput", "goodput"), + ("MFU (estimated)", "mfu"), + ] + + def _perf_result(actual, expected, metric_key: str) -> str: + if actual is None or expected is None: + return "-" + a, e = float(actual), float(expected) + if "ms" in metric_key.lower(): + return "PASS" if a <= e else "FAIL" + return "PASS" if a >= e else "FAIL" + + perf_rows = [] + for key, host_dict in inf_res_dict.items(): + model, gpu, isl, osl, policy, conc = key + for host, m in host_dict.items(): + for label, metric_key in PERF_METRICS: + actual = m.get(metric_key) + if actual is None: + continue + + expected = perf_expected.get(metric_key) + perf_rows.append( + [ + model, + gpu, + host, + label, + f"{float(actual):.4f}", + f"{float(expected):.4f}" if expected is not None else "-", + _perf_result(actual, expected, metric_key), + ] + ) + + if perf_rows: + log.info( + "\n\n\n\n======== Performance results ========\n%s", + tabulate( + perf_rows, + headers=["Model", "GPU", "Host", "Metric", "Actual", "Expected", "Result"], + tablefmt="github", + ), + ) + elif not smoke_results and not acc_rows: + log.info("inf_res_dict empty, nothing to print") diff --git a/cvs/tests/inference/sglang/conftest.py b/cvs/tests/inference/sglang/conftest.py new file mode 100644 index 000000000..5f6eb324b --- /dev/null +++ b/cvs/tests/inference/sglang/conftest.py @@ -0,0 +1,292 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Fixtures and hooks for ``sglang_disagg_distributed`` (multi-node PSSH + Docker). + +``--config_file`` must be JSON with top-level ``"config"`` and ``"benchmark_params"``. +Use ``active_benchmark`` (or ``SGLANG_BENCHMARK_KEY``) when multiple models are defined. + +Each ``benchmark_params`` variant may set ``threshold_file`` to a +JSON file beside the config; that file supplies pass/fail thresholds for performance +and lm-eval benchmarks. +''' + +import json +from pathlib import Path +from typing import Any, Mapping + +import pytest + +from cvs.lib import globals +from cvs.lib.parallel_ssh_lib import Pssh +from cvs.lib.inference import sglang_disagg_lib +from cvs.lib.utils_lib import ( + get_model_from_rocm_smi_output, + resolve_cluster_config_placeholders, + resolve_test_config_placeholders, +) + +from cvs.tests.inference.sglang._shared import SGLANG_DISAGG_TEST_ORDER, resolve_benchmark_variant_key + +log = globals.log + + +def _threshold_file_path(bp_dict: Mapping[str, Any]) -> str | None: + """Return the threshold file path from a benchmark_params variant.""" + path = bp_dict.get("threshold_file") + if path: + return str(path).strip() + return None + + +def _resolve_threshold_path(threshold_path: str) -> Path: + """Resolve an absolute or repo-relative threshold path from config.""" + path = Path(threshold_path) + if path.is_absolute(): + return path + return path.resolve() + + +def _load_thresholds_file(path: Path) -> dict[str, Any]: + """Load threshold JSON and drop comment keys (e.g. ``_comment``).""" + try: + with open(path, encoding="utf-8") as fp: + raw = json.load(fp) + except FileNotFoundError: + pytest.fail(f"threshold file not found: {path}") + except OSError as e: + pytest.fail(f"cannot read threshold file {path}: {e}") + except json.JSONDecodeError as e: + pytest.fail(f"invalid JSON in threshold file {path}: {e}") + + if not isinstance(raw, dict): + pytest.fail(f"threshold file must be a JSON object: {path}") + + return {k: v for k, v in raw.items() if not str(k).startswith("_")} + + +def perf_cell_key(bp_dict: Mapping[str, Any]) -> str: + """Build the performance threshold cell key, e.g. ``ISL=1024,OSL=1024,TP=8,CONC=25``.""" + bench = (bp_dict.get("inference_tests") or {}).get("bench_serv_random") or {} + return ( + f"ISL={bench.get('input_length', '-')}," + f"OSL={bench.get('output_length', '-')}," + f"TP={bp_dict.get('tensor_parallelism', '-')}," + f"CONC={bp_dict.get('max_concurrency', '-')}" + ) + + +def bench_cell_key(bench_name: str) -> str: + """Build an lm-eval threshold cell key, e.g. ``BENCH=lm_eval_hellaswag``.""" + return f"BENCH={bench_name}" + + +def flat_expected_from_specs(specs: Mapping[str, Any]) -> dict[str, float]: + """Convert ``{metric: {kind, value}}`` threshold specs to flat floats for legacy checks.""" + out: dict[str, float] = {} + for metric, spec in specs.items(): + if isinstance(spec, dict) and "value" in spec: + out[metric] = float(spec["value"]) + else: + out[metric] = float(spec) + return out + + +def _inject_thresholds_into_bp_dict(bp_dict: dict[str, Any], thresholds: Mapping[str, Any]) -> None: + """Merge external thresholds into ``inference_tests.*.expected_results`` in-place.""" + inference_tests = bp_dict.setdefault("inference_tests", {}) + + perf_key = perf_cell_key(bp_dict) + perf_specs = thresholds.get(perf_key) + if perf_specs: + bench = inference_tests.setdefault("bench_serv_random", {}) + expected = bench.setdefault("expected_results", {}) + expected["auto"] = flat_expected_from_specs(perf_specs) + log.info("Loaded performance thresholds from cell %r", perf_key) + else: + log.warning("No performance thresholds for cell %r in threshold file", perf_key) + + for bench_name in ("lm_eval_hellaswag", "lm_eval_gsm8k", "lm_eval_mmlu"): + cell = bench_cell_key(bench_name) + acc_specs = thresholds.get(cell) + if not acc_specs: + continue + bench = inference_tests.setdefault(bench_name, {}) + expected = bench.setdefault("expected_results", {}) + task_key = bench_name.removeprefix("lm_eval_") + expected[task_key] = flat_expected_from_specs(acc_specs) + log.info("Loaded accuracy thresholds from cell %r", cell) + + +@pytest.fixture(scope="module") +def cluster_file(pytestconfig): + path = pytestconfig.getoption("cluster_file") + if not path: + pytest.fail("--cluster_file is required") + return path + + +@pytest.fixture(scope="module") +def inference_config_file(pytestconfig): + path = pytestconfig.getoption("config_file") + if not path: + pytest.fail("--config_file is required") + return path + + +@pytest.fixture(scope="module") +def cluster_dict(cluster_file): + with open(cluster_file, encoding="utf-8") as fp: + d = json.load(fp) + return resolve_cluster_config_placeholders(d) + + +@pytest.fixture(scope="module") +def inference_config_root(inference_config_file): + with open(inference_config_file, encoding="utf-8") as fp: + return json.load(fp) + + +@pytest.fixture(scope="module") +def inference_dict(inference_config_root, cluster_dict): + if isinstance(inference_config_root, dict) and "config" in inference_config_root: + cfg = inference_config_root["config"] + else: + cfg = inference_config_root + return resolve_test_config_placeholders(cfg, cluster_dict) + + +@pytest.fixture(scope="module") +def benchmark_params_dict(inference_config_root, cluster_dict): + bp = inference_config_root["benchmark_params"] + return resolve_test_config_placeholders(bp, cluster_dict) + + +@pytest.fixture(scope="module") +def benchmark_variant(inference_config_root, inference_config_file): + return resolve_benchmark_variant_key(inference_config_root, inference_config_file) + + +@pytest.fixture(scope="module") +def benchmark_params(benchmark_params_dict, benchmark_variant): + return benchmark_params_dict[benchmark_variant] + + +@pytest.fixture(scope="module") +def thresholds_dict(benchmark_params, benchmark_variant): + """Load thresholds from the path in ``threshold_file``.""" + threshold_path_str = _threshold_file_path(benchmark_params) + if not threshold_path_str: + pytest.fail(f"benchmark_params[{benchmark_variant!r}] missing 'threshold_file' in --config_file") + + threshold_path = _resolve_threshold_path(threshold_path_str) + thresholds = _load_thresholds_file(threshold_path) + log.info("Loaded thresholds from %s (%d cells)", threshold_path, len(thresholds)) + return thresholds + + +@pytest.fixture(scope="module") +def hf_token(inference_dict): + hf_token_file = inference_dict["hf_token_file"] + try: + with open(hf_token_file, encoding="utf-8") as fp: + return fp.read().rstrip("\n") + except FileNotFoundError: + pytest.fail(f"hf_token file not found: {hf_token_file}") + except OSError as e: + pytest.fail(f"cannot read hf_token file {hf_token_file}: {e}") + + +@pytest.fixture(scope="module") +def p_phdl(cluster_dict, inference_dict): + env_vars = cluster_dict.get("env_vars") + return Pssh( + log, + inference_dict["prefill_node_list"], + user=cluster_dict["username"], + pkey=cluster_dict["priv_key_file"], + env_vars=env_vars, + ) + + +@pytest.fixture(scope="module") +def d_phdl(cluster_dict, inference_dict): + env_vars = cluster_dict.get("env_vars") + return Pssh( + log, + inference_dict["decode_node_list"], + user=cluster_dict["username"], + pkey=cluster_dict["priv_key_file"], + env_vars=env_vars, + ) + + +@pytest.fixture(scope="module") +def r_phdl(cluster_dict, inference_dict): + env_vars = cluster_dict.get("env_vars") + return Pssh( + log, + [inference_dict["proxy_router_node"]], + user=cluster_dict["username"], + pkey=cluster_dict["priv_key_file"], + env_vars=env_vars, + ) + + +@pytest.fixture(scope="module") +def b_phdl(cluster_dict, inference_dict): + env_vars = cluster_dict.get("env_vars") + return Pssh( + log, + [inference_dict["benchmark_serv_node"]], + user=cluster_dict["username"], + pkey=cluster_dict["priv_key_file"], + env_vars=env_vars, + ) + + +@pytest.fixture(scope="module") +def gpu_type(p_phdl, cluster_dict): + head_node = p_phdl.host_list[0] + smi_out_dict = p_phdl.exec("rocm-smi -a | head -30") + smi_out = smi_out_dict[head_node] + return get_model_from_rocm_smi_output(smi_out) + + +@pytest.fixture(scope="module") +def inf_res_dict(): + return {} + + +@pytest.fixture(scope="module") +def im_obj( + p_phdl, + d_phdl, + r_phdl, + b_phdl, + gpu_type, + inference_dict, + benchmark_params, + thresholds_dict, + hf_token, +): + bp_dict = dict(benchmark_params) + _inject_thresholds_into_bp_dict(bp_dict, thresholds_dict) + + return sglang_disagg_lib.SglangDisaggPD( + bp_dict["model"], + inference_dict, + bp_dict, + hf_token, + p_phdl, + d_phdl, + r_phdl, + b_phdl, + gpu_type, + ) + + +def pytest_collection_modifyitems(items): + rank = SGLANG_DISAGG_TEST_ORDER + items.sort(key=lambda it: rank.get(it.originalname or it.name.split("[")[0], 99)) diff --git a/cvs/tests/inference/sglang/sglang_deepseek_r1_671b_distributed.py b/cvs/tests/inference/sglang/sglang_deepseek_r1_671b_distributed.py deleted file mode 100644 index e8692c441..000000000 --- a/cvs/tests/inference/sglang/sglang_deepseek_r1_671b_distributed.py +++ /dev/null @@ -1,439 +0,0 @@ -''' -Copyright 2025 Advanced Micro Devices, Inc. -All rights reserved. This notice is intended as a precaution against inadvertent publication and does not imply publication or any waiver of confidentiality. -The year included in the foregoing notice is the year of creation of the work. -All code contained here is Property of Advanced Micro Devices, Inc. -''' - -import pytest - -import re -import time -import json - -from cvs.lib.parallel_ssh_lib import * -from cvs.lib.utils_lib import * -from cvs.lib import docker_lib -from cvs.lib import sglang_disagg_lib -from cvs.lib import globals - -log = globals.log - - -# Importing additional cmd line args to script .. -@pytest.fixture(scope="module") -def cluster_file(pytestconfig): - """ - Retrieve the --cluster_file CLI option provided to pytest. - - Args: - pytestconfig: Built-in pytest fixture exposing command-line options. - - Returns: - str: Path to the cluster JSON file specified via --cluster_file. - - Notes: - - Ensure your pytest.ini or CLI includes: --cluster_file=/path/to/cluster.json - - Use module scope so the value is resolved once per test module. - """ - return pytestconfig.getoption("cluster_file") - - -@pytest.fixture(scope="module") -def inference_config_file(pytestconfig): - """ - Retrieve the --config_file CLI option provided to pytest. - - Args: - pytestconfig: Built-in pytest fixture exposing command-line options. - - Returns: - str: Path to the training config JSON file specified via --config_file. - - Notes: - - Ensure your pytest.ini or CLI includes: --config_file=/path/to/training_config.json - - Module scope avoids re-fetching the option across tests in this module. - """ - return pytestconfig.getoption("config_file") - - -# Importing the cluster and cofig files to script to access node, switch, test config params -@pytest.fixture(scope="module") -def cluster_dict(cluster_file): - """ - Load the entire cluster configuration from the provided JSON file. - - Args: - cluster_file (str): Path to the cluster JSON file. - - Returns: - dict: Parsed JSON representing the cluster (nodes, credentials, etc.). - - Notes: - - Logs the loaded structure for visibility; consider using log.debug if verbose. - """ - with open(cluster_file) as json_file: - cluster_dict = json.load(json_file) - - # Resolve path placeholders like {user-id} in cluster config - cluster_dict = resolve_cluster_config_placeholders(cluster_dict) - log.info("%s", cluster_dict) - return cluster_dict - - -@pytest.fixture(scope="module") -def inference_dict(inference_config_file, cluster_dict): - with open(inference_config_file) as json_file: - inference_dict_t = json.load(json_file) - inference_dict = inference_dict_t['config'] - - # Resolve path placeholders like {user-id}, {home-mount-dir}, etc. - inference_dict = resolve_test_config_placeholders(inference_dict, cluster_dict) - return inference_dict - - -@pytest.fixture(scope="module") -def benchmark_params_dict(inference_config_file, cluster_dict): - with open(inference_config_file) as json_file: - inference_dict_t = json.load(json_file) - benchmark_params_dict = inference_dict_t['benchmark_params'] - - # Resolve path placeholders like {user-id}, {home-mount-dir}, etc. - benchmark_params_dict = resolve_test_config_placeholders(benchmark_params_dict, cluster_dict) - - log.info("%s", benchmark_params_dict) - return benchmark_params_dict - - -@pytest.fixture(scope="module") -def hf_token(inference_dict): - """ - Load the Hugging Face access token from the file path specified in the training config. - - Args: - inference_dict (dict): Training configuration dict that includes: - - 'hf_token_file': Path to the file containing the HF token. - - Returns: - str: The HF token string read from the file. - - Behavior: - - Reads the token from inference_dict['hf_token_file'] (already resolved for placeholders). - - Strips the trailing newline from the token. - """ - hf_token_file = inference_dict['hf_token_file'] - try: - with open(hf_token_file, 'r') as fp: - hf_token = fp.read().rstrip("\n") - except FileNotFoundError: - log.error(f"Error: The file '{hf_token_file}' was not found.") - except Exception as e: - log.error(f"An error occurred: {e}") - return hf_token - - -@pytest.fixture(scope="module") -def p_phdl(cluster_dict, inference_dict): - log.info("%s", cluster_dict) - env_vars = cluster_dict.get("env_vars") - p_phdl = Pssh( - log, - inference_dict['prefill_node_list'], - user=cluster_dict['username'], - pkey=cluster_dict['priv_key_file'], - env_vars=env_vars, - ) - return p_phdl - - -@pytest.fixture(scope="module") -def d_phdl(cluster_dict, inference_dict): - env_vars = cluster_dict.get("env_vars") - d_phdl = Pssh( - log, - inference_dict['decode_node_list'], - user=cluster_dict['username'], - pkey=cluster_dict['priv_key_file'], - env_vars=env_vars, - ) - return d_phdl - - -@pytest.fixture(scope="module") -def r_phdl(cluster_dict, inference_dict): - node_list = [] - env_vars = cluster_dict.get("env_vars") - node_list.append(inference_dict['proxy_router_node']) - r_phdl = Pssh(log, node_list, user=cluster_dict['username'], pkey=cluster_dict['priv_key_file'], env_vars=env_vars) - return r_phdl - - -@pytest.fixture(scope="module") -def b_phdl(cluster_dict, inference_dict): - node_list = [] - env_vars = cluster_dict.get("env_vars") - node_list.append(inference_dict['benchmark_serv_node']) - b_phdl = Pssh(log, node_list, user=cluster_dict['username'], pkey=cluster_dict['priv_key_file'], env_vars=env_vars) - return b_phdl - - -@pytest.fixture(scope="module") -def gpu_type(p_phdl, cluster_dict): - """ - Provide the GPU type string for the test module. - - Args: - cluster_dict (dict): Cluster configuration that includes the GPU type. - - Returns: - str: The GPU type (e.g., 'mi300', 'mi300x') used to select model parameters and logic. - - Notes: - - Module scope ensures this is evaluated once per test module. - - Consider validating this value against an expected set of GPU types to catch typos early. - """ - - log.info("%s", p_phdl) - head_node = p_phdl.host_list[0] - smi_out_dict = p_phdl.exec('rocm-smi -a | head -30') - smi_out = smi_out_dict[head_node] - gpu_type = get_model_from_rocm_smi_output(smi_out) - return gpu_type - - -def test_cleanup_stale_containers(p_phdl, d_phdl, r_phdl, b_phdl, inference_dict): - """ - Pytest: Clean up potentially stale Docker containers and volumes before tests. - - Notes: - - This performs a broad cleanup via delete_all_containers_and_volumes; ensure the - test environment is isolated so this doesn?t remove unrelated containers/volumes. - - Consider narrowing cleanup scope if other workloads may be present on the hosts. - """ - - container_name = inference_dict['container_name'] - for a_phdl in [p_phdl, d_phdl, r_phdl, b_phdl]: - docker_lib.kill_docker_container(a_phdl, container_name) - docker_lib.delete_all_containers_and_volumes(a_phdl) - - # Cleanup log directory from one of the nodes - log.info('Cleaning up log directory') - r_phdl.exec(f"sudo rm -rf {inference_dict['log_dir']}") - time.sleep(5) - - -def test_launch_inference_containers(p_phdl, d_phdl, r_phdl, b_phdl, inference_dict): - log.info('Testcase launch SGLang containers') - globals.error_list = [] - container_name = inference_dict['container_name'] - # Launch the containers .. - hdl_list = [p_phdl, d_phdl] - # Users can use the one of the prefill, decode nodes as proxy, benchmark, so - # check before scheduling - if inference_dict['proxy_router_node'] == inference_dict['benchmark_serv_node']: - if (inference_dict['proxy_router_node'] in inference_dict['prefill_node_list']) or ( - inference_dict['proxy_router_node'] in inference_dict['decode_node_list'] - ): - log.info('Already part of the handle list, no need to add') - else: - hdl_list.extend(r_phdl) - else: - if (inference_dict['proxy_router_node'] in inference_dict['prefill_node_list']) or ( - inference_dict['proxy_router_node'] in inference_dict['decode_node_list'] - ): - log.info('Already part of the handle list, no need to add') - else: - hdl_list.extend(r_phdl) - if (inference_dict['benchmark_serv_node'] in inference_dict['prefill_node_list']) or ( - inference_dict['benchmark_serv_node'] in inference_dict['decode_node_list'] - ): - log.info('Already part of the handle list, no need to add') - else: - hdl_list.extend(b_phdl) - - for a_phdl in hdl_list: - docker_lib.launch_docker_container( - a_phdl, - container_name, - inference_dict['container_image'], - inference_dict['container_config']['device_list'], - inference_dict['container_config']['volume_dict'], - inference_dict['container_config']['env_dict'], - shm_size='48G', - timeout=60 * 20, - ) - # ADD verifications .. - time.sleep(30) - log.info('Verify if the containers have been launched properly') - for a_phdl in [p_phdl, d_phdl, r_phdl, b_phdl]: - out_dict = a_phdl.exec('docker ps') - for node in out_dict.keys(): - if not re.search(f'{container_name}', out_dict[node], re.I): - fail_test(f'Failed to launch container on node {node}') - update_test_result() - - -# Setup the ib devices and ensure they show up in the container -def test_setup_ibv_devices(im_obj): - """ - Validate that InfiniBand / RDMA devices are: - - Properly configured on the host - - Visible inside the inference container - - Ready for high-performance communication (Prefill/Decode) - - This test is foundational and must pass before any inference tests - that rely on RDMA-based KV cache transfer. - """ - globals.error_list = [] - im_obj.check_ibv_devices() - im_obj.exec_nic_setup_scripts() - update_test_result() - - -# Create the SGlang Inference Object Fixture -@pytest.fixture(scope="module") -def im_obj(p_phdl, d_phdl, r_phdl, b_phdl, gpu_type, inference_dict, benchmark_params_dict, hf_token): - globals.error_list = [] - bp_dict = benchmark_params_dict['deepseek-r1'] - im_obj = sglang_disagg_lib.SglangDisaggPD( - bp_dict['model'], inference_dict, bp_dict, hf_token, p_phdl, d_phdl, r_phdl, b_phdl, gpu_type - ) - return im_obj - - -def test_rms_norm(im_obj): - """ - Run RMSNorm operator tests to validate: - - GPU kernel correctness - - AITer backend functionality - - Basic compute stability before inference - - This serves as a low-level sanity check before launching servers. - """ - globals.error_list = [] - im_obj.run_test_rmsnorm() - update_test_result() - - -# Test to start the prefill servers using sglang.launch_server -def test_launch_prefill_servers(im_obj): - """ - Start SGLang Prefill servers in disaggregated PD mode. - - Prefill servers: - - Handle prompt processing - - Generate KV cache - - Serve as upstream for Decode servers - - This test prepares the Prefill side of the inference pipeline. - """ - globals.error_list = [] - im_obj.setup_prefill_container_env() - im_obj.launch_prefill_servers() - update_test_result() - - -# Test to start the decode servers using sglang.launch_server -def test_launch_decode_servers(im_obj): - """ - Start SGLang Decode servers in disaggregated PD mode. - - Decode servers: - - Consume KV cache from Prefill servers - - Generate output tokens - - Drive decode throughput and latency - - This test completes the inference data plane. - """ - globals.error_list = [] - im_obj.setup_decode_container_env() - im_obj.launch_decode_servers() - update_test_result() - - -# Test to validate the Prefill and Decode servers are ready to serve -# Inference traffic -def test_poll_for_server_ready(im_obj): - """ - Poll Prefill and Decode server logs to ensure: - - Servers have fully started - - Models are loaded - - HTTP endpoints are responding (200 OK) - - Systems are stable before inference begins - - This test prevents inference traffic from being sent too early. - """ - globals.error_list = [] - im_obj.poll_and_check_server_ready() - update_test_result() - - -# Start the proxy router serving using sglang_router.launch_router -def test_launch_proxy_router(im_obj): - """ - Start the SGLang Proxy Router. - - The Proxy Router: - - Accepts inference requests - - Routes Prefill and Decode traffic - - Coordinates disaggregated PD execution - - This test completes the control plane setup for inference serving. - """ - globals.error_list = [] - im_obj.setup_proxy_router_container_env() - im_obj.launch_proxy_router() - update_test_result() - - -# Test to run the canned gsm8k benchmark packaged with the container -def test_run_gsm8k_benchmark_test(im_obj): - """ - Execute the GSM8K benchmark using the SGLang inference serving stack. - - Purpose: - -------- - This test validates: - - End-to-end inference correctness using a real-world dataset - - Sustained decode throughput under realistic query patterns - - Proper interaction between Proxy Router, Prefill, and Decode servers - - GSM8K is a commonly used reasoning benchmark, making it a strong - signal for both correctness and performance regression detection. - """ - globals.error_list = [] - im_obj.setup_benchmark_serv_container_env() - im_obj.run_gsm8k_benchmark_test() - update_test_result() - - -# Test to run the sglang Benchmarking Testing using bench_serv -def test_run_benchmark_test(im_obj): - """ - Execute a synthetic serving benchmark using sglang.bench_serving - with a random dataset. - - Purpose: - -------- - This test focuses on: - - Stress-testing the serving infrastructure - - Evaluating scheduling, batching, and throughput under load - - Isolating serving performance independent of dataset semantics - - Randomized workloads are useful for detecting performance regressions - and scaling bottlenecks. - """ - globals.error_list = [] - im_obj.setup_benchmark_serv_container_env() - im_obj.benchserv_test_random(d_type='auto') - update_test_result() - - -# Test to validate the prefill/decode GPU layout -def test_disagg_gpu_topology(im_obj): - """ - Report occupied GPUs on prefill/decode nodes after model load. - """ - globals.error_list = [] - im_obj.sglang_disagg_gpu_counts() - update_test_result() diff --git a/cvs/tests/inference/sglang/sglang_disagg_distributed.py b/cvs/tests/inference/sglang/sglang_disagg_distributed.py new file mode 100644 index 000000000..8f558d1db --- /dev/null +++ b/cvs/tests/inference/sglang/sglang_disagg_distributed.py @@ -0,0 +1,175 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Single SGLang disaggregated (PD) benchmark module: model is selected from +``benchmark_params`` via ``active_benchmark`` / env / single-key auto (see ``_shared``). +''' + +import re +import time + +from cvs.lib import docker_lib, globals +from cvs.lib.utils_lib import fail_test, update_test_result +from cvs.tests.inference.sglang._shared import test_print_results_table # noqa: F401 + +log = globals.log + + +def test_cleanup_stale_containers(p_phdl, d_phdl, r_phdl, b_phdl, inference_dict): + container_name = inference_dict["container_name"] + for a_phdl in (p_phdl, d_phdl, r_phdl, b_phdl): + docker_lib.kill_docker_container(a_phdl, container_name) + docker_lib.delete_all_containers_and_volumes(a_phdl) + log.info("Cleaning up log directory") + r_phdl.exec(f"sudo rm -rf {inference_dict['log_dir']}") + time.sleep(5) + + +def test_launch_inference_containers(p_phdl, d_phdl, r_phdl, b_phdl, inference_dict): + log.info("Testcase launch SGLang containers") + globals.error_list = [] + container_name = inference_dict["container_name"] + hdl_list = [p_phdl, d_phdl] + + if inference_dict["proxy_router_node"] == inference_dict["benchmark_serv_node"]: + if (inference_dict["proxy_router_node"] in inference_dict["prefill_node_list"]) or ( + inference_dict["proxy_router_node"] in inference_dict["decode_node_list"] + ): + log.info("Already part of the handle list, no need to add") + else: + hdl_list.append(r_phdl) + else: + if (inference_dict["proxy_router_node"] in inference_dict["prefill_node_list"]) or ( + inference_dict["proxy_router_node"] in inference_dict["decode_node_list"] + ): + log.info("Already part of the handle list, no need to add") + else: + hdl_list.append(r_phdl) + if (inference_dict["benchmark_serv_node"] in inference_dict["prefill_node_list"]) or ( + inference_dict["benchmark_serv_node"] in inference_dict["decode_node_list"] + ): + log.info("Already part of the handle list, no need to add") + else: + hdl_list.append(b_phdl) + + for a_phdl in hdl_list: + docker_lib.launch_docker_container( + a_phdl, + container_name, + inference_dict["container_image"], + inference_dict["container_config"]["device_list"], + inference_dict["container_config"]["volume_dict"], + inference_dict["container_config"]["env_dict"], + shm_size="48G", + timeout=60 * 20, + ) + time.sleep(30) + log.info("Verify if the containers have been launched properly") + for a_phdl in (p_phdl, d_phdl, r_phdl, b_phdl): + out_dict = a_phdl.exec("docker ps") + for node, out in out_dict.items(): + if not re.search(re.escape(container_name), out or "", re.I): + fail_test(f"Failed to launch container on node {node}") + update_test_result() + + +def test_setup_ibv_devices(im_obj): + globals.error_list = [] + im_obj.check_ibv_devices() + im_obj.exec_nic_setup_scripts() + update_test_result() + + +def test_rms_norm(im_obj): + globals.error_list = [] + im_obj.run_test_rmsnorm() + update_test_result() + + +def test_launch_prefill_servers(im_obj): + globals.error_list = [] + im_obj.setup_prefill_container_env() + im_obj.launch_prefill_servers() + update_test_result() + + +def test_launch_decode_servers(im_obj): + globals.error_list = [] + im_obj.setup_decode_container_env() + im_obj.launch_decode_servers() + update_test_result() + + +def test_poll_for_server_ready(im_obj): + globals.error_list = [] + im_obj.poll_and_check_server_ready() + update_test_result() + + +def test_launch_proxy_router(im_obj): + globals.error_list = [] + im_obj.setup_proxy_router_container_env() + im_obj.launch_proxy_router() + update_test_result() + + +def test_openai_compatible_http_endpoints(im_obj, inf_res_dict): + globals.error_list = [] + results = im_obj.verify_openai_compatible_endpoints() + inf_res_dict["__smoke_probe_results__"] = results + update_test_result() + + +def test_run_lm_eval_hellaswag_benchmark_test(im_obj, inf_res_dict): + globals.error_list = [] + im_obj.setup_benchmark_serv_container_env() + h = im_obj.run_lm_eval_hellaswag_benchmark_test() + inf_res_dict.setdefault("__phase_labels__", {})["accuracy_hellaswag"] = h + update_test_result() + + +def test_run_lm_eval_gsm8k_benchmark_test(im_obj, inf_res_dict): + globals.error_list = [] + im_obj.setup_benchmark_serv_container_env() + g = im_obj.run_lm_eval_gsm8k_benchmark_test() + inf_res_dict.setdefault("__phase_labels__", {})["accuracy_gsm8k"] = g + update_test_result() + + +def test_run_lm_eval_mmlu_benchmark_test(im_obj, inf_res_dict): + globals.error_list = [] + im_obj.setup_benchmark_serv_container_env() + m = im_obj.run_lm_eval_mmlu_benchmark_test() + inf_res_dict.setdefault("__phase_labels__", {})["accuracy_mmlu"] = m + update_test_result() + + +def test_run_performance_benchmark_test(im_obj, inf_res_dict): + globals.error_list = [] + im_obj.setup_benchmark_serv_container_env() + im_obj.benchserv_test_random(d_type="auto") + + bench = (im_obj.bp_dict.get("inference_tests") or {}).get("bench_serv_random") or {} + expected = (bench.get("expected_results") or {}).get("auto") or {} + + key = ( + im_obj.model_name, + im_obj.gpu_type, + str(bench.get("input_length", "-")), + str(bench.get("output_length", "-")), + "bench_serv_random", + str(im_obj.bp_dict.get("max_concurrency", "-")), + ) + labels = inf_res_dict.setdefault("__phase_labels__", {}) + labels["performance_expected"] = expected + labels["performance_test"] = "PASS" if not globals.error_list else "FAIL" + + inf_res_dict[key] = dict(im_obj.inference_results_dict or {}) + update_test_result() + + +def test_disagg_gpu_topology(im_obj): + globals.error_list = [] + im_obj.sglang_disagg_gpu_counts() + update_test_result() diff --git a/cvs/tests/inference/sglang/sglang_llama_70b_distributed.py b/cvs/tests/inference/sglang/sglang_llama_70b_distributed.py deleted file mode 100644 index 5ed25ce11..000000000 --- a/cvs/tests/inference/sglang/sglang_llama_70b_distributed.py +++ /dev/null @@ -1,439 +0,0 @@ -''' -Copyright 2025 Advanced Micro Devices, Inc. -All rights reserved. This notice is intended as a precaution against inadvertent publication and does not imply publication or any waiver of confidentiality. -The year included in the foregoing notice is the year of creation of the work. -All code contained here is Property of Advanced Micro Devices, Inc. -''' - -import pytest - -import re -import time -import json - -from cvs.lib.parallel_ssh_lib import * -from cvs.lib.utils_lib import * -from cvs.lib import docker_lib -from cvs.lib import sglang_disagg_lib -from cvs.lib import globals - -log = globals.log - - -# Importing additional cmd line args to script .. -@pytest.fixture(scope="module") -def cluster_file(pytestconfig): - """ - Retrieve the --cluster_file CLI option provided to pytest. - - Args: - pytestconfig: Built-in pytest fixture exposing command-line options. - - Returns: - str: Path to the cluster JSON file specified via --cluster_file. - - Notes: - - Ensure your pytest.ini or CLI includes: --cluster_file=/path/to/cluster.json - - Use module scope so the value is resolved once per test module. - """ - return pytestconfig.getoption("cluster_file") - - -@pytest.fixture(scope="module") -def inference_config_file(pytestconfig): - """ - Retrieve the --config_file CLI option provided to pytest. - - Args: - pytestconfig: Built-in pytest fixture exposing command-line options. - - Returns: - str: Path to the training config JSON file specified via --config_file. - - Notes: - - Ensure your pytest.ini or CLI includes: --config_file=/path/to/training_config.json - - Module scope avoids re-fetching the option across tests in this module. - """ - return pytestconfig.getoption("config_file") - - -# Importing the cluster and cofig files to script to access node, switch, test config params -@pytest.fixture(scope="module") -def cluster_dict(cluster_file): - """ - Load the entire cluster configuration from the provided JSON file. - - Args: - cluster_file (str): Path to the cluster JSON file. - - Returns: - dict: Parsed JSON representing the cluster (nodes, credentials, etc.). - - Notes: - - Logs the loaded structure for visibility; consider using log.debug if verbose. - """ - with open(cluster_file) as json_file: - cluster_dict = json.load(json_file) - - # Resolve path placeholders like {user-id} in cluster config - cluster_dict = resolve_cluster_config_placeholders(cluster_dict) - log.info("%s", cluster_dict) - return cluster_dict - - -@pytest.fixture(scope="module") -def inference_dict(inference_config_file, cluster_dict): - with open(inference_config_file) as json_file: - inference_dict_t = json.load(json_file) - inference_dict = inference_dict_t['config'] - - # Resolve path placeholders like {user-id}, {home-mount-dir}, etc. - inference_dict = resolve_test_config_placeholders(inference_dict, cluster_dict) - return inference_dict - - -@pytest.fixture(scope="module") -def benchmark_params_dict(inference_config_file, cluster_dict): - with open(inference_config_file) as json_file: - inference_dict_t = json.load(json_file) - benchmark_params_dict = inference_dict_t['benchmark_params'] - - # Resolve path placeholders like {user-id}, {home-mount-dir}, etc. - benchmark_params_dict = resolve_test_config_placeholders(benchmark_params_dict, cluster_dict) - - log.info("%s", benchmark_params_dict) - return benchmark_params_dict - - -@pytest.fixture(scope="module") -def hf_token(inference_dict): - """ - Load the Hugging Face access token from the file path specified in the training config. - - Args: - inference_dict (dict): Training configuration dict that includes: - - 'hf_token_file': Path to the file containing the HF token. - - Returns: - str: The HF token string read from the file. - - Behavior: - - Reads the token from inference_dict['hf_token_file'] (already resolved for placeholders). - - Strips the trailing newline from the token. - """ - hf_token_file = inference_dict['hf_token_file'] - try: - with open(hf_token_file, 'r') as fp: - hf_token = fp.read().rstrip("\n") - except FileNotFoundError: - log.error(f"Error: The file '{hf_token_file}' was not found.") - except Exception as e: - log.error(f"An error occurred: {e}") - return hf_token - - -@pytest.fixture(scope="module") -def p_phdl(cluster_dict, inference_dict): - log.info("%s", cluster_dict) - env_vars = cluster_dict.get("env_vars") - p_phdl = Pssh( - log, - inference_dict['prefill_node_list'], - user=cluster_dict['username'], - pkey=cluster_dict['priv_key_file'], - env_vars=env_vars, - ) - return p_phdl - - -@pytest.fixture(scope="module") -def d_phdl(cluster_dict, inference_dict): - env_vars = cluster_dict.get("env_vars") - d_phdl = Pssh( - log, - inference_dict['decode_node_list'], - user=cluster_dict['username'], - pkey=cluster_dict['priv_key_file'], - env_vars=env_vars, - ) - return d_phdl - - -@pytest.fixture(scope="module") -def r_phdl(cluster_dict, inference_dict): - node_list = [] - env_vars = cluster_dict.get("env_vars") - node_list.append(inference_dict['proxy_router_node']) - r_phdl = Pssh(log, node_list, user=cluster_dict['username'], pkey=cluster_dict['priv_key_file'], env_vars=env_vars) - return r_phdl - - -@pytest.fixture(scope="module") -def b_phdl(cluster_dict, inference_dict): - node_list = [] - env_vars = cluster_dict.get("env_vars") - node_list.append(inference_dict['benchmark_serv_node']) - b_phdl = Pssh(log, node_list, user=cluster_dict['username'], pkey=cluster_dict['priv_key_file'], env_vars=env_vars) - return b_phdl - - -@pytest.fixture(scope="module") -def gpu_type(p_phdl, cluster_dict): - """ - Provide the GPU type string for the test module. - - Args: - cluster_dict (dict): Cluster configuration that includes the GPU type. - - Returns: - str: The GPU type (e.g., 'mi300', 'mi300x') used to select model parameters and logic. - - Notes: - - Module scope ensures this is evaluated once per test module. - - Consider validating this value against an expected set of GPU types to catch typos early. - """ - - log.info("%s", p_phdl) - head_node = p_phdl.host_list[0] - smi_out_dict = p_phdl.exec('rocm-smi -a | head -30') - smi_out = smi_out_dict[head_node] - gpu_type = get_model_from_rocm_smi_output(smi_out) - return gpu_type - - -def test_cleanup_stale_containers(p_phdl, d_phdl, r_phdl, b_phdl, inference_dict): - """ - Pytest: Clean up potentially stale Docker containers and volumes before tests. - - Notes: - - This performs a broad cleanup via delete_all_containers_and_volumes; ensure the - test environment is isolated so this doesn?t remove unrelated containers/volumes. - - Consider narrowing cleanup scope if other workloads may be present on the hosts. - """ - - container_name = inference_dict['container_name'] - for a_phdl in [p_phdl, d_phdl, r_phdl, b_phdl]: - docker_lib.kill_docker_container(a_phdl, container_name) - docker_lib.delete_all_containers_and_volumes(a_phdl) - - # Cleanup log directory from one of the nodes - log.info('Cleaning up log directory') - r_phdl.exec(f"sudo rm -rf {inference_dict['log_dir']}") - time.sleep(5) - - -def test_launch_inference_containers(p_phdl, d_phdl, r_phdl, b_phdl, inference_dict): - log.info('Testcase launch SGLang containers') - globals.error_list = [] - container_name = inference_dict['container_name'] - # Launch the containers .. - hdl_list = [p_phdl, d_phdl] - # Users can use the one of the prefill, decode nodes as proxy, benchmark, so - # check before scheduling - if inference_dict['proxy_router_node'] == inference_dict['benchmark_serv_node']: - if (inference_dict['proxy_router_node'] in inference_dict['prefill_node_list']) or ( - inference_dict['proxy_router_node'] in inference_dict['decode_node_list'] - ): - log.info('Already part of the handle list, no need to add') - else: - hdl_list.extend(r_phdl) - else: - if (inference_dict['proxy_router_node'] in inference_dict['prefill_node_list']) or ( - inference_dict['proxy_router_node'] in inference_dict['decode_node_list'] - ): - log.info('Already part of the handle list, no need to add') - else: - hdl_list.extend(r_phdl) - if (inference_dict['benchmark_serv_node'] in inference_dict['prefill_node_list']) or ( - inference_dict['benchmark_serv_node'] in inference_dict['decode_node_list'] - ): - log.info('Already part of the handle list, no need to add') - else: - hdl_list.extend(b_phdl) - - for a_phdl in hdl_list: - docker_lib.launch_docker_container( - a_phdl, - container_name, - inference_dict['container_image'], - inference_dict['container_config']['device_list'], - inference_dict['container_config']['volume_dict'], - inference_dict['container_config']['env_dict'], - shm_size='48G', - timeout=60 * 20, - ) - # ADD verifications .. - time.sleep(30) - log.info('Verify if the containers have been launched properly') - for a_phdl in [p_phdl, d_phdl, r_phdl, b_phdl]: - out_dict = a_phdl.exec('docker ps') - for node in out_dict.keys(): - if not re.search(f'{container_name}', out_dict[node], re.I): - fail_test(f'Failed to launch container on node {node}') - update_test_result() - - -# Setup the ib devices and ensure they show up in the container -def test_setup_ibv_devices(im_obj): - """ - Validate that InfiniBand / RDMA devices are: - - Properly configured on the host - - Visible inside the inference container - - Ready for high-performance communication (Prefill/Decode) - - This test is foundational and must pass before any inference tests - that rely on RDMA-based KV cache transfer. - """ - globals.error_list = [] - im_obj.check_ibv_devices() - im_obj.exec_nic_setup_scripts() - update_test_result() - - -# Create the SGlang Inference Object Fixture -@pytest.fixture(scope="module") -def im_obj(p_phdl, d_phdl, r_phdl, b_phdl, gpu_type, inference_dict, benchmark_params_dict, hf_token): - globals.error_list = [] - bp_dict = benchmark_params_dict['llama-70b'] - im_obj = sglang_disagg_lib.SglangDisaggPD( - bp_dict['model'], inference_dict, bp_dict, hf_token, p_phdl, d_phdl, r_phdl, b_phdl, gpu_type - ) - return im_obj - - -def test_rms_norm(im_obj): - """ - Run RMSNorm operator tests to validate: - - GPU kernel correctness - - AITer backend functionality - - Basic compute stability before inference - - This serves as a low-level sanity check before launching servers. - """ - globals.error_list = [] - im_obj.run_test_rmsnorm() - update_test_result() - - -# Test to start the prefill servers using sglang.launch_server -def test_launch_prefill_servers(im_obj): - """ - Start SGLang Prefill servers in disaggregated PD mode. - - Prefill servers: - - Handle prompt processing - - Generate KV cache - - Serve as upstream for Decode servers - - This test prepares the Prefill side of the inference pipeline. - """ - globals.error_list = [] - im_obj.setup_prefill_container_env() - im_obj.launch_prefill_servers() - update_test_result() - - -# Test to start the decode servers using sglang.launch_server -def test_launch_decode_servers(im_obj): - """ - Start SGLang Decode servers in disaggregated PD mode. - - Decode servers: - - Consume KV cache from Prefill servers - - Generate output tokens - - Drive decode throughput and latency - - This test completes the inference data plane. - """ - globals.error_list = [] - im_obj.setup_decode_container_env() - im_obj.launch_decode_servers() - update_test_result() - - -# Test to validate the Prefill and Decode servers are ready to serve -# Inference traffic -def test_poll_for_server_ready(im_obj): - """ - Poll Prefill and Decode server logs to ensure: - - Servers have fully started - - Models are loaded - - HTTP endpoints are responding (200 OK) - - Systems are stable before inference begins - - This test prevents inference traffic from being sent too early. - """ - globals.error_list = [] - im_obj.poll_and_check_server_ready() - update_test_result() - - -# Start the proxy router serving using sglang_router.launch_router -def test_launch_proxy_router(im_obj): - """ - Start the SGLang Proxy Router. - - The Proxy Router: - - Accepts inference requests - - Routes Prefill and Decode traffic - - Coordinates disaggregated PD execution - - This test completes the control plane setup for inference serving. - """ - globals.error_list = [] - im_obj.setup_proxy_router_container_env() - im_obj.launch_proxy_router() - update_test_result() - - -# Test to run the canned gsm8k benchmark packaged with the container -def test_run_gsm8k_benchmark_test(im_obj): - """ - Execute the GSM8K benchmark using the SGLang inference serving stack. - - Purpose: - -------- - This test validates: - - End-to-end inference correctness using a real-world dataset - - Sustained decode throughput under realistic query patterns - - Proper interaction between Proxy Router, Prefill, and Decode servers - - GSM8K is a commonly used reasoning benchmark, making it a strong - signal for both correctness and performance regression detection. - """ - globals.error_list = [] - im_obj.setup_benchmark_serv_container_env() - im_obj.run_gsm8k_benchmark_test() - update_test_result() - - -# Test to run the sglang Benchmarking Testing using bench_serv -def test_run_benchmark_test(im_obj): - """ - Execute a synthetic serving benchmark using sglang.bench_serving - with a random dataset. - - Purpose: - -------- - This test focuses on: - - Stress-testing the serving infrastructure - - Evaluating scheduling, batching, and throughput under load - - Isolating serving performance independent of dataset semantics - - Randomized workloads are useful for detecting performance regressions - and scaling bottlenecks. - """ - globals.error_list = [] - im_obj.setup_benchmark_serv_container_env() - im_obj.benchserv_test_random(d_type='auto') - update_test_result() - - -# Test to validate the prefill/decode GPU layout -def test_disagg_gpu_topology(im_obj): - """ - Report occupied GPUs on prefill/decode nodes after model load. - """ - globals.error_list = [] - im_obj.sglang_disagg_gpu_counts() - update_test_result() diff --git a/cvs/tests/inference/vllm/_shared.py b/cvs/tests/inference/vllm/_shared.py new file mode 100644 index 000000000..daedddde3 --- /dev/null +++ b/cvs/tests/inference/vllm/_shared.py @@ -0,0 +1,70 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Shared test helpers for the unified vllm suite. + +`test_print_results_table` is exported and imported by `vllm.py` as a sibling +test that pytest runs LAST (lexically after `test_vllm_inference`). +''' + +from tabulate import tabulate + +from cvs.lib import globals + +log = globals.log + +__all__ = ["test_print_results_table"] + + +def _cell(m, key): + """Table cell: a missing OR present-but-None metric renders as '-'.""" + v = m.get(key) + return "-" if v is None else v + + +def test_print_results_table(inf_res_dict): + if not inf_res_dict: + log.info("inf_res_dict empty, nothing to print") + return + headers = [ + "Model", + "GPU", + "ISL", + "OSL", + "Policy", + "Conc", + "Host", + "Req/s", + "Total tok/s", + "Mean TTFT (ms)", + "P95 TTFT (ms)", + "Mean TPOT (ms)", + "P95 TPOT (ms)", + "P99 ITL (ms)", + "Goodput (req/s)", + ] + rows = [] + for key, host_dict in inf_res_dict.items(): + model, gpu, isl, osl, policy, conc = key + for host, m in host_dict.items(): + rows.append( + [ + model, + gpu, + isl, + osl, + policy, + conc, + host, + _cell(m, "client.request_throughput"), + _cell(m, "client.total_token_throughput"), + _cell(m, "client.mean_ttft_ms"), + _cell(m, "client.p95_ttft_ms"), + _cell(m, "client.mean_tpot_ms"), + _cell(m, "client.p95_tpot_ms"), + _cell(m, "client.p99_itl_ms"), + _cell(m, "client.goodput"), + ] + ) + log.info("\n" + tabulate(rows, headers=headers, tablefmt="github")) diff --git a/cvs/tests/inference/vllm/conftest.py b/cvs/tests/inference/vllm/conftest.py new file mode 100644 index 000000000..ee10326cb --- /dev/null +++ b/cvs/tests/inference/vllm/conftest.py @@ -0,0 +1,176 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. +''' + +import json +import os + +import pytest + +from cvs.core.orchestrators.factory import OrchestratorConfig, OrchestratorFactory +from cvs.lib import globals +from cvs.lib.inference.utils.vllm_config_loader import load_variant +from cvs.lib.utils_lib import resolve_cluster_config_placeholders + +log = globals.log + + +def _deep_merge(base, override): + """Recursively merge `override` onto `base` (dicts merged key-wise, scalars/lists replaced). + + Protects cluster-set SCALAR and DICT container keys (e.g. shm_size, an env + map) from being wiped by a top-level replace: they survive unless the variant + overrides that same key. List keys (e.g. runtime.args, volume mounts) are + REPLACED here, not unioned -- the cluster's list values are recombined with + the variant's additively further downstream, in container.py's getters. + """ + if not (isinstance(base, dict) and isinstance(override, dict)): + return override + out = dict(base) + for k, v in override.items(): + out[k] = _deep_merge(base[k], v) if k in base else v + return out + + +@pytest.fixture(scope="module") +def cluster_dict(pytestconfig): + cluster_file = pytestconfig.getoption("cluster_file") + if not cluster_file: + pytest.fail("--cluster_file is required") + with open(cluster_file) as fp: + d = json.load(fp) + return resolve_cluster_config_placeholders(d) + + +@pytest.fixture(scope="module") +def variant_config(pytestconfig, cluster_dict): + config_file = pytestconfig.getoption("config_file") + if not config_file: + pytest.fail("--config_file is required") + return load_variant(config_file, cluster_dict) + + +class _Lifecycle: + """Cross-test state for the lifecycle-as-tests model. + + The container launch / sshd / fetch / teardown stages are individual tests + (so each is a timed, pass/fail row in the HTML) rather than fixture body + code. They share this object: `failed` lets a broken stage skip the rest + instead of cascading; `torn_down` lets the explicit teardown test suppress + the fixture's leak-guard finalizer; `report` maps a test's nodeid to the + rows it recorded, each carrying its own unit, so attach_inference_suite_lifecycle_table + renders only that test's stages -- not every stage on every row. + """ + + def __init__(self): + self.failed = False + self.torn_down = False + self.report = {} # nodeid -> list[(label, value, unit)] + + def record(self, nodeid, label, value, unit="s"): + self.report.setdefault(nodeid, []).append((label, value, unit)) + + +@pytest.fixture(scope="module") +def lifecycle(): + return _Lifecycle() + + +@pytest.fixture(scope="module") +def orch(cluster_dict, variant_config, lifecycle): + """Construct a ContainerOrchestrator and own ONLY its teardown safety net. + + The actual launch/sshd happen in test_launch_container / test_setup_sshd + so they appear as timed rows. This fixture builds the object and registers a + leak-guard finalizer: if a mid-sweep test fails before test_teardown runs, + the container is still torn down here. When test_teardown ran successfully + it sets lifecycle.torn_down, so the finalizer no-ops (no double teardown). + """ + # OrchestratorConfig.from_configs does a top-level dict.update, so a bare variant + # container block would wipe the cluster file's container settings. Deep-merge the + # variant ONTO the cluster block so cluster-set scalar/dict keys survive, with the + # variant winning on conflicting keys. (List keys like runtime.args are replaced + # here but recombined additively downstream in container.py's getters.) + container_block = _deep_merge( + cluster_dict.get("container", {}), + variant_config.container.model_dump(), + ) + testsuite_config = { + "orchestrator": "container", + "container": container_block, + } + cfg = OrchestratorConfig.from_configs(cluster_dict, testsuite_config) + o = OrchestratorFactory.create_orchestrator(log, cfg) + yield o + if not lifecycle.torn_down: + log.info("orch fixture leak-guard: tearing down container (explicit teardown did not run)") + o.teardown_containers() + + +@pytest.fixture(scope="module") +def hf_token(variant_config): + path = variant_config.paths.hf_token_file + if not os.path.isfile(path): + if variant_config.model.remote == 0: + # Pre-staged model: token not needed for download; server env sets + # HF_HUB_OFFLINE=1 to skip Hub auth checks entirely. + return "" + pytest.skip(f"hf_token file missing: {path}") + with open(path) as fp: + return fp.read().strip() + + +@pytest.fixture(scope="module") +def inf_res_dict(): + return {} + + +def pytest_collection_modifyitems(items): + """Pin the lifecycle order explicitly instead of relying on definition order. + + `test_print_results_table` is an imported function (its source line points + into _shared.py), so default ordering collects it FIRST -- which would log an + empty table before any cell ran. Sort deterministically: launch, sshd, fetch, + the benchmark cells, the results table, then teardown last. Items from other + modules keep their relative order. + """ + rank = { + "test_launch_container": 0, + "test_setup_sshd": 1, + "test_discover_topology": 2, + "test_model_fetch": 3, + "test_vllm_inference": 4, + "test_metric": 5, + "test_print_results_table": 6, + "test_teardown": 7, + } + items.sort(key=lambda it: rank.get(it.originalname or it.name.split("[")[0], 99)) + + +def pytest_html_results_table_header(cells): + """Add Value + Unit columns just before the trailing Links column. + + Populated for test_metric rows; blank for lifecycle/inference rows (they + record no metric_value user-property). Scoped to this suite's conftest, so + other suites' result tables are unaffected. + """ + cells.insert(-1, "Value") + cells.insert(-1, "Unit") + + +def pytest_html_results_table_row(report, cells): + props = dict(report.user_properties) + has = "metric_value" in props + val = props.get("metric_value") + unit = props.get("metric_unit", "") if has else "" + if not has: + shown = "" + elif val is None: + shown = "-" + elif isinstance(val, float): + shown = f"{val:.3f}" + else: + shown = str(val) + cells.insert(-1, f"{shown}") + cells.insert(-1, f"{unit}") diff --git a/cvs/tests/inference/vllm/vllm.py b/cvs/tests/inference/vllm/vllm.py new file mode 100644 index 000000000..845fc3c11 --- /dev/null +++ b/cvs/tests/inference/vllm/vllm.py @@ -0,0 +1,317 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unified vLLM benchmark suite for single-node and multinode distributed runs. + +Replaces vllm_single.py (single-node) and tests/inference/vllm_distributed/ +vllm_distributed.py (distributed) with one parametrized suite. + +The topology is determined entirely by the config file: + nnodes=1 (default) -> single-node, no distributed flags + nnodes=2 + pipeline_parallel_size=2 -> 2-node PP distributed + +IB device discovery (test_discover_topology) runs once per lifecycle for +distributed runs, before the benchmark sweep. Results are stored in the +lifecycle object and passed into VllmJob per cell. +''' + +import json +import os +import shlex +import time + +import pytest + +from cvs.lib import globals +from cvs.lib.inference.utils.vllm_config_loader import GoodputSlo, validate_sweep_selector +from cvs.lib.utils.verdict import evaluate_all +from cvs.lib.inference.utils.vllm_parsing import CLIENT_METRICS as _METRICS, CLIENT_METRIC_UNITS as _METRIC_UNITS +from cvs.lib.inference.vllm_job import VllmJob + +import importlib.util as _ilu +import pathlib as _pl + +_spec = _ilu.spec_from_file_location("_vllm_shared", _pl.Path(__file__).with_name("_shared.py")) +_mod = _ilu.module_from_spec(_spec) +_spec.loader.exec_module(_mod) +test_print_results_table = _mod.test_print_results_table # exported as a sibling test # noqa: F841 + +log = globals.log + +_FETCH_POLL_COUNT = 80 +_FETCH_POLL_WAIT_S = 30 +_FETCH_PRESENCE_RETRIES = 5 + + +def pytest_generate_tests(metafunc): + """Parametrize test_vllm_inference from the sweep's named-combo + runs selector. + + Runs at collection time (before fixtures exist), so it reads the raw + config_file JSON directly. Validates GoodputSlo and sweep selector against + the same rules the typed loader uses so collection-time and load-time checks + cannot drift. + """ + config_file = metafunc.config.getoption("config_file") + if not config_file or not os.path.isfile(config_file): + return + with open(config_file) as fp: + raw = json.load(fp) + sweep = raw.get("sweep", {}) + combos = sweep.get("sequence_combinations", []) + runs = sweep.get("runs", []) + for combo in combos: + if combo.get("goodput_slo") is not None: + GoodputSlo(**combo["goodput_slo"]) + validate_sweep_selector([c["name"] for c in combos], [r["combo"] for r in runs]) + by_name = {c["name"]: c for c in combos} + cases = [] + ids = [] + for run in runs: + combo = by_name[run["combo"]] + conc = run["concurrency"] + cases.append((combo, conc)) + ids.append(run["combo"] + "-conc" + str(conc)) + if "metric" in metafunc.fixturenames: + if cases: + metric_cases = [] + metric_ids = [] + for (combo, c), cid in zip(cases, ids): + for short, _unit in _METRICS: + metric_cases.append((combo, c, short)) + metric_ids.append(cid + "-" + short) + metafunc.parametrize("seq_combo,concurrency,metric", metric_cases, ids=metric_ids) + elif "seq_combo" in metafunc.fixturenames and "concurrency" in metafunc.fixturenames and cases: + metafunc.parametrize("seq_combo,concurrency", cases, ids=ids) + + +def _du_bytes(orch, path): + """Total bytes under `path` inside the container, or 0 if it doesn't exist yet.""" + out = orch.exec(f"bash -c {shlex.quote(f'du -sb {shlex.quote(path)} 2>/dev/null | cut -f1')}") + total = 0 + for text in (out or {}).values(): + for tok in (text or "").split(): + if tok.isdigit(): + total = max(total, int(tok)) + return total + + +def test_launch_container(orch, variant_config, lifecycle, request): + """Stage 1: launch the container. Asserts it is independently observed running.""" + t = time.monotonic() + ok = orch.setup_containers() + lifecycle.record(request.node.nodeid, "container_launch", time.monotonic() - t) + if not ok: + lifecycle.failed = True + name = orch.get_container_name(orch.container_config, orch.container_config["image"]) + pytest.fail(f"setup_containers() returned False for {name}") + name = orch.get_container_name(orch.container_config, orch.container_config["image"]) + if not orch.verify_containers_running(name): + lifecycle.failed = True + pytest.fail(f"container {name} not running after setup_containers()") + + +def test_setup_sshd(orch, lifecycle, request): + """Stage 2: no-op for vllm — distributed execution uses NCCL/gloo over host network, not MPI/sshd.""" + pytest.skip("vllm uses --distributed-executor-backend mp + NCCL; no inter-container sshd needed") + + +def test_discover_topology(orch, variant_config, lifecycle, request): + """Stage 3: discover IB HCA devices on all nodes. + + Skipped for single-node runs (nnodes=1) since IB HCA selection is not + needed for NCCL_IB_HCA on single-node. + + For distributed runs: + - Runs ibv_devinfo -l on all nodes + - If ib_hca_devices in config is an explicit list, validates it against + the discovered devices (fails loudly if a named device is absent) + - If ib_hca_devices is absent or "auto", uses the full discovered list + - Stores the resolved HCA list in lifecycle.ib_hcas for use per cell + """ + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + + nn = int(variant_config.params.nnodes) + if nn == 1: + lifecycle.ib_hcas = [] + return + + from cvs.lib.utils.ib_discovery import discover_ib_hca_names, validate_ib_hca_preflight + + t = time.monotonic() + try: + discovered = discover_ib_hca_names(orch) + except RuntimeError as e: + lifecycle.failed = True + lifecycle.record(request.node.nodeid, "topology_discovery", time.monotonic() - t) + pytest.fail(str(e)) + + requested = variant_config.roles.server.ib_hca_devices + if requested and requested != "auto": + try: + validate_ib_hca_preflight(discovered, requested) + except RuntimeError as e: + lifecycle.failed = True + lifecycle.record(request.node.nodeid, "topology_discovery", time.monotonic() - t) + pytest.fail(str(e)) + resolved = requested + else: + # "auto" or absent: use whatever the first host reported (symmetry verified above). + resolved = next(iter(discovered.values())) + + lifecycle.ib_hcas = resolved + lifecycle.record(request.node.nodeid, "topology_discovery", time.monotonic() - t) + log.info("test_discover_topology: resolved HCAs=%s", resolved) + + +def test_model_fetch(orch, variant_config, lifecycle, request): + """Stage 4: ensure the model is present in the HF cache (mounted models dir).""" + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + models_dir = variant_config.paths.models_dir + if not models_dir: + pytest.skip("paths.models_dir unset; cannot locate/verify the HF cache") + + remote = getattr(variant_config.model, "remote", 0) + t = time.monotonic() + orch.exec(f"mkdir -p {shlex.quote(models_dir)}") + + if not remote: + final = 0 + for it in range(_FETCH_PRESENCE_RETRIES): + final = _du_bytes(orch, models_dir) + log.info("[fetch presence %d] size=%.1fGB", it, final / 1e9) + if final > 0: + break + time.sleep(_FETCH_POLL_WAIT_S) + else: + fetch = ( + f"HF_HUB_CACHE={shlex.quote(models_dir)} " + f"nohup hf download {shlex.quote(variant_config.model.id)} " + f"> /tmp/hf_fetch.log 2>&1 &" + ) + orch.exec("bash -c " + shlex.quote(fetch)) + + prev = -1 + stable = 0 + final = _du_bytes(orch, models_dir) + for it in range(_FETCH_POLL_COUNT): + cur = _du_bytes(orch, models_dir) + final = cur + log.info("[fetch poll %d] size=%.1fGB", it, cur / 1e9) + if cur > 0 and cur == prev: + stable += 1 + if stable >= 2: + break + else: + stable = 0 + prev = cur + time.sleep(_FETCH_POLL_WAIT_S) + + lifecycle.record(request.node.nodeid, "model_fetch", time.monotonic() - t) + lifecycle.record(request.node.nodeid, "model_size", final / 1e9, "GB") + if final <= 0: + lifecycle.failed = True + pytest.fail(f"no model bytes under {models_dir} after fetch") + + +def test_vllm_inference(orch, variant_config, hf_token, seq_combo, concurrency, inf_res_dict, lifecycle, request): + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + isl = seq_combo["isl"] + osl = seq_combo["osl"] + job = VllmJob( + orch=orch, + variant=variant_config, + hf_token=hf_token, + isl=isl, + osl=osl, + concurrency=concurrency, + num_prompts=variant_config.params.num_prompts, + ib_hcas=getattr(lifecycle, "ib_hcas", []), + goodput_slo=seq_combo.get("goodput_slo"), + client_poll_count=int(variant_config.params.client_poll_count), + ) + + try: + # Reuse the already-running server when this cell needs an identical one + # (cells that differ only in concurrency share a server signature, since + # concurrency is a client-only knob). This skips a full stop + weight + # reload + warmup between such cells. The server keeps serving on the + # same port; only the client args change. + sig = job.server_signature() + if getattr(lifecycle, "live_server_sig", None) == sig: + log.info("reusing running vllm server (same server args); skipping restart") + lifecycle.record(request.node.nodeid, "server_ready", 0.0) + else: + job.stop_server() + job.build_server_cmd() + t = time.monotonic() + job.start_server() + job.wait_ready() + lifecycle.record(request.node.nodeid, "server_ready", time.monotonic() - t) + lifecycle.live_server_sig = sig + job.run_client() + job.wait_client_complete() + results = job.parse_results() + except Exception: + lifecycle.failed = True + # A failed cell may have left the server in a bad state; force the next + # cell to do a clean bringup rather than reuse a possibly-dead server. + lifecycle.live_server_sig = None + raise + + key = ( + variant_config.model.id, + variant_config.gpu_arch, + isl, + osl, + seq_combo.get("name", "default"), + concurrency, + ) + inf_res_dict[key] = results + + +def test_metric(seq_combo, concurrency, metric, inf_res_dict, variant_config, lifecycle, request): + """One pytest test (= one HTML row) per perf metric per cell.""" + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + isl = seq_combo["isl"] + osl = seq_combo["osl"] + key = ( + variant_config.model.id, + variant_config.gpu_arch, + isl, + osl, + seq_combo.get("name", "default"), + concurrency, + ) + if key not in inf_res_dict: + pytest.skip(f"no recorded results for cell {key!r} (inference did not run)") + host_dict = inf_res_dict[key] + _host, actuals = next(iter(host_dict.items())) + full = "client." + metric + value = actuals.get(full) + unit = _METRIC_UNITS.get(metric, "-") + request.node.user_properties.append(("metric_value", value)) + request.node.user_properties.append(("metric_unit", unit)) + + if not variant_config.enforce_thresholds: + return + cell = variant_config.cell_key(isl, osl, concurrency) + spec = (variant_config.thresholds.get(cell) or {}).get(full) + if spec is None: + return + evaluate_all(actuals, {full: spec}) + + +def test_teardown(orch, lifecycle, request): + """Final stage: explicit container teardown, timed, asserting it is gone.""" + name = orch.get_container_name(orch.container_config, orch.container_config["image"]) + t = time.monotonic() + orch.teardown_containers() + lifecycle.record(request.node.nodeid, "teardown", time.monotonic() - t) + if orch.verify_containers_running(name): + pytest.fail(f"container {name} still running after teardown_containers()") + lifecycle.torn_down = True diff --git a/cvs/tests/inference/vllm/vllm_deepseek31_685b_single.py b/cvs/tests/inference/vllm/vllm_deepseek31_685b_single.py deleted file mode 100644 index 429f5e889..000000000 --- a/cvs/tests/inference/vllm/vllm_deepseek31_685b_single.py +++ /dev/null @@ -1,453 +0,0 @@ -''' -Copyright 2025 Advanced Micro Devices, Inc. -All rights reserved. This notice is intended as a precaution against inadvertent publication and does not imply publication or any waiver of confidentiality. -The year included in the foregoing notice is the year of creation of the work. -All code contained here is Property of Advanced Micro Devices, Inc. -''' - -import pytest - -import re -import os -import time -import json -from pprint import pprint -from tabulate import tabulate - - -from cvs.lib.parallel_ssh_lib import * -from cvs.lib.utils_lib import * -from cvs.lib import docker_lib -from cvs.lib.inference.vllm import VllmJob -from cvs.lib import globals - -log = globals.log - -# Model name for this test suite -MODEL_NAME = "deepseek-v31" - -inf_res_dict = {} - - -# Importing additional cmd line args to script .. -@pytest.fixture(scope="module") -def cluster_file(pytestconfig): - """ - Retrieve the --cluster_file CLI option provided to pytest. - - Args: - pytestconfig: Built-in pytest fixture exposing command-line options. - - Returns: - str: Path to the cluster JSON file specified via --cluster_file. - - Notes: - - Ensure your pytest.ini or CLI includes: --cluster_file=/path/to/cluster.json - - Use module scope so the value is resolved once per test module. - """ - return pytestconfig.getoption("cluster_file") - - -@pytest.fixture(scope="module") -def training_config_file(pytestconfig): - """ - Retrieve the --config_file CLI option provided to pytest. - - Args: - pytestconfig: Built-in pytest fixture exposing command-line options. - - Returns: - str: Path to the training config JSON file specified via --config_file. - - Notes: - - Ensure your pytest.ini or CLI includes: --config_file=/path/to/training_config.json - - Module scope avoids re-fetching the option across tests in this module. - """ - return pytestconfig.getoption("config_file") - - -# Importing the cluster and cofig files to script to access node, switch, test config params -@pytest.fixture(scope="module") -def cluster_dict(cluster_file): - """ - Load the entire cluster configuration from the provided JSON file. - - Args: - cluster_file (str): Path to the cluster JSON file. - - Returns: - dict: Parsed JSON representing the cluster (nodes, credentials, etc.). - - Notes: - - Logs the loaded structure for visibility; consider using log.debug if verbose. - """ - with open(cluster_file) as json_file: - cluster_dict = json.load(json_file) - - # Resolve path placeholders like {user-id} in cluster config - cluster_dict = resolve_cluster_config_placeholders(cluster_dict) - log.info("%s", cluster_dict) - return cluster_dict - - -@pytest.fixture(scope="module") -def inference_dict(training_config_file, cluster_dict): - with open(training_config_file) as json_file: - inference_dict_t = json.load(json_file) - inference_dict = inference_dict_t['config'] - - # Resolve path placeholders like {user-id}, {home-mount-dir}, etc. - inference_dict = resolve_test_config_placeholders(inference_dict, cluster_dict) - return inference_dict - - -@pytest.fixture(scope="module") -def benchmark_params_dict(training_config_file, cluster_dict): - with open(training_config_file) as json_file: - inference_dict_t = json.load(json_file) - benchmark_params_dict = inference_dict_t['benchmark_params'] - - # Resolve path placeholders like {user-id}, {home-mount-dir}, etc. - benchmark_params_dict = resolve_test_config_placeholders(benchmark_params_dict, cluster_dict) - - log.info("%s", benchmark_params_dict) - return benchmark_params_dict - - -def pytest_generate_tests(metafunc): - """ - Dynamically parametrize inference tests based on sequence combinations and concurrency levels - for the DeepSeek-V3.1 model. - - Behavior: - - Reads the config file path from pytest's --config_file option. - - Loads the JSON and extracts deepseek-v31 model configuration. - - Extracts sequence_combinations (ISL/OSL pairs) and concurrency_levels. - - Creates test cases for each sequence combination × concurrency level. - - Test Matrix: - - Test IDs: "combination_name-concX" (e.g., "balanced-conc16") - - Notes: - - If no config_file is provided, the hook returns without parametrizing. - - Each combination gets a separate test case with clear ID. - """ - config_file = metafunc.config.getoption("config_file") - if not config_file or not os.path.exists(config_file): - log.warning(f'Warning: Missing or invalid config file {config_file}') - return - - with open(config_file) as fp: - cfg = json.load(fp) - - # Extract deepseek-v31 model config (now directly under benchmark_params, no single_node nesting) - benchmark_params = cfg.get("benchmark_params", {}) - model_config = benchmark_params.get(MODEL_NAME, {}) - - if not model_config: - log.warning(f'Warning: Model {MODEL_NAME} not found in config') - return - - # Build test parameters: list of (seq_combo_dict, concurrency, test_id) - test_params = [] - - # Check if model uses sequence_combinations or legacy ISL/OSL - seq_combos = model_config.get("sequence_combinations", []) - - if seq_combos: - # New format: multiple combinations per model - for combo in seq_combos: - combo_name = combo.get("name", f"isl{combo['isl']}_osl{combo['osl']}") - - # Check if model has concurrency_levels array or single max_concurrency - conc_levels = model_config.get("concurrency_levels", []) - if conc_levels: - # Parametrize across concurrency levels - for conc in conc_levels: - test_id = f"{combo_name}-conc{conc}" - test_params.append((combo, conc, test_id)) - else: - # Backward compatibility: use max_concurrency as single value - max_conc = int(model_config.get("max_concurrency", "64")) - test_id = f"{combo_name}" - test_params.append((combo, max_conc, test_id)) - else: - # Legacy format: single ISL/OSL values - isl = model_config.get("input_sequence_length", "1024") - osl = model_config.get("output_sequence_length", "1024") - combo = {"isl": isl, "osl": osl, "name": "default"} - - conc_levels = model_config.get("concurrency_levels", []) - if conc_levels: - for conc in conc_levels: - test_id = f"conc{conc}" - test_params.append((combo, conc, test_id)) - else: - max_conc = int(model_config.get("max_concurrency", "64")) - test_params.append((combo, max_conc, "default")) - - # Parametrize if test uses these fixtures - if "seq_combo" in metafunc.fixturenames and "concurrency" in metafunc.fixturenames: - if test_params: - combos, concs, ids = zip(*test_params) - metafunc.parametrize("seq_combo,concurrency", list(zip(combos, concs)), ids=ids) - - -@pytest.fixture(scope="module") -def hf_token(inference_dict): - """ - Load the Hugging Face access token from the file path specified in the training config. - - Args: - inference_dict (dict): Training configuration dict that includes: - - 'hf_token_file': Path to the file containing the HF token. - - Returns: - str: The HF token string read from the file. - - Behavior: - - Reads the token from inference_dict['hf_token_file'] (already resolved for placeholders). - - Strips the trailing newline from the token. - """ - hf_token_file = inference_dict['hf_token_file'] - try: - with open(hf_token_file, 'r') as fp: - hf_token = fp.read().rstrip("\n") - except FileNotFoundError: - log.error(f"Error: The file '{hf_token_file}' was not found.") - raise - except Exception as e: - log.error(f"An error occurred: {e}") - raise - return hf_token - - -@pytest.fixture(scope="module") -def s_phdl(cluster_dict): - """ - Create and return a parallel SSH handle for all cluster nodes (server). - - Args: - cluster_dict (dict): Cluster configuration loaded by another fixture. Expected keys: - - 'node_dict': dict of node_name -> node_details (used to derive the node list) - - 'username': SSH username for connecting to nodes - - 'priv_key_file': path to the SSH private key file - - Returns: - Pssh: An initialized Pssh handle for issuing commands across all nodes. - - Behavior: - - Prints the full cluster_dict for quick debugging (consider switching to log.debug to reduce noise). - - Collects all node names from cluster_dict['node_dict'] and constructs a Pssh handle. - - Notes: - - This fixture has module scope, so a single connection handle is reused for all tests in the module. - """ - log.info("%s", cluster_dict) - env_vars = cluster_dict.get("env_vars") - node_list = list(cluster_dict['node_dict'].keys()) - s_phdl = Pssh(log, node_list, user=cluster_dict['username'], pkey=cluster_dict['priv_key_file'], env_vars=env_vars) - return s_phdl - - -@pytest.fixture(scope="module") -def c_phdl(cluster_dict): - """ - Create and return a parallel SSH handle for all cluster nodes (client). - - Args: - cluster_dict (dict): Cluster configuration loaded by another fixture. Expected keys: - - 'node_dict': dict of node_name -> node_details (used to derive the node list) - - 'username': SSH username for connecting to nodes - - 'priv_key_file': path to the SSH private key file - - Returns: - Pssh: An initialized Pssh handle for issuing commands across all nodes. - - Behavior: - - Prints the full cluster_dict for quick debugging (consider switching to log.debug to reduce noise). - - Collects all node names from cluster_dict['node_dict'] and constructs a Pssh handle. - - Notes: - - This fixture has module scope, so a single connection handle is reused for all tests in the module. - """ - log.info("%s", cluster_dict) - env_vars = cluster_dict.get("env_vars") - node_list = list(cluster_dict['node_dict'].keys()) - c_phdl = Pssh(log, node_list, user=cluster_dict['username'], pkey=cluster_dict['priv_key_file'], env_vars=env_vars) - return c_phdl - - -def test_cleanup_stale_containers(s_phdl, inference_dict): - """ - Pytest: Clean up potentially stale Docker containers and volumes before tests. - - Args: - s_phdl: Parallel SSH/process handle used by docker_lib to run commands on nodes. - inference_dict (dict): Training configuration dict that includes: - - 'container_name': Name of the container to be killed if running. - - Behavior: - - Kills the specific container identified by inference_dict['container_name']. - - Deletes all containers and volumes on the target nodes (broad cleanup). - - Notes: - - This performs a broad cleanup via delete_all_containers_and_volumes; ensure the - test environment is isolated so this doesn't remove unrelated containers/volumes. - - Consider narrowing cleanup scope if other workloads may be present on the hosts. - """ - - container_name = inference_dict['container_name'] - docker_lib.kill_docker_container(s_phdl, container_name) - docker_lib.delete_all_containers_and_volumes(s_phdl) - - -def test_launch_inference_containers(s_phdl, inference_dict, benchmark_params_dict): - """ - Launch vLLM inference containers on all nodes. - - Note: Container image can be model-specific or use global default. - """ - - log.info(f'Testcase launch vLLM containers for {MODEL_NAME}') - globals.error_list = [] - container_name = inference_dict['container_name'] - - # Get model-specific container image or use global default - container_image = benchmark_params_dict.get(MODEL_NAME, {}).get( - 'container_image', inference_dict['container_image'] - ) - - # Launch the containers .. - docker_lib.launch_docker_container( - s_phdl, - container_name, - container_image, - inference_dict['container_config']['device_list'], - inference_dict['container_config']['volume_dict'], - inference_dict['container_config']['env_dict'], - shm_size='16G', - timeout=60 * 20, - ) - # ADD verifications .. - time.sleep(30) - log.info('Verify if the containers have been launched properly') - out_dict = s_phdl.exec('docker ps') - for node in out_dict.keys(): - if not re.search(f'{container_name}', out_dict[node], re.I): - fail_test(f'Failed to launch container on node {node}') - update_test_result() - - -def test_vllm_inference(c_phdl, s_phdl, inference_dict, benchmark_params_dict, hf_token, seq_combo, concurrency): - """ - Test vLLM inference for DeepSeek-V3.1 with specific sequence combination and concurrency level. - - This test is parametrized via pytest_generate_tests to run once per: - - Sequence combination (ISL/OSL pair) defined in model's sequence_combinations - - Concurrency level defined in model's concurrency_levels - - The factory will automatically create the correct VllmJob instance with - the model-specific container image and parameters. - - Args: - seq_combo: Dict with 'isl', 'osl', 'name' keys for this test iteration - concurrency: Integer concurrency level for this test iteration - """ - # Since this is a per-GPU-type config file (mi355x), gpu_type is implicit - gpu_type = "mi355x" - - log.info( - f"Starting inference test for model: {MODEL_NAME}, GPU: {gpu_type}, combination: {seq_combo['name']} (ISL={seq_combo['isl']}, OSL={seq_combo['osl']}), concurrency: {concurrency}" - ) - globals.error_list = [] - - # Override ISL/OSL and concurrency in benchmark_params for this specific test iteration - # Config is now fully flattened, so access directly under benchmark_params - model_params = benchmark_params_dict[MODEL_NAME] - model_params['input_sequence_length'] = seq_combo['isl'] - model_params['output_sequence_length'] = seq_combo['osl'] - model_params['max_concurrency'] = str(concurrency) - - # Calculate num_prompts based on OSL (matching recipe logic) - osl = int(seq_combo['osl']) - if osl == 8192: - model_params['num_prompts'] = str(concurrency * 20) - else: - model_params['num_prompts'] = str(concurrency * 50) - - # Create VllmJob instance - vllm_job = VllmJob( - c_phdl=c_phdl, - s_phdl=s_phdl, - model_name=MODEL_NAME, - inference_config_dict=inference_dict, - benchmark_params_dict=benchmark_params_dict, - hf_token=hf_token, - gpu_type=gpu_type, - distributed_inference=False, - ) - - # Stop any existing server process for clean state - vllm_job.stop_server() - - # Build and start server with current test parameters - vllm_job.build_server_inference_job_cmd() - vllm_job.start_inference_server_job() - - # Run benchmark client - vllm_job.start_inference_client_job() - # res_dict will have status, results from base inference class - # - {"status": "success", "results": self.inference_result_dict} - res_dict = vllm_job.poll_for_inference_completion() - res_index = (MODEL_NAME, gpu_type, seq_combo['isl'], seq_combo['osl'], seq_combo['name'], concurrency) - inf_res_dict[res_index] = res_dict - vllm_job.verify_inference_results() - update_test_result() - - log.info( - f"Completed inference test for model: {MODEL_NAME}, GPU: {gpu_type}, combination: {seq_combo['name']}, concurrency: {concurrency}" - ) - - -def test_print_results_table(): - globals.error_list = [] - log.info("%s", inf_res_dict) - pprint(inf_res_dict, depth=3) - rows = [] - headers = [ - "Model", - "GPU", - "ISL", - "OSL", - "Policy", - "Concurrency", - "Host", - "Req/s", - "Total tok/s", - "Mean TTFT (ms)", - "Mean TPOT (ms)", - "P99 ITL (ms)", - ] - - for (model, gpu, isl, osl, policy, concurrency), entry in inf_res_dict.items(): - for host, m in entry["results"].items(): - rows.append( - [ - model, - gpu, - isl, - osl, - policy, - concurrency, - host, - m["successful_requests"], - m["total_throughput_per_sec"], - m["mean_ttft_ms"], - m["mean_tpot_ms"], - m["p99_itl_ms"], - ] - ) - - log.info(tabulate(rows, headers=headers, tablefmt="github")) - update_test_result() diff --git a/cvs/tests/inference/vllm/vllm_gpt_oss_120b_single.py b/cvs/tests/inference/vllm/vllm_gpt_oss_120b_single.py deleted file mode 100644 index 5c6c8e5f6..000000000 --- a/cvs/tests/inference/vllm/vllm_gpt_oss_120b_single.py +++ /dev/null @@ -1,451 +0,0 @@ -''' -Copyright 2025 Advanced Micro Devices, Inc. -All rights reserved. This notice is intended as a precaution against inadvertent publication and does not imply publication or any waiver of confidentiality. -The year included in the foregoing notice is the year of creation of the work. -All code contained here is Property of Advanced Micro Devices, Inc. -''' - -import pytest - -import re -import os -import time -import json -from pprint import pprint -from tabulate import tabulate - -from cvs.lib.parallel_ssh_lib import * -from cvs.lib.utils_lib import * -from cvs.lib import docker_lib -from cvs.lib.inference.vllm import VllmJob -from cvs.lib import globals - -log = globals.log - -# Model name for this test suite -MODEL_NAME = "gpt-oss-120b" - -inf_res_dict = {} - - -# Importing additional cmd line args to script .. -@pytest.fixture(scope="module") -def cluster_file(pytestconfig): - """ - Retrieve the --cluster_file CLI option provided to pytest. - - Args: - pytestconfig: Built-in pytest fixture exposing command-line options. - - Returns: - str: Path to the cluster JSON file specified via --cluster_file. - - Notes: - - Ensure your pytest.ini or CLI includes: --cluster_file=/path/to/cluster.json - - Use module scope so the value is resolved once per test module. - """ - return pytestconfig.getoption("cluster_file") - - -@pytest.fixture(scope="module") -def training_config_file(pytestconfig): - """ - Retrieve the --config_file CLI option provided to pytest. - - Args: - pytestconfig: Built-in pytest fixture exposing command-line options. - - Returns: - str: Path to the training config JSON file specified via --config_file. - - Notes: - - Ensure your pytest.ini or CLI includes: --config_file=/path/to/training_config.json - - Module scope avoids re-fetching the option across tests in this module. - """ - return pytestconfig.getoption("config_file") - - -# Importing the cluster and cofig files to script to access node, switch, test config params -@pytest.fixture(scope="module") -def cluster_dict(cluster_file): - """ - Load the entire cluster configuration from the provided JSON file. - - Args: - cluster_file (str): Path to the cluster JSON file. - - Returns: - dict: Parsed JSON representing the cluster (nodes, credentials, etc.). - - Notes: - - Logs the loaded structure for visibility; consider using log.debug if verbose. - """ - with open(cluster_file) as json_file: - cluster_dict = json.load(json_file) - - # Resolve path placeholders like {user-id} in cluster config - cluster_dict = resolve_cluster_config_placeholders(cluster_dict) - log.info("%s", cluster_dict) - return cluster_dict - - -@pytest.fixture(scope="module") -def inference_dict(training_config_file, cluster_dict): - with open(training_config_file) as json_file: - inference_dict_t = json.load(json_file) - inference_dict = inference_dict_t['config'] - - # Resolve path placeholders like {user-id}, {home-mount-dir}, etc. - inference_dict = resolve_test_config_placeholders(inference_dict, cluster_dict) - return inference_dict - - -@pytest.fixture(scope="module") -def benchmark_params_dict(training_config_file, cluster_dict): - with open(training_config_file) as json_file: - inference_dict_t = json.load(json_file) - benchmark_params_dict = inference_dict_t['benchmark_params'] - - # Resolve path placeholders like {user-id}, {home-mount-dir}, etc. - benchmark_params_dict = resolve_test_config_placeholders(benchmark_params_dict, cluster_dict) - - log.info("%s", benchmark_params_dict) - return benchmark_params_dict - - -def pytest_generate_tests(metafunc): - """ - Dynamically parametrize inference tests based on sequence combinations and concurrency levels - for the GPT-OSS-120B model. - - Behavior: - - Reads the config file path from pytest's --config_file option. - - Loads the JSON and extracts gpt-oss-120b model configuration. - - Extracts sequence_combinations (ISL/OSL pairs) and concurrency_levels. - - Creates test cases for each sequence combination × concurrency level. - - Test Matrix: - - Test IDs: "combination_name-concX" (e.g., "balanced-conc16") - - Notes: - - If no config_file is provided, the hook returns without parametrizing. - - Each combination gets a separate test case with clear ID. - """ - config_file = metafunc.config.getoption("config_file") - if not config_file or not os.path.exists(config_file): - log.warning(f'Warning: Missing or invalid config file {config_file}') - return - - with open(config_file) as fp: - cfg = json.load(fp) - - # Extract gpt-oss-120b model config (now directly under benchmark_params, no single_node nesting) - benchmark_params = cfg.get("benchmark_params", {}) - model_config = benchmark_params.get(MODEL_NAME, {}) - - if not model_config: - log.warning(f'Warning: Model {MODEL_NAME} not found in config') - return - - # Build test parameters: list of (seq_combo_dict, concurrency, test_id) - test_params = [] - - # Check if model uses sequence_combinations or legacy ISL/OSL - seq_combos = model_config.get("sequence_combinations", []) - - if seq_combos: - # New format: multiple combinations per model - for combo in seq_combos: - combo_name = combo.get("name", f"isl{combo['isl']}_osl{combo['osl']}") - - # Check if model has concurrency_levels array or single max_concurrency - conc_levels = model_config.get("concurrency_levels", []) - if conc_levels: - # Parametrize across concurrency levels - for conc in conc_levels: - test_id = f"{combo_name}-conc{conc}" - test_params.append((combo, conc, test_id)) - else: - # Backward compatibility: use max_concurrency as single value - max_conc = int(model_config.get("max_concurrency", "64")) - test_id = f"{combo_name}" - test_params.append((combo, max_conc, test_id)) - else: - # Legacy format: single ISL/OSL values - isl = model_config.get("input_sequence_length", "1024") - osl = model_config.get("output_sequence_length", "1024") - combo = {"isl": isl, "osl": osl, "name": "default"} - - conc_levels = model_config.get("concurrency_levels", []) - if conc_levels: - for conc in conc_levels: - test_id = f"conc{conc}" - test_params.append((combo, conc, test_id)) - else: - max_conc = int(model_config.get("max_concurrency", "64")) - test_params.append((combo, max_conc, "default")) - - # Parametrize if test uses these fixtures - if "seq_combo" in metafunc.fixturenames and "concurrency" in metafunc.fixturenames: - if test_params: - combos, concs, ids = zip(*test_params) - metafunc.parametrize("seq_combo,concurrency", list(zip(combos, concs)), ids=ids) - - -@pytest.fixture(scope="module") -def hf_token(inference_dict): - """ - Load the Hugging Face access token from the file path specified in the training config. - - Args: - inference_dict (dict): Training configuration dict that includes: - - 'hf_token_file': Path to the file containing the HF token. - - Returns: - str: The HF token string read from the file. - - Behavior: - - Reads the token from inference_dict['hf_token_file'] (already resolved for placeholders). - - Strips the trailing newline from the token. - """ - hf_token_file = inference_dict['hf_token_file'] - try: - with open(hf_token_file, 'r') as fp: - hf_token = fp.read().rstrip("\n") - except FileNotFoundError: - log.error(f"Error: The file '{hf_token_file}' was not found.") - raise - except Exception as e: - log.error(f"An error occurred: {e}") - raise - return hf_token - - -@pytest.fixture(scope="module") -def s_phdl(cluster_dict): - """ - Create and return a parallel SSH handle for all cluster nodes (server). - - Args: - cluster_dict (dict): Cluster configuration loaded by another fixture. Expected keys: - - 'node_dict': dict of node_name -> node_details (used to derive the node list) - - 'username': SSH username for connecting to nodes - - 'priv_key_file': path to the SSH private key file - - Returns: - Pssh: An initialized Pssh handle for issuing commands across all nodes. - - Behavior: - - Prints the full cluster_dict for quick debugging (consider switching to log.debug to reduce noise). - - Collects all node names from cluster_dict['node_dict'] and constructs a Pssh handle. - - Notes: - - This fixture has module scope, so a single connection handle is reused for all tests in the module. - """ - log.info("%s", cluster_dict) - env_vars = cluster_dict.get("env_vars") - node_list = list(cluster_dict['node_dict'].keys()) - s_phdl = Pssh(log, node_list, user=cluster_dict['username'], pkey=cluster_dict['priv_key_file'], env_vars=env_vars) - return s_phdl - - -@pytest.fixture(scope="module") -def c_phdl(cluster_dict): - """ - Create and return a parallel SSH handle for all cluster nodes (client). - - Args: - cluster_dict (dict): Cluster configuration loaded by another fixture. Expected keys: - - 'node_dict': dict of node_name -> node_details (used to derive the node list) - - 'username': SSH username for connecting to nodes - - 'priv_key_file': path to the SSH private key file - - Returns: - Pssh: An initialized Pssh handle for issuing commands across all nodes. - - Behavior: - - Prints the full cluster_dict for quick debugging (consider switching to log.debug to reduce noise). - - Collects all node names from cluster_dict['node_dict'] and constructs a Pssh handle. - - Notes: - - This fixture has module scope, so a single connection handle is reused for all tests in the module. - """ - log.info("%s", cluster_dict) - env_vars = cluster_dict.get("env_vars") - node_list = list(cluster_dict['node_dict'].keys()) - c_phdl = Pssh(log, node_list, user=cluster_dict['username'], pkey=cluster_dict['priv_key_file'], env_vars=env_vars) - return c_phdl - - -def test_cleanup_stale_containers(s_phdl, inference_dict): - """ - Pytest: Clean up potentially stale Docker containers and volumes before tests. - - Args: - s_phdl: Parallel SSH/process handle used by docker_lib to run commands on nodes. - inference_dict (dict): Training configuration dict that includes: - - 'container_name': Name of the container to be killed if running. - - Behavior: - - Kills the specific container identified by inference_dict['container_name']. - - Deletes all containers and volumes on the target nodes (broad cleanup). - - Notes: - - This performs a broad cleanup via delete_all_containers_and_volumes; ensure the - test environment is isolated so this doesn't remove unrelated containers/volumes. - - Consider narrowing cleanup scope if other workloads may be present on the hosts. - """ - - container_name = inference_dict['container_name'] - docker_lib.kill_docker_container(s_phdl, container_name) - docker_lib.delete_all_containers_and_volumes(s_phdl) - - -def test_launch_inference_containers(s_phdl, inference_dict, benchmark_params_dict): - """ - Launch vLLM inference containers on all nodes. - - Note: Container image can be model-specific or use global default. - """ - - log.info(f'Testcase launch vLLM containers for {MODEL_NAME}') - globals.error_list = [] - container_name = inference_dict['container_name'] - - # Get model-specific container image or use global default - container_image = benchmark_params_dict.get(MODEL_NAME, {}).get( - 'container_image', inference_dict['container_image'] - ) - - # Launch the containers .. - docker_lib.launch_docker_container( - s_phdl, - container_name, - container_image, - inference_dict['container_config']['device_list'], - inference_dict['container_config']['volume_dict'], - inference_dict['container_config']['env_dict'], - shm_size='16G', - timeout=60 * 20, - ) - # ADD verifications .. - time.sleep(30) - log.info('Verify if the containers have been launched properly') - out_dict = s_phdl.exec('docker ps') - for node in out_dict.keys(): - if not re.search(f'{container_name}', out_dict[node], re.I): - fail_test(f'Failed to launch container on node {node}') - update_test_result() - - -def test_vllm_inference(c_phdl, s_phdl, inference_dict, benchmark_params_dict, hf_token, seq_combo, concurrency): - """ - Test vLLM inference for GPT-OSS-120B with specific sequence combination and concurrency level. - - This test is parametrized via pytest_generate_tests to run once per: - - Sequence combination (ISL/OSL pair) defined in model's sequence_combinations - - Concurrency level defined in model's concurrency_levels - - The factory will automatically create the correct VllmJob instance with - the model-specific container image and parameters. - - Args: - seq_combo: Dict with 'isl', 'osl', 'name' keys for this test iteration - concurrency: Integer concurrency level for this test iteration - """ - # Since this is a per-GPU-type config file (mi355x), gpu_type is implicit - gpu_type = "mi355x" - - log.info( - f"Starting inference test for model: {MODEL_NAME}, GPU: {gpu_type}, combination: {seq_combo['name']} (ISL={seq_combo['isl']}, OSL={seq_combo['osl']}), concurrency: {concurrency}" - ) - globals.error_list = [] - - # Override ISL/OSL and concurrency in benchmark_params for this specific test iteration - # Config is now fully flattened, so access directly under benchmark_params - model_params = benchmark_params_dict[MODEL_NAME] - model_params['input_sequence_length'] = seq_combo['isl'] - model_params['output_sequence_length'] = seq_combo['osl'] - model_params['max_concurrency'] = str(concurrency) - - # Calculate num_prompts based on OSL (matching recipe logic) - osl = int(seq_combo['osl']) - if osl == 8192: - model_params['num_prompts'] = str(concurrency * 20) - else: - model_params['num_prompts'] = str(concurrency * 50) - - # Create VllmJob instance - vllm_job = VllmJob( - c_phdl=c_phdl, - s_phdl=s_phdl, - model_name=MODEL_NAME, - inference_config_dict=inference_dict, - benchmark_params_dict=benchmark_params_dict, - hf_token=hf_token, - gpu_type=gpu_type, - distributed_inference=False, - ) - - # Stop any existing server process for clean state - vllm_job.stop_server() - - # Build and start server with current test parameters - vllm_job.build_server_inference_job_cmd() - vllm_job.start_inference_server_job() - - # Run benchmark client - vllm_job.start_inference_client_job() - # res_dict will have status, results from base inference class - # - {"status": "success", "results": self.inference_result_dict} - res_dict = vllm_job.poll_for_inference_completion() - res_index = (MODEL_NAME, gpu_type, seq_combo['isl'], seq_combo['osl'], seq_combo['name'], concurrency) - inf_res_dict[res_index] = res_dict - vllm_job.verify_inference_results() - update_test_result() - - log.info( - f"Completed inference test for model: {MODEL_NAME}, GPU: {gpu_type}, combination: {seq_combo['name']}, concurrency: {concurrency}" - ) - - -def test_print_results_table(): - globals.error_list = [] - pprint(inf_res_dict, depth=3) - rows = [] - headers = [ - "Model", - "GPU", - "ISL", - "OSL", - "Policy", - "Concurrency", - "Host", - "Req/s", - "Total tok/s", - "Mean TTFT (ms)", - "Mean TPOT (ms)", - "P99 ITL (ms)", - ] - - for (model, gpu, isl, osl, policy, concurrency), entry in inf_res_dict.items(): - for host, m in entry["results"].items(): - rows.append( - [ - model, - gpu, - isl, - osl, - policy, - concurrency, - host, - m["successful_requests"], - m["total_throughput_per_sec"], - m["mean_ttft_ms"], - m["mean_tpot_ms"], - m["p99_itl_ms"], - ] - ) - - log.info(tabulate(rows, headers=headers, tablefmt="github")) - update_test_result() diff --git a/cvs/tests/inference/vllm/vllm_qwen3_235b_single.py b/cvs/tests/inference/vllm/vllm_qwen3_235b_single.py deleted file mode 100644 index f46c44c10..000000000 --- a/cvs/tests/inference/vllm/vllm_qwen3_235b_single.py +++ /dev/null @@ -1,449 +0,0 @@ -''' -Copyright 2025 Advanced Micro Devices, Inc. -All rights reserved. This notice is intended as a precaution against inadvertent publication and does not imply publication or any waiver of confidentiality. -The year included in the foregoing notice is the year of creation of the work. -All code contained here is Property of Advanced Micro Devices, Inc. -''' - -import pytest - -import re -import os -import time -import json -from pprint import pprint -from tabulate import tabulate - -from cvs.lib.parallel_ssh_lib import * -from cvs.lib.utils_lib import * -from cvs.lib import docker_lib -from cvs.lib.inference.vllm import VllmJob -from cvs.lib import globals - -log = globals.log - -# Model name for this test suite -MODEL_NAME = "qwen3-235b" - -inf_res_dict = {} - - -# Importing additional cmd line args to script .. -@pytest.fixture(scope="module") -def cluster_file(pytestconfig): - """ - Retrieve the --cluster_file CLI option provided to pytest. - - Args: - pytestconfig: Built-in pytest fixture exposing command-line options. - - Returns: - str: Path to the cluster JSON file specified via --cluster_file. - - Notes: - - Ensure your pytest.ini or CLI includes: --cluster_file=/path/to/cluster.json - - Use module scope so the value is resolved once per test module. - """ - return pytestconfig.getoption("cluster_file") - - -@pytest.fixture(scope="module") -def training_config_file(pytestconfig): - """ - Retrieve the --config_file CLI option provided to pytest. - - Args: - pytestconfig: Built-in pytest fixture exposing command-line options. - - Returns: - str: Path to the training config JSON file specified via --config_file. - - Notes: - - Ensure your pytest.ini or CLI includes: --config_file=/path/to/training_config.json - - Module scope avoids re-fetching the option across tests in this module. - """ - return pytestconfig.getoption("config_file") - - -# Importing the cluster and cofig files to script to access node, switch, test config params -@pytest.fixture(scope="module") -def cluster_dict(cluster_file): - """ - Load the entire cluster configuration from the provided JSON file. - - Args: - cluster_file (str): Path to the cluster JSON file. - - Returns: - dict: Parsed JSON representing the cluster (nodes, credentials, etc.). - - Notes: - - Logs the loaded structure for visibility; consider using log.debug if verbose. - """ - with open(cluster_file) as json_file: - cluster_dict = json.load(json_file) - - # Resolve path placeholders like {user-id} in cluster config - cluster_dict = resolve_cluster_config_placeholders(cluster_dict) - log.info("%s", cluster_dict) - return cluster_dict - - -@pytest.fixture(scope="module") -def inference_dict(training_config_file, cluster_dict): - with open(training_config_file) as json_file: - inference_dict_t = json.load(json_file) - inference_dict = inference_dict_t['config'] - - # Resolve path placeholders like {user-id}, {home-mount-dir}, etc. - inference_dict = resolve_test_config_placeholders(inference_dict, cluster_dict) - return inference_dict - - -@pytest.fixture(scope="module") -def benchmark_params_dict(training_config_file, cluster_dict): - with open(training_config_file) as json_file: - inference_dict_t = json.load(json_file) - benchmark_params_dict = inference_dict_t['benchmark_params'] - - # Resolve path placeholders like {user-id}, {home-mount-dir}, etc. - benchmark_params_dict = resolve_test_config_placeholders(benchmark_params_dict, cluster_dict) - - log.info("%s", benchmark_params_dict) - return benchmark_params_dict - - -def pytest_generate_tests(metafunc): - """ - Dynamically parametrize inference tests based on sequence combinations and concurrency levels - for the Qwen3-235B model. - - Behavior: - - Reads the config file path from pytest's --config_file option. - - Loads the JSON and extracts qwen3-235b model configuration. - - Extracts sequence_combinations (ISL/OSL pairs) and concurrency_levels. - - Creates test cases for each sequence combination × concurrency level. - - Test Matrix: - - Test IDs: "combination_name-concX" (e.g., "balanced-conc16") - - Notes: - - If no config_file is provided, the hook returns without parametrizing. - - Each combination gets a separate test case with clear ID. - """ - config_file = metafunc.config.getoption("config_file") - if not config_file or not os.path.exists(config_file): - log.warning(f'Warning: Missing or invalid config file {config_file}') - return - - with open(config_file) as fp: - cfg = json.load(fp) - - # Extract qwen3-235b model config (now directly under benchmark_params, no single_node nesting) - benchmark_params = cfg.get("benchmark_params", {}) - model_config = benchmark_params.get(MODEL_NAME, {}) - - if not model_config: - log.warning(f'Warning: Model {MODEL_NAME} not found in config') - return - - # Build test parameters: list of (seq_combo_dict, concurrency, test_id) - test_params = [] - - # Check if model uses sequence_combinations or legacy ISL/OSL - seq_combos = model_config.get("sequence_combinations", []) - - if seq_combos: - # New format: multiple combinations per model - for combo in seq_combos: - combo_name = combo.get("name", f"isl{combo['isl']}_osl{combo['osl']}") - - # Check if model has concurrency_levels array or single max_concurrency - conc_levels = model_config.get("concurrency_levels", []) - if conc_levels: - # Parametrize across concurrency levels - for conc in conc_levels: - test_id = f"{combo_name}-conc{conc}" - test_params.append((combo, conc, test_id)) - else: - # Backward compatibility: use max_concurrency as single value - max_conc = int(model_config.get("max_concurrency", "64")) - test_id = f"{combo_name}" - test_params.append((combo, max_conc, test_id)) - else: - # Legacy format: single ISL/OSL values - isl = model_config.get("input_sequence_length", "1024") - osl = model_config.get("output_sequence_length", "1024") - combo = {"isl": isl, "osl": osl, "name": "default"} - - conc_levels = model_config.get("concurrency_levels", []) - if conc_levels: - for conc in conc_levels: - test_id = f"conc{conc}" - test_params.append((combo, conc, test_id)) - else: - max_conc = int(model_config.get("max_concurrency", "64")) - test_params.append((combo, max_conc, "default")) - - # Parametrize if test uses these fixtures - if "seq_combo" in metafunc.fixturenames and "concurrency" in metafunc.fixturenames: - if test_params: - combos, concs, ids = zip(*test_params) - metafunc.parametrize("seq_combo,concurrency", list(zip(combos, concs)), ids=ids) - - -@pytest.fixture(scope="module") -def hf_token(inference_dict): - """ - Load the Hugging Face access token from the file path specified in the training config. - - Args: - inference_dict (dict): Training configuration dict that includes: - - 'hf_token_file': Path to the file containing the HF token. - - Returns: - str: The HF token string read from the file. - - Behavior: - - Reads the token from inference_dict['hf_token_file'] (already resolved for placeholders). - - Strips the trailing newline from the token. - """ - hf_token_file = inference_dict['hf_token_file'] - try: - with open(hf_token_file, 'r') as fp: - hf_token = fp.read().rstrip("\n") - except FileNotFoundError: - log.error(f"Error: The file '{hf_token_file}' was not found.") - raise - except Exception as e: - log.error(f"An error occurred: {e}") - raise - return hf_token - - -@pytest.fixture(scope="module") -def s_phdl(cluster_dict): - """ - Create and return a parallel SSH handle for all cluster nodes (server). - - Args: - cluster_dict (dict): Cluster configuration loaded by another fixture. Expected keys: - - 'node_dict': dict of node_name -> node_details (used to derive the node list) - - 'username': SSH username for connecting to nodes - - 'priv_key_file': path to the SSH private key file - - Returns: - Pssh: An initialized Pssh handle for issuing commands across all nodes. - - Behavior: - - Prints the full cluster_dict for quick debugging (consider switching to log.debug to reduce noise). - - Collects all node names from cluster_dict['node_dict'] and constructs a Pssh handle. - - Notes: - - This fixture has module scope, so a single connection handle is reused for all tests in the module. - """ - log.info("%s", cluster_dict) - env_vars = cluster_dict.get("env_vars") - node_list = list(cluster_dict['node_dict'].keys()) - s_phdl = Pssh(log, node_list, user=cluster_dict['username'], pkey=cluster_dict['priv_key_file'], env_vars=env_vars) - return s_phdl - - -@pytest.fixture(scope="module") -def c_phdl(cluster_dict): - """ - Create and return a parallel SSH handle for all cluster nodes (client). - - Args: - cluster_dict (dict): Cluster configuration loaded by another fixture. Expected keys: - - 'node_dict': dict of node_name -> node_details (used to derive the node list) - - 'username': SSH username for connecting to nodes - - 'priv_key_file': path to the SSH private key file - - Returns: - Pssh: An initialized Pssh handle for issuing commands across all nodes. - - Behavior: - - Prints the full cluster_dict for quick debugging (consider switching to log.debug to reduce noise). - - Collects all node names from cluster_dict['node_dict'] and constructs a Pssh handle. - - Notes: - - This fixture has module scope, so a single connection handle is reused for all tests in the module. - """ - log.info("%s", cluster_dict) - env_vars = cluster_dict.get("env_vars") - node_list = list(cluster_dict['node_dict'].keys()) - c_phdl = Pssh(log, node_list, user=cluster_dict['username'], pkey=cluster_dict['priv_key_file'], env_vars=env_vars) - return c_phdl - - -def test_cleanup_stale_containers(s_phdl, inference_dict): - """ - Pytest: Clean up potentially stale Docker containers and volumes before tests. - - Args: - s_phdl: Parallel SSH/process handle used by docker_lib to run commands on nodes. - inference_dict (dict): Training configuration dict that includes: - - 'container_name': Name of the container to be killed if running. - - Behavior: - - Kills the specific container identified by inference_dict['container_name']. - - Deletes all containers and volumes on the target nodes (broad cleanup). - - Notes: - - This performs a broad cleanup via delete_all_containers_and_volumes; ensure the - test environment is isolated so this doesn't remove unrelated containers/volumes. - - Consider narrowing cleanup scope if other workloads may be present on the hosts. - """ - - container_name = inference_dict['container_name'] - docker_lib.kill_docker_container(s_phdl, container_name) - docker_lib.delete_all_containers_and_volumes(s_phdl) - - -def test_launch_inference_containers(s_phdl, inference_dict, benchmark_params_dict): - """ - Launch vLLM inference containers on all nodes. - - Note: Container image can be model-specific or use global default. - """ - - log.info(f'Testcase launch vLLM containers for {MODEL_NAME}') - globals.error_list = [] - container_name = inference_dict['container_name'] - - # Get model-specific container image or use global default - container_image = benchmark_params_dict.get(MODEL_NAME, {}).get( - 'container_image', inference_dict['container_image'] - ) - - # Launch the containers .. - docker_lib.launch_docker_container( - s_phdl, - container_name, - container_image, - inference_dict['container_config']['device_list'], - inference_dict['container_config']['volume_dict'], - inference_dict['container_config']['env_dict'], - shm_size='16G', - timeout=60 * 20, - ) - # ADD verifications .. - time.sleep(30) - log.info('Verify if the containers have been launched properly') - out_dict = s_phdl.exec('docker ps') - for node in out_dict.keys(): - if not re.search(f'{container_name}', out_dict[node], re.I): - fail_test(f'Failed to launch container on node {node}') - update_test_result() - - -def test_vllm_inference(c_phdl, s_phdl, inference_dict, benchmark_params_dict, hf_token, seq_combo, concurrency): - """ - Test vLLM inference for Qwen3-235B with specific sequence combination and concurrency level. - - This test is parametrized via pytest_generate_tests to run once per: - - Sequence combination (ISL/OSL pair) defined in model's sequence_combinations - - Concurrency level defined in model's concurrency_levels - - The factory will automatically create the correct VllmJob instance with - the model-specific container image and parameters. - - Args: - seq_combo: Dict with 'isl', 'osl', 'name' keys for this test iteration - concurrency: Integer concurrency level for this test iteration - """ - # Since this is a per-GPU-type config file (mi355x), gpu_type is implicit - gpu_type = "mi355x" - - log.info( - f"Starting inference test for model: {MODEL_NAME}, GPU: {gpu_type}, combination: {seq_combo['name']} (ISL={seq_combo['isl']}, OSL={seq_combo['osl']}), concurrency: {concurrency}" - ) - globals.error_list = [] - - # Override ISL/OSL and concurrency in benchmark_params for this specific test iteration - # Config is now fully flattened, so access directly under benchmark_params - model_params = benchmark_params_dict[MODEL_NAME] - model_params['input_sequence_length'] = seq_combo['isl'] - model_params['output_sequence_length'] = seq_combo['osl'] - model_params['max_concurrency'] = str(concurrency) - - # Calculate num_prompts based on OSL (matching recipe logic) - osl = int(seq_combo['osl']) - if osl == 8192: - model_params['num_prompts'] = str(concurrency * 20) - else: - model_params['num_prompts'] = str(concurrency * 50) - - # Create VllmJob instance - vllm_job = VllmJob( - c_phdl=c_phdl, - s_phdl=s_phdl, - model_name=MODEL_NAME, - inference_config_dict=inference_dict, - benchmark_params_dict=benchmark_params_dict, - hf_token=hf_token, - gpu_type=gpu_type, - distributed_inference=False, - ) - - # Stop any existing server process for clean state - vllm_job.stop_server() - - # Build and start server with current test parameters - vllm_job.build_server_inference_job_cmd() - vllm_job.start_inference_server_job() - - # Run benchmark client - vllm_job.start_inference_client_job() - res_dict = vllm_job.poll_for_inference_completion() - res_index = (MODEL_NAME, gpu_type, seq_combo['isl'], seq_combo['osl'], seq_combo['name'], concurrency) - inf_res_dict[res_index] = res_dict - vllm_job.verify_inference_results() - update_test_result() - - log.info( - f"Completed inference test for model: {MODEL_NAME}, GPU: {gpu_type}, combination: {seq_combo['name']}, concurrency: {concurrency}" - ) - - -def test_print_results_table(): - globals.error_list = [] - pprint(inf_res_dict, depth=3) - rows = [] - headers = [ - "Model", - "GPU", - "ISL", - "OSL", - "Policy", - "Concurrency", - "Host", - "Req/s", - "Total tok/s", - "Mean TTFT (ms)", - "Mean TPOT (ms)", - "P99 ITL (ms)", - ] - - for (model, gpu, isl, osl, policy, concurrency), entry in inf_res_dict.items(): - for host, m in entry["results"].items(): - rows.append( - [ - model, - gpu, - isl, - osl, - policy, - concurrency, - host, - m["successful_requests"], - m["total_throughput_per_sec"], - m["mean_ttft_ms"], - m["mean_tpot_ms"], - m["p99_itl_ms"], - ] - ) - - log.info(tabulate(rows, headers=headers, tablefmt="github")) - update_test_result() diff --git a/cvs/tests/inference/vllm/vllm_qwen3_80b_single.py b/cvs/tests/inference/vllm/vllm_qwen3_80b_single.py deleted file mode 100644 index 0c9e68170..000000000 --- a/cvs/tests/inference/vllm/vllm_qwen3_80b_single.py +++ /dev/null @@ -1,480 +0,0 @@ -''' -Copyright 2025 Advanced Micro Devices, Inc. -All rights reserved. This notice is intended as a precaution against inadvertent publication and does not imply publication or any waiver of confidentiality. -The year included in the foregoing notice is the year of creation of the work. -All code contained here is Property of Advanced Micro Devices, Inc. -''' - -import pytest - -import re -import os -import time -import json -from pprint import pprint -from tabulate import tabulate - -from cvs.lib.parallel_ssh_lib import * -from cvs.lib.utils_lib import * -from cvs.lib import docker_lib -from cvs.lib.inference.vllm import VllmJob -from cvs.lib import globals - -log = globals.log - -# Model name for this test suite -MODEL_NAME = "qwen3-80b" - -inf_res_dict = {} - - -# Importing additional cmd line args to script .. -@pytest.fixture(scope="module") -def cluster_file(pytestconfig): - """ - Retrieve the --cluster_file CLI option provided to pytest. - - Args: - pytestconfig: Built-in pytest fixture exposing command-line options. - - Returns: - str: Path to the cluster JSON file specified via --cluster_file. - - Notes: - - Ensure your pytest.ini or CLI includes: --cluster_file=/path/to/cluster.json - - Use module scope so the value is resolved once per test module. - """ - return pytestconfig.getoption("cluster_file") - - -@pytest.fixture(scope="module") -def training_config_file(pytestconfig): - """ - Retrieve the --config_file CLI option provided to pytest. - - Args: - pytestconfig: Built-in pytest fixture exposing command-line options. - - Returns: - str: Path to the training config JSON file specified via --config_file. - - Notes: - - Ensure your pytest.ini or CLI includes: --config_file=/path/to/training_config.json - - Module scope avoids re-fetching the option across tests in this module. - """ - return pytestconfig.getoption("config_file") - - -# Importing the cluster and cofig files to script to access node, switch, test config params -@pytest.fixture(scope="module") -def cluster_dict(cluster_file): - """ - Load the entire cluster configuration from the provided JSON file. - - Args: - cluster_file (str): Path to the cluster JSON file. - - Returns: - dict: Parsed JSON representing the cluster (nodes, credentials, etc.). - - Notes: - - Logs the loaded structure for visibility; consider using log.debug if verbose. - """ - with open(cluster_file) as json_file: - cluster_dict = json.load(json_file) - - # Resolve path placeholders like {user-id} in cluster config - cluster_dict = resolve_cluster_config_placeholders(cluster_dict) - log.info("%s", cluster_dict) - return cluster_dict - - -@pytest.fixture(scope="module") -def inference_dict(training_config_file, cluster_dict): - with open(training_config_file) as json_file: - inference_dict_t = json.load(json_file) - inference_dict = inference_dict_t['config'] - - # Resolve path placeholders like {user-id}, {home-mount-dir}, etc. - inference_dict = resolve_test_config_placeholders(inference_dict, cluster_dict) - return inference_dict - - -@pytest.fixture(scope="module") -def benchmark_params_dict(training_config_file, cluster_dict): - with open(training_config_file) as json_file: - inference_dict_t = json.load(json_file) - benchmark_params_dict = inference_dict_t['benchmark_params'] - - # Resolve path placeholders like {user-id}, {home-mount-dir}, etc. - benchmark_params_dict = resolve_test_config_placeholders(benchmark_params_dict, cluster_dict) - - log.info("%s", benchmark_params_dict) - return benchmark_params_dict - - -def pytest_generate_tests(metafunc): - """ - Dynamically parametrize inference tests based on sequence combinations and concurrency levels - for the Qwen3-80B model. - - Behavior: - - Reads the config file path from pytest's --config_file option. - - Loads the JSON and extracts qwen3-80b model configuration. - - Extracts sequence_combinations (ISL/OSL pairs) and concurrency_levels. - - Creates test cases for each sequence combination × concurrency level. - - Test Matrix: - - Test IDs: "combination_name-concX" (e.g., "balanced-conc16") - - Notes: - - If no config_file is provided, the hook returns without parametrizing. - - Each combination gets a separate test case with clear ID. - """ - config_file = metafunc.config.getoption("config_file") - if not config_file or not os.path.exists(config_file): - log.warning(f'Warning: Missing or invalid config file {config_file}') - return - - with open(config_file) as fp: - cfg = json.load(fp) - - # Extract qwen3-80b model config (now directly under benchmark_params, no single_node nesting) - benchmark_params = cfg.get("benchmark_params", {}) - model_config = benchmark_params.get(MODEL_NAME, {}) - - if not model_config: - log.warning(f'Warning: Model {MODEL_NAME} not found in config') - return - - # Build test parameters: list of (seq_combo_dict, concurrency, test_id) - test_params = [] - - # Check if model uses sequence_combinations or legacy ISL/OSL - seq_combos = model_config.get("sequence_combinations", []) - - if seq_combos: - # New format: multiple combinations per model - for combo in seq_combos: - combo_name = combo.get("name", f"isl{combo['isl']}_osl{combo['osl']}") - - # Check if model has concurrency_levels array or single max_concurrency - conc_levels = model_config.get("concurrency_levels", []) - if conc_levels: - # Parametrize across concurrency levels - for conc in conc_levels: - test_id = f"{combo_name}-conc{conc}" - test_params.append((combo, conc, test_id)) - else: - # Backward compatibility: use max_concurrency as single value - max_conc = int(model_config.get("max_concurrency", "64")) - test_id = f"{combo_name}" - test_params.append((combo, max_conc, test_id)) - else: - # Legacy format: single ISL/OSL values - isl = model_config.get("input_sequence_length", "1024") - osl = model_config.get("output_sequence_length", "1024") - combo = {"isl": isl, "osl": osl, "name": "default"} - - conc_levels = model_config.get("concurrency_levels", []) - if conc_levels: - for conc in conc_levels: - test_id = f"conc{conc}" - test_params.append((combo, conc, test_id)) - else: - max_conc = int(model_config.get("max_concurrency", "64")) - test_params.append((combo, max_conc, "default")) - - # Parametrize if test uses these fixtures - if "seq_combo" in metafunc.fixturenames and "concurrency" in metafunc.fixturenames: - if test_params: - combos, concs, ids = zip(*test_params) - metafunc.parametrize("seq_combo,concurrency", list(zip(combos, concs)), ids=ids) - - -@pytest.fixture(scope="module") -def hf_token(inference_dict): - """ - Load the Hugging Face access token from the file path specified in the training config. - - Args: - inference_dict (dict): Training configuration dict that includes: - - 'hf_token_file': Path to the file containing the HF token. - - Returns: - str: The HF token string read from the file. - - Behavior: - - Reads the token from inference_dict['hf_token_file'] (already resolved for placeholders). - - Strips the trailing newline from the token. - """ - hf_token_file = inference_dict['hf_token_file'] - try: - with open(hf_token_file, 'r') as fp: - hf_token = fp.read().rstrip("\n") - except FileNotFoundError: - log.error(f"Error: The file '{hf_token_file}' was not found.") - raise - except Exception as e: - log.error(f"An error occurred: {e}") - raise - return hf_token - - -@pytest.fixture(scope="module") -def s_phdl(cluster_dict): - """ - Create and return a parallel SSH handle for all cluster nodes (server). - - Args: - cluster_dict (dict): Cluster configuration loaded by another fixture. Expected keys: - - 'node_dict': dict of node_name -> node_details (used to derive the node list) - - 'username': SSH username for connecting to nodes - - 'priv_key_file': path to the SSH private key file - - Returns: - Pssh: An initialized Pssh handle for issuing commands across all nodes. - - Behavior: - - Prints the full cluster_dict for quick debugging (consider switching to log.debug to reduce noise). - - Collects all node names from cluster_dict['node_dict'] and constructs a Pssh handle. - - Notes: - - This fixture has module scope, so a single connection handle is reused for all tests in the module. - """ - log.info("%s", cluster_dict) - env_vars = cluster_dict.get("env_vars") - node_list = list(cluster_dict['node_dict'].keys()) - s_phdl = Pssh(log, node_list, user=cluster_dict['username'], pkey=cluster_dict['priv_key_file'], env_vars=env_vars) - return s_phdl - - -@pytest.fixture(scope="module") -def c_phdl(cluster_dict): - """ - Create and return a parallel SSH handle for all cluster nodes (client). - - Args: - cluster_dict (dict): Cluster configuration loaded by another fixture. Expected keys: - - 'node_dict': dict of node_name -> node_details (used to derive the node list) - - 'username': SSH username for connecting to nodes - - 'priv_key_file': path to the SSH private key file - - Returns: - Pssh: An initialized Pssh handle for issuing commands across all nodes. - - Behavior: - - Prints the full cluster_dict for quick debugging (consider switching to log.debug to reduce noise). - - Collects all node names from cluster_dict['node_dict'] and constructs a Pssh handle. - - Notes: - - This fixture has module scope, so a single connection handle is reused for all tests in the module. - """ - log.info("%s", cluster_dict) - env_vars = cluster_dict.get("env_vars") - node_list = list(cluster_dict['node_dict'].keys()) - c_phdl = Pssh(log, node_list, user=cluster_dict['username'], pkey=cluster_dict['priv_key_file'], env_vars=env_vars) - return c_phdl - - -@pytest.fixture(scope="module", autouse=True) -def cleanup_on_exit(s_phdl, inference_dict): - """ - Automatically clean up containers after all tests in the module complete. - - This fixture runs automatically (autouse=True) and ensures cleanup happens - even if tests fail, providing proper test isolation. - - Args: - s_phdl: Parallel SSH handle for server nodes - inference_dict: Inference configuration containing container_name - - Yields: - None (all tests run between yield statement and cleanup) - - Behavior: - - Runs setup code before yield (currently none) - - Yields control to run all module tests - - After all tests complete (success or failure), kills container and cleans up volumes - """ - # Setup (before tests) - nothing needed currently - yield - # Teardown (after all tests, even on failure) - try: - container_name = inference_dict['container_name'] - log.info(f"Cleaning up container {container_name} after test module completion") - docker_lib.kill_docker_container(s_phdl, container_name) - docker_lib.delete_all_containers_and_volumes(s_phdl) - except Exception as e: - log.warning(f"Cleanup failed (non-critical): {e}") - - -def test_cleanup_stale_containers(s_phdl, inference_dict): - """ - Pytest: Clean up potentially stale Docker containers and volumes before tests. - - Args: - s_phdl: Parallel SSH/process handle used by docker_lib to run commands on nodes. - inference_dict (dict): Training configuration dict that includes: - - 'container_name': Name of the container to be killed if running. - - Behavior: - - Kills the specific container identified by inference_dict['container_name']. - - Deletes all containers and volumes on the target nodes (broad cleanup). - - Notes: - - This performs a broad cleanup via delete_all_containers_and_volumes; ensure the - test environment is isolated so this doesn't remove unrelated containers/volumes. - - Consider narrowing cleanup scope if other workloads may be present on the hosts. - """ - - container_name = inference_dict['container_name'] - docker_lib.kill_docker_container(s_phdl, container_name) - docker_lib.delete_all_containers_and_volumes(s_phdl) - - -def test_launch_inference_containers(s_phdl, inference_dict, benchmark_params_dict): - """ - Launch vLLM inference containers on all nodes. - - Note: Container image can be model-specific or use global default. - """ - - log.info(f'Testcase launch vLLM containers for {MODEL_NAME}') - globals.error_list = [] - container_name = inference_dict['container_name'] - - # Get model-specific container image or use global default - container_image = benchmark_params_dict.get(MODEL_NAME, {}).get( - 'container_image', inference_dict['container_image'] - ) - - # Launch the containers .. - docker_lib.launch_docker_container( - s_phdl, - container_name, - container_image, - inference_dict['container_config']['device_list'], - inference_dict['container_config']['volume_dict'], - inference_dict['container_config']['env_dict'], - shm_size='16G', - timeout=60 * 20, - ) - # ADD verifications .. - time.sleep(30) - log.info('Verify if the containers have been launched properly') - out_dict = s_phdl.exec('docker ps') - for node in out_dict.keys(): - if not re.search(f'{container_name}', out_dict[node], re.I): - fail_test(f'Failed to launch container on node {node}') - update_test_result() - - -def test_vllm_inference(c_phdl, s_phdl, inference_dict, benchmark_params_dict, hf_token, seq_combo, concurrency): - """ - Test vLLM inference for Qwen3-80B with specific sequence combination and concurrency level. - - This test is parametrized via pytest_generate_tests to run once per: - - Sequence combination (ISL/OSL pair) defined in model's sequence_combinations - - Concurrency level defined in model's concurrency_levels - - The vllm_server fixture provides a running server that is reused across all iterations. - - Args: - vllm_server: VllmJob instance with running server (from fixture) - seq_combo: Dict with 'isl', 'osl', 'name' keys for this test iteration - concurrency: Integer concurrency level for this test iteration - """ - gpu_type = "mi355x" - - log.info( - f"Starting inference test for model: {MODEL_NAME}, GPU: {gpu_type}, combination: {seq_combo['name']} (ISL={seq_combo['isl']}, OSL={seq_combo['osl']}), concurrency: {concurrency}" - ) - globals.error_list = [] - - # Override ISL/OSL and concurrency in benchmark_params for this specific test iteration - model_params = benchmark_params_dict[MODEL_NAME] - model_params['input_sequence_length'] = seq_combo['isl'] - model_params['output_sequence_length'] = seq_combo['osl'] - model_params['max_concurrency'] = str(concurrency) - - # Calculate num_prompts based on OSL (matching recipe logic) - osl = int(seq_combo['osl']) - if osl == 8192: - model_params['num_prompts'] = str(concurrency * 20) - else: - model_params['num_prompts'] = str(concurrency * 50) - - # Create VllmJob instance - vllm_job = VllmJob( - c_phdl=c_phdl, - s_phdl=s_phdl, - model_name=MODEL_NAME, - inference_config_dict=inference_dict, - benchmark_params_dict=benchmark_params_dict, - hf_token=hf_token, - gpu_type=gpu_type, - distributed_inference=False, - server_launch_poll_count=30, - ) - - # Stop any existing server process for clean state - vllm_job.stop_server() - - # Build and start server with current test parameters - vllm_job.build_server_inference_job_cmd() - vllm_job.start_inference_server_job() - - # Run benchmark client - vllm_job.start_inference_client_job() - res_dict = vllm_job.poll_for_inference_completion() - res_index = (MODEL_NAME, gpu_type, seq_combo['isl'], seq_combo['osl'], seq_combo['name'], concurrency) - inf_res_dict[res_index] = res_dict - vllm_job.verify_inference_results() - update_test_result() - - log.info( - f"Completed inference test for model: {MODEL_NAME}, GPU: {gpu_type}, combination: {seq_combo['name']}, concurrency: {concurrency}" - ) - - -def test_print_results_table(): - globals.error_list = [] - pprint(inf_res_dict, depth=3) - rows = [] - headers = [ - "Model", - "GPU", - "ISL", - "OSL", - "Policy", - "Concurrency", - "Host", - "Req/s", - "Total tok/s", - "Mean TTFT (ms)", - "Mean TPOT (ms)", - "P99 ITL (ms)", - ] - - for (model, gpu, isl, osl, policy, concurrency), entry in inf_res_dict.items(): - for host, m in entry["results"].items(): - rows.append( - [ - model, - gpu, - isl, - osl, - policy, - concurrency, - host, - m["successful_requests"], - m["total_throughput_per_sec"], - m["mean_ttft_ms"], - m["mean_tpot_ms"], - m["p99_itl_ms"], - ] - ) - - log.info(tabulate(rows, headers=headers, tablefmt="github")) - update_test_result() diff --git a/cvs/tests/preflight/preflight_checks.py b/cvs/tests/preflight/preflight_checks.py index cbd3850da..ac079449f 100644 --- a/cvs/tests/preflight/preflight_checks.py +++ b/cvs/tests/preflight/preflight_checks.py @@ -13,6 +13,7 @@ from cvs.lib.preflight.version_check import RocmVersionCheck from cvs.lib.preflight.interface_consistency import InterfaceConsistencyCheck from cvs.lib.preflight.ifoe_l2_connectivity import IfoeL2ConnectivityCheck +from cvs.lib.preflight.node_smoke import NodeSmokeCheck # RdmaConnectivityCheck not used - using legacy function temporarily from cvs.lib.preflight.report import PreflightReportGenerator @@ -455,6 +456,58 @@ def test_gid_consistency(phdl, config_dict): preflight_update_test_result() +def test_node_smoke(phdl, config_dict): + """ + Run Primus ``node_smoke`` checks on each reachable node via primus-cli. + + Opt-in via ``node_smoke.connectivity_mode`` in the preflight config (default + ``skip``). Uses parallel SSH — no Slurm required. + + Nodes that fail are reported but are **not** pruned from ``phdl``. + """ + global preflight_results + + if not phdl.reachable_hosts: + log.warning("Primus node_smoke skipped: no reachable hosts remain after earlier preflight pruning") + preflight_results['node_smoke'] = { + 'mode': 'skip', + 'skipped': True, + 'message': 'No reachable nodes available for Primus node_smoke', + 'node_results': {}, + } + preflight_update_test_result() + return + + node_list = list(phdl.reachable_hosts) + log.info("Running Primus node_smoke on %d reachable host(s)", len(node_list)) + + checker = NodeSmokeCheck(phdl, node_list, config_dict) + results = checker.run() + preflight_results['node_smoke'] = results + + if results.get('skipped'): + log.info("Primus node_smoke: %s", results.get('message', 'skipped')) + preflight_update_test_result() + return + + failed_nodes = results.get('failed_nodes') or [] + unknown_nodes = results.get('unknown_nodes') or [] + total = results.get('total_nodes', 0) + passing = len(results.get('passing_nodes') or []) + + if failed_nodes or unknown_nodes: + log.warning( + "Primus node_smoke FAIL on %d/%d node(s): %s", + len(failed_nodes) + len(unknown_nodes), + total, + ", ".join(failed_nodes + unknown_nodes), + ) + else: + log.info("Primus node_smoke PASS on %d/%d nodes", passing, total) + + preflight_update_test_result() + + def test_ifoe_l2_connectivity(phdl, config_dict): """ Test IFoE L2 connectivity using ``afmctl test ping`` (AIMVT-180). @@ -727,6 +780,7 @@ def test_generate_preflight_report(phdl, config_dict, request): 'gid_consistency', 'rocm_versions', 'interface_names', + 'node_smoke', 'ifoe_l2_connectivity', 'rdma_connectivity', ] diff --git a/docs/how-to/run-cvs-tests.rst b/docs/how-to/run-cvs-tests.rst index 49320b6e0..c6a375a88 100644 --- a/docs/how-to/run-cvs-tests.rst +++ b/docs/how-to/run-cvs-tests.rst @@ -41,8 +41,8 @@ You can list available tests using either `cvs run` (with no arguments) or `cvs • ib_perf_bw_test • install_ibperf_tools - cvs.tests.inference.inferencemax (1 test suite) - • inferencemax_gpt_oss_120b_single + cvs.tests.inference.inferencex_atom (1 test suite) + • inferencex_atom cvs.tests.inference.pytorch_xdit (2 test suites) • pytorch_xdit_flux1_dev_single @@ -52,11 +52,8 @@ You can list available tests using either `cvs run` (with no arguments) or `cvs • sglang_deepseek_r1_671b_distributed • sglang_llama_70b_distributed - cvs.tests.inference.vllm (4 test suites) - • vllm_deepseek31_685b_single - • vllm_gpt_oss_120b_single - • vllm_qwen3_235b_single - • vllm_qwen3_80b_single + cvs.tests.inference.vllm (1 test suite) + • vllm_single cvs.tests.mori (1 test suite) • mori_benchmark_test @@ -80,7 +77,7 @@ You can list available tests using either `cvs run` (with no arguments) or `cvs • megatron_llama3_1_8b_single ================================================================================ - Total: 34 test suites across 1 package(s) + Total: 29 test suites across 1 package(s) Run all tests in a file: @@ -627,27 +624,39 @@ Use these scripts to run the Mori tests. cvs run mori_benchmark_test --cluster_file input/cluster_file/cluster.json --config_file input/config_file/mori/mi35x_mori_config.json --html=/var/www/html/cvs/mori.html --capture=tee-sys --self-contained-html --log-file=/tmp/mori.log -vvv -s -Inferencemax test scripts +InferenceX ATOM test scripts ------------------------------ -You can list all available Inferencemax test cases using the CLI: +You can list all available InferenceX ATOM test cases using the CLI: .. code:: bash - cvs list inferencemax_gpt_oss_120b_single + cvs list inferencex_atom .. code:: text - Available tests in inferencemax_gpt_oss_120b_single: - - test_cleanup_stale_containers - - test_gpt_oss_120_single_node - - test_launch_inference_containers + Available tests in inferencex_atom: + - test_launch_container + - test_inferencex_atom_inference + - test_print_results_table + - test_teardown -Use these scripts to run the Inferencemax tests. +Use these scripts to run the InferenceX ATOM tests. Supply your own suite JSON +(``schema_version: 1`` variant config); see :doc:`../reference/configuration-files/inferencex_atom`. +After ``cvs copy-config``, keep **one** ``*threshold.json`` in the same directory as the +``--config_file`` you pass (per-variant subdirs under ``~/input/.../inferencex_atom/``). +Copy-paste lab commands: ``cvs/input/config_file/inference/inferencex_atom/README.md``. .. code:: bash - cvs run inferencemax_gpt_oss_120b_single --cluster_file input/cluster_file/cluster.json --config_file input/config_file/inference/inferencemax/mi300x_inferencemax_gpt_oss_120b_single.json --html=/var/www/html/cvs/inferencemax.html --capture=tee-sys --self-contained-html --log-file=/tmp/inferencemax.log -vvv -s + TS=$(date +%Y%m%d_%H%M%S) + cvs run inferencex_atom \ + --cluster_file ~/input/cluster_file/inferencex_atom_cluster.json \ + --config_file ~/input/config_file/inference/inferencex_atom/single/mi300x_inferencex-atom_deepseek-r1_fp8_single.json \ + --html=~/cvs_results/${TS}_ix-atom-single_mi300x.html \ + --self-contained-html \ + --log-file=~/cvs_results/${TS}_ix-atom-single_mi300x.log \ + -vvv -s Pytorch xdit test scripts @@ -747,73 +756,34 @@ Use these scripts to run the Sglang tests. VLLM test scripts ------------------------------ -You can list all available VLLM test cases using the CLI: +Single-node vLLM benchmarks use one parametrized suite, ``vllm_single``. Each **variant** +is a directory under ``cvs/input/config_file/inference/vllm_single//`` containing +``*_config.json`` and a sibling ``*_threshold.json`` (see :func:`cvs.lib.inference.utils.inferencing_config_loader.load_variant` for vLLM, or :func:`cvs.lib.inference.inferencex_atom.inferencex_atom_config_loader.load_variant` for InferenceX ATOM). +Point ``--config_file`` at the variant's ``*_config.json`` and ``--cluster_file`` at a cluster +JSON that matches your hardware (for example ``input/cluster_file/mi300x_vllm_single.json``). .. code:: bash - cvs list vllm_deepseek31_685b_single + cvs list vllm_single .. code:: text - Available tests in vllm_deepseek31_685b_single: - - test_cleanup_stale_containers - - test_launch_inference_containers + Available tests in vllm_single: + - test_launch_container + - test_setup_sshd + - test_model_fetch + - test_vllm_inference[throughput-conc64] + - test_vllm_inference[throughput-conc128] + - test_vllm_inference[throughput-conc256] - test_print_results_table - - test_vllm_inference - -.. code:: bash - - cvs list vllm_gpt_oss_120b_single - -.. code:: text - - Available tests in vllm_gpt_oss_120b_single: - - test_cleanup_stale_containers - - test_launch_inference_containers - - test_print_results_table - - test_vllm_inference - -.. code:: bash - - cvs list vllm_qwen3_235b_single - -.. code:: text - - Available tests in vllm_qwen3_235b_single: - - test_cleanup_stale_containers - - test_launch_inference_containers - - test_print_results_table - - test_vllm_inference - -.. code:: bash - - cvs list vllm_qwen3_80b_single - -.. code:: text - - Available tests in vllm_qwen3_80b_single: - - test_cleanup_stale_containers - - test_launch_inference_containers - - test_print_results_table - - test_vllm_inference - -Use these scripts to run the VLLM tests. - -.. code:: bash - - cvs run vllm_deepseek31_685b_single --cluster_file input/cluster_file/cluster.json --config_file input/config_file/inference/vllm/mi355x_vllm_single.json --html=/var/www/html/cvs/deepseek.html --capture=tee-sys --self-contained-html --log-file=/tmp/deepseek.log -vvv -s - -.. code:: bash - - cvs run vllm_gpt_oss_120b_single --cluster_file input/cluster_file/cluster.json --config_file input/config_file/inference/vllm/mi355x_vllm_single.json --html=/var/www/html/cvs/gpt.html --capture=tee-sys --self-contained-html --log-file=/tmp/gpt.log -vvv -s - -.. code:: bash + - test_teardown - cvs run vllm_qwen3_235b_single --cluster_file input/cluster_file/cluster.json --config_file input/config_file/inference/vllm/mi355x_vllm_single.json --html=/var/www/html/cvs/qwen235.html --capture=tee-sys --self-contained-html --log-file=/tmp/qwen235.log -vvv -s +The ``test_vllm_inference[...]`` names come from the ``sweep`` block in the variant config +(sequence combination ``name`` plus ``conc``); your list may differ if you use another variant. .. code:: bash - cvs run vllm_qwen3_80b_single --cluster_file input/cluster_file/cluster.json --config_file input/config_file/inference/vllm/mi355x_vllm_single.json --html=/var/www/html/cvs/qwen80.html --capture=tee-sys --self-contained-html --log-file=/tmp/qwen80.log -vvv -s + cvs run vllm_single --cluster_file input/cluster_file/mi300x_vllm_single.json --config_file input/config_file/inference/vllm_single/w1_llama31_70b_fp8kv/w1_llama31_70b_fp8kv_config.json --html=/var/www/html/cvs/vllm_single.html --capture=tee-sys --self-contained-html --log-file=/tmp/vllm_single.log -vvv -s Test results diff --git a/docs/install/cvs-install.rst b/docs/install/cvs-install.rst index db1da4f3b..26a5f2893 100644 --- a/docs/install/cvs-install.rst +++ b/docs/install/cvs-install.rst @@ -306,15 +306,16 @@ Inference CVS provides comprehensive inference testing configurations for various LLM serving frameworks and models. -**InferenceMAX (vLLM Benchmarking)** +**InferenceX ATOM (vLLM Benchmarking)** -1. Copy the InferenceMAX configuration file: +1. Copy the InferenceX ATOM configuration files (main ``*.json`` and optional sibling ``*_threshold.json``): - .. code:: bash +.. code:: bash - cvs copy-config inference/mi300x_singlenode_inferencemax.json --output ~/my_inferencemax_config.json + cvs copy-config inference/inferencex_atom/mi300x_inferencex-atom_gpt-oss-120b_bf16.json --output ~/my_inferencex_atom_config.json + cvs copy-config inference/inferencex_atom/mi300x_inferencex-atom_gpt-oss-120b_bf16_threshold.json --output ~/my_inferencex_atom_threshold.json -2. Edit the file and modify these parameters: +2. Edit the files and modify these parameters: - ``container_image``: Docker image with vLLM - ``nnodes``: Number of nodes in the cluster diff --git a/docs/reference/configuration-files/configure-config.rst b/docs/reference/configuration-files/configure-config.rst index 51b10c4c6..5a36c6d9d 100644 --- a/docs/reference/configuration-files/configure-config.rst +++ b/docs/reference/configuration-files/configure-config.rst @@ -32,7 +32,7 @@ The following list provides a link to code snippets and the parameters for each - :doc:`Megatron ` - :doc:`MORI (RDMA Performance) ` - :doc:`Aorta (Distributed Training) ` -- :doc:`InferenceMAX (vLLM Benchmarking) ` +- :doc:`InferenceX ATOM (vLLM Benchmarking) ` - :doc:`vLLM Single-Node (MI355X) ` - :doc:`SGLang Disaggregated Prefill-Decode ` - :doc:`Flux.1 Text-to-Image ` diff --git a/docs/reference/configuration-files/inferencemax.rst b/docs/reference/configuration-files/inferencemax.rst deleted file mode 100644 index 480e6962d..000000000 --- a/docs/reference/configuration-files/inferencemax.rst +++ /dev/null @@ -1,210 +0,0 @@ -.. meta:: - :description: Configure the variables in the InferenceMAX configuration files - :keywords: inference, ROCm, install, cvs, InferenceMAX, vLLM - -******************************************* -InferenceMAX inference configuration file -******************************************* - -InferenceMAX tests validate inference performance for large language models (LLMs) using vLLM backend on AMD GPU clusters. These tests ensure optimal inference throughput, latency, and token generation performance for AI serving workloads. - -The InferenceMAX tests check: - -- **Container orchestration**: Docker setup with ROCm for inference workloads -- **Model serving**: vLLM backend initialization and model loading -- **Performance metrics**: Output throughput, Time to First Token (TTFT), and Time Per Output Token (TPOT) -- **Benchmarking**: Load testing with various concurrency levels and sequence lengths -- **Result verification**: Expected throughput and latency metrics - -Change the parameters as needed in the InferenceMAX configuration file: ``mi300x_singlenode_inferencemax.json`` for single node inference configurations. - -.. note:: - - - Parameters with the ```` value must have that value modified to your specifications. - - ``{user-id}`` will be resolved to the current username in the runtime. You can also manually change this value to your username. - -``mi300x_singlenode_inferencemax.json`` -======================================== - -Here's a code snippet of the ``mi300x_singlenode_inferencemax.json`` file for reference: - -.. dropdown:: ``mi300x_singlenode_inferencemax.json`` - - .. code:: json - - { - "config": { - "container_image": "rocm/7.0:rocm7.0_ubuntu_22.04_vllm_0.10.1_instinct_20250927_rc1", - "container_name": "inference_max_rocm", - "_example_nnodes": "4", - "nnodes": "4", - "inferencemax_repo": "https://github.com/InferenceMAX/InferenceMAX.git", - "benchmark_script_repo": "https://github.com/kimbochen/bench_serving.git", - "hf_token_file": "/home/{user-id}/.hf_token", - "shm_size": "128G", - "log_dir": "/home/{user-id}/LOGS", - "container_config": { - "device_list": [ - "/dev/dri", - "/dev/kfd" - ], - "volume_dict": { - "/home/{user-id}": "/home/{user-id}" - }, - "env_dict": {} - } - }, - "benchmark_params": { - "gpt-oss-120b": { - "backend": "vllm", - "base_url": "http://0.0.0.0", - "port_no": "8000", - "_example_dataset_name": "sharegpt|hf|random|sonnet|burstgpt", - "dataset_name": "random", - "max_concurrency": "64", - "model": "openai/gpt-oss-120b", - "num_prompts": "1000", - "input_sequence_length": "8192", - "output_sequence_length": "1024", - "burstiness": "1.0", - "seed": "0", - "max_model_length": "9216", - "random_range_ratio": "0.8", - "random_prefix_len": "0", - "tensor_parallelism": "8", - "_example_tokenizer_mode": "auto|slow|mistral|custom", - "tokenizer_mode": "auto", - "percentiles_metrics": "ttft,tpot,itl,e2el", - "metric_percentiles": "99", - "server_script": "gptoss_fp4_mi300x_docker.sh", - "bench_serv_script": "benchmark_serving.py", - "result_dict": { - "output_throughput_per_sec": "4200", - "mean_ttft_ms": "500", - "mean_tpot_ms": "15" - } - } - } - } - -Parameters -========== - -Use the parameters in this table to configure the InferenceMAX configuration file. - -.. |br| raw:: html - -
    - -.. list-table:: - :widths: 3 3 5 - :header-rows: 1 - - * - Configuration parameters - - Default values - - Description - * - ``container_image`` - - rocm/7.0:rocm7.0_ubuntu_22.04_ |br| vllm_0.10.1_instinct_20250927_rc1 - - Docker container image with ROCm and vLLM for inference - * - ``container_name`` - - inference_max_rocm - - Name of the Docker container instance - * - ``nnodes`` - - 4 - - Number of nodes in the cluster - * - ``inferencemax_repo`` - - https://github.com/InferenceMAX/ |br| InferenceMAX.git - - Git repository URL for InferenceMAX framework - * - ``benchmark_script_repo`` - - https://github.com/kimbochen/ |br| bench_serving.git - - Git repository URL for benchmarking scripts - * - ``hf_token_file`` - - ``/home/{user-id}/`` |br| ``.hf_token`` - - Path to HuggingFace authentication token file for model access - * - ``shm_size`` - - 128G - - Shared memory size allocated to the container - * - ``log_dir`` - - ``/home/{user-id}/LOGS`` - - Directory where inference logs are stored - * - ``container_config.`` |br| ``device_list`` - - Values: |br| - ``"/dev/dri"`` |br| - ``"/dev/kfd"`` - - List of device paths to mount in the container for GPU access - * - ``container_config.`` |br| ``volume_dict`` - - ``{"/home/{user-id}": "/home/{user-id}"}`` - - Dictionary mapping host paths to container paths for volume mounts - * - ``/home/{user-id}`` - - ``/home/{user-id}`` - - User home directory mount - * - ``container_config.`` |br| ``env_dict`` - - Empty - - Dictionary of environment variables to set in the container - * - ``benchmark_params.`` |br| ``gpt-oss-120b.backend`` - - vllm - - Inference backend to use (vLLM) - * - ``benchmark_params.`` |br| ``gpt-oss-120b.base_url`` - - http://0.0.0.0 - - Base URL for the inference server - * - ``benchmark_params.`` |br| ``gpt-oss-120b.port_no`` - - 8000 - - Port number for the inference server - * - ``benchmark_params.`` |br| ``gpt-oss-120b.`` |br| ``dataset_name`` - - random - - Dataset type for benchmarking (sharegpt, hf, random, sonnet, burstgpt) - * - ``benchmark_params.`` |br| ``gpt-oss-120b.`` |br| ``max_concurrency`` - - 64 - - Maximum number of concurrent requests during benchmarking - * - ``benchmark_params.`` |br| ``gpt-oss-120b.model`` - - openai/gpt-oss-120b - - HuggingFace model identifier or path - * - ``benchmark_params.`` |br| ``gpt-oss-120b.`` |br| ``num_prompts`` - - 1000 - - Total number of prompts to send during the benchmark - * - ``benchmark_params.`` |br| ``gpt-oss-120b.`` |br| ``input_sequence_length`` - - 8192 - - Length of input sequences in tokens - * - ``benchmark_params.`` |br| ``gpt-oss-120b.`` |br| ``output_sequence_`` |br| ``length`` - - 1024 - - Expected length of output sequences in tokens - * - ``benchmark_params.`` |br| ``gpt-oss-120b.burstiness`` - - 1.0 - - Request burstiness factor (1.0 = uniform distribution) - * - ``benchmark_params.`` |br| ``gpt-oss-120b.seed`` - - 0 - - Random seed for reproducible benchmark results - * - ``benchmark_params.`` |br| ``gpt-oss-120b.`` |br| ``max_model_length`` - - 9216 - - Maximum total sequence length the model can handle - * - ``benchmark_params.`` |br| ``gpt-oss-120b.`` |br| ``random_range_ratio`` - - 0.8 - - Range ratio for random dataset generation - * - ``benchmark_params.`` |br| ``gpt-oss-120b.`` |br| ``random_prefix_len`` - - 0 - - Prefix length for random dataset generation - * - ``benchmark_params.`` |br| ``gpt-oss-120b.`` |br| ``tensor_parallelism`` - - 8 - - Number of GPUs to use for tensor parallelism - * - ``benchmark_params.`` |br| ``gpt-oss-120b.`` |br| ``tokenizer_mode`` - - auto - - Tokenizer mode (auto, slow, mistral, custom) - * - ``benchmark_params.`` |br| ``gpt-oss-120b.`` |br| ``percentiles_metrics`` - - ttft,tpot,itl,e2el - - Comma-separated list of metrics to compute percentiles for (ttft: Time to First Token, tpot: Time Per Output Token, itl: Inter-Token Latency, e2el: End-to-End Latency) - * - ``benchmark_params.`` |br| ``gpt-oss-120b.`` |br| ``metric_percentiles`` - - 99 - - Percentile values to compute for metrics (e.g., 99 for 99th percentile) - * - ``benchmark_params.`` |br| ``gpt-oss-120b.server_script`` - - gptoss_fp4_mi300x_docker.sh - - Script to launch the inference server - * - ``benchmark_params.`` |br| ``gpt-oss-120b.`` |br| ``bench_serv_script`` - - benchmark_serving.py - - Script to run the benchmarking client - * - ``benchmark_params.`` |br| ``gpt-oss-120b.result_dict.`` |br| ``output_throughput_`` |br| ``per_sec`` - - 4200 - - Expected number of output tokens generated per second - * - ``benchmark_params.`` |br| ``gpt-oss-120b.result_dict.`` |br| ``mean_ttft_ms`` - - 500 - - Expected mean Time to First Token in milliseconds - * - ``benchmark_params.`` |br| ``gpt-oss-120b.result_dict.`` |br| ``mean_tpot_ms`` - - 15 - - Expected mean Time Per Output Token in milliseconds diff --git a/docs/reference/configuration-files/inferencex_atom.rst b/docs/reference/configuration-files/inferencex_atom.rst new file mode 100644 index 000000000..654214522 --- /dev/null +++ b/docs/reference/configuration-files/inferencex_atom.rst @@ -0,0 +1,192 @@ +.. meta:: :description: Configure the variables in the InferenceX ATOM configuration files + :keywords: inference, ROCm, install, cvs, InferenceX ATOM, ATOM + +*************************************** +InferenceX ATOM inference configuration file +*************************************** + +InferenceX ATOM tests validate LLM serving on AMD GPU clusters using the **ATOM** stack +(``atom.entrypoints.openai_server`` + ``atom.benchmarks.benchmark_serving``). W1 workloads +use ``params.driver: atom``. Multinode **pipeline parallel** (``PP=2``) uses a framework +coordinator: ``params.driver: vllm_atom`` (vLLM + ATOM ROCm kernels) or ``params.driver: sglang``. +A legacy ``params.driver: vllm`` path remains for GPT-OSS uplift only. + +The suite checks: + +- **Container orchestration**: Docker with ROCm; cluster + variant container blocks merged +- **Model serving**: ATOM OpenAI-compatible server, health + warmup probes +- **Performance metrics**: Throughput, per-GPU throughput, TTFT/TPOT (including p99/p95 tails) +- **Benchmarking**: Named ISL/OSL combos with explicit concurrency sweep cells +- **Result verification**: Tiered ``client.*`` thresholds when ``enforce_thresholds`` is true + +Configs use flat sibling pairs under +``cvs/input/config_file/inference/inferencex_atom/``, matching ``inference/vllm/`` naming: +``{gpu}_inferencex-atom_{model}_{precision}[_{mode}].json`` plus optional +``…_threshold.json``. Pass ``--config_file`` to the main JSON; +:func:`cvs.lib.utils.config_loader.substitute_config` discovers the sole sibling ``*threshold.json`` in the **config file's parent directory** when +``threshold_json`` is omitted. If that directory contains more than one ``*threshold.json``, +loading fails with an ambiguous-threshold ``ValueError``. + +**Lab ``~/input`` layout:** the repo keeps every variant flat in one tree, but after +``cvs copy-config`` you should place each run's config + threshold pair in a dedicated +subdirectory (for example ``~/input/.../inferencex_atom/single/``) so only one +threshold file sits beside the config you pass to ``--config_file``. Alternatively set +``"threshold_json"`` in the config to an explicit path. See the in-tree README at +``cvs/input/config_file/inference/inferencex_atom/README.md`` for copy-paste commands. + +**Cluster file:** use ``cvs/input/cluster_file/inferencex_atom_cluster.json``. Edit ``node_dict`` so host count matches variant ``params.nnodes`` (one host for single-node sweeps; two for multinode). +Container ``name`` must match the variant (``inferencex_atom_mi300x`` / ``inferencex_atom_mi355x``); +the suite deep-merges variant ``container`` over the cluster file. + +**Launcher vs GPU node:** pytest and HTML/log output run on the host where you invoke +``cvs run``. :class:`cvs.core.orchestrators.container.ContainerOrchestrator` SSHes to +cluster nodes (``cluster_file`` ``mgmt_ip`` / ``node_dict``) and runs ``sudo docker`` there. +``paths.models_dir`` and the ATOM image must exist on the GPU node; ``priv_key_file`` and +``paths.hf_token_file`` are read on the launcher. Local Docker on the launcher is not required. + +**ATOM server CLI:** set ``roles.server.atom_args`` inline in the config (vLLM-style, analogous to +``roles.server.serve_args`` on ``vllm_single``). When ``params.driver`` is ``atom``, ``atom_args`` +is required. MTP3 variants also set ``params.bench_extra_args`` (for example ``--use-chat-template``). + +Pytest and HTML layout (inferencex_atom) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + :widths: 10 35 55 + :header-rows: 1 + + * - Stage + - Test + - Notes + * - 1 + - ``test_launch_container`` + - Host Docker launch; records ``container_launch``. + * - 2 + - ``test_setup_sshd`` + - Multinode only; single-node skips sshd probe. + * - 3 + - ``test_model_fetch`` + - Ensures model bytes under ``paths.models_dir``. + * - 4 + - ``test_inferencex_atom_inference`` + - One parametrized cell; server start (or reuse), bench, parse ``results.json``. + * - 5 + - ``test_cell_metrics`` + - One HTML row per **metric tier** per cell (throughput, ttft, tpot, health, record). + * - 6 + - ``test_print_results_table`` + - Session results grid from ``inf_res_dict``. + * - 7 + - ``test_teardown`` + - Explicit teardown; sets ``lifecycle.torn_down``. + +Example variant layout +====================== + +Each stem has ``.json`` (``schema_version: 1``, ``framework: inferencex_atom``) +and sibling ``_threshold.json``. In the CVS source tree many stems share one directory; +on a lab machine, copy only the pair you need into a per-variant subdirectory (or set +``threshold_json``). See ``mi300x_inferencex-atom_deepseek-r1_fp8_single.json`` +for the W1 MI300X reference. + +.. dropdown:: Example ``mi300x_inferencex-atom_deepseek-r1_fp8_single_threshold.json`` (excerpt) + + .. code:: json + + { + "ISL=1024,OSL=1024,TP=8,CONC=128": { + "client.output_throughput": {"kind": "min_tok_s", "value": 2590.98}, + "client.per_gpu_throughput": {"kind": "min_tok_s", "value": 648.65}, + "client.p99_ttft_ms": {"kind": "max_ms", "value": 1834.3}, + "client.p95_tpot_ms": {"kind": "max_ms", "value": 53.76}, + "client.success_rate": {"kind": "min", "value": 1}, + "client.failed": {"kind": "max", "value": 0} + } + } + + Every member of :data:`cvs.lib.inference.inferencex_atom.inferencex_atom_parsing.GATED_METRICS` needs a + spec in each cell when ``enforce_thresholds`` is true. W1 perf gates include + ``per_gpu_throughput``, ``output_tput_per_gpu``, ``p99_ttft_ms``, and ``p95_tpot_ms``. + +Parameters +========== + +Top-level blocks follow the DTNI variant schema. InferenceX ATOM-specific keys: + +.. list-table:: + :widths: 3 3 5 + :header-rows: 1 + + * - Block / key + - Example + - Description + * - ``framework`` + - ``inferencex_atom`` + - Suite identifier for :func:`load_variant`. + * - ``gpu_arch`` + - ``mi300x`` + - Hardware profile for the variant. + * - ``roles.server.atom_args`` + - ``["-tp", "8", "--kv_cache_dtype", "fp8"]`` + - Inline ATOM ``openai_server`` CLI tokens after ``--model`` / ``--server-port``. + * - ``roles.server.serve_args`` + - ``{"kv-cache-dtype": "fp8", "enforce-eager": true}`` + - vLLM / vLLM-ATOM path (``params.driver: vllm`` or ``vllm_atom``); merged into ``vllm serve`` argv. + * - ``roles.server.sglang_args`` + - ``["--trust-remote-code", "--disable-cuda-graph"]`` + - Extra tokens appended to ``sglang.launch_server`` when ``params.driver: sglang``. + * - ``roles.server.ib_netdev`` + - ``ens51f1np1`` + - **Required** when ``nnodes > 1`` and ``driver`` is ``vllm_atom`` or ``sglang``; sets ``NCCL_SOCKET_IFNAME``. + * - ``enforce_thresholds`` + - ``true`` / ``false`` + - When true, ``test_cell_metrics`` asserts via :func:`cvs.lib.utils.verdict.evaluate_all`. + * - ``paths.*`` + - ``shared_fs``, ``models_dir`` (``/home/models``), ``log_dir``, ``hf_token_file`` + - ``models_dir`` is an absolute HF hub cache path on GPU nodes; variant configs also bind-mount ``/home/models`` into the container. + * - ``model.id`` + - ``deepseek-ai/DeepSeek-R1-0528`` + - HuggingFace model id for ATOM server and bench. + * - ``container.image`` / ``container.name`` + - ``rocm/atom-dev:latest``, ``inferencex_atom_mi300x`` + - Docker image and container name (override cluster file defaults). + * - ``roles.server.atom_args`` + - ``-tp``, ``--kv_cache_dtype`` + - Extra CLI tokens after ``--model`` / ``--server-port`` (ATOM driver). + * - ``roles.server.env`` + - ``ATOM_DISABLE_MMAP`` + - Merged into ``/tmp/server_env_script.sh`` before server launch. + * - ``params.driver`` + - ``atom`` / ``vllm`` / ``vllm_atom`` / ``sglang`` + - ``atom`` = standalone ATOM server + ``benchmark_serving`` (no native PP). ``vllm_atom`` = vLLM multinode PP coordinator + ATOM kernels. ``sglang`` = SGLang PP coordinator. ``vllm`` = interim uplift. + * - ``params.tensor_parallelism`` + - ``8`` + - TP size; appears in threshold cell keys as ``TP``. + * - ``params.reuse_server_across_sweep`` + - ``true`` + - Skip server restart when only concurrency changes between sweep cells. + * - ``params.nnodes`` / ``params.pipeline_parallel_size`` + - ``2`` / ``2`` (multinode PP) + - True multinode pipeline parallel: set ``driver=vllm_atom`` or ``sglang`` with ``nnodes=2`` and ``pipeline_parallel_size=2`` (cell keys use ``PP=2``). Requires ``roles.server.ib_netdev``. Standalone ``driver=atom`` multinode uses SPMD data parallel (``DP`` in cell keys), not PP. + * - ``params.master_addr`` / ``params.master_port`` + - head VPC IP / ``29501`` + - Rendezvous for vLLM/SGLang distributed executor (``dist-init-addr`` for SGLang). + * - ``params.scaling_baseline_output_throughput`` + - ``1500`` + - Single-node reference ``output_throughput`` for ``scaling.efficiency_pct`` (record-only). + * - ``params.server_warmup_wait_s`` / ``client_initial_wait_s`` + - ``330`` / ``120`` + - Config-driven server warmup and client poll floor (use `-k` for a one-cell smoke). + * - ``params.metric_percentiles`` + - ``95,99`` + - Tail percentiles for W1 gates (p95 TPOT, p99 TTFT). + * - ``params.bench_max_failed_requests`` + - ``0`` + - Runtime cap on bench ``Failed requests``; pair with threshold ``client.failed``. + * - ``sweep.sequence_combinations`` / ``sweep.runs`` + - named ISL/OSL + ``{combo, concurrency}`` + - Explicit cell list (not a cartesian product). + +Metric tiers and parsing live in :mod:`cvs.lib.inference.inferencex_atom.inferencex_atom_parsing` +(see ``cvs/lib/inference/utils/docs/inferencex-atom-parsing.md``). Legacy monolithic JSON +(``config`` + ``benchmark_params``) and the deprecated ``inferencemax`` suite are not used. diff --git a/docs/sphinx/_toc.yml.in b/docs/sphinx/_toc.yml.in index 6a1aa4887..daa4d3b1a 100644 --- a/docs/sphinx/_toc.yml.in +++ b/docs/sphinx/_toc.yml.in @@ -46,8 +46,8 @@ subtrees: title: MORI - file: reference/configuration-files/aorta title: Aorta - - file: reference/configuration-files/inferencemax - title: InferenceMAX + - file: reference/configuration-files/inferencex_atom + title: InferenceX ATOM - file: reference/configuration-files/vllm_singlenode_mi355x title: vLLM Single-Node MI355X - file: reference/configuration-files/sglang diff --git a/docs/what-is-cvs.rst b/docs/what-is-cvs.rst index 1e03b45f9..587501b5e 100644 --- a/docs/what-is-cvs.rst +++ b/docs/what-is-cvs.rst @@ -23,7 +23,7 @@ Here are the tests available in the CVS: - **RDMA performance tests**: Validate RDMA (Remote Direct Memory Access) bandwidth and latency with MORI for high-speed inter-node communication using AMD Pensando AINIC and other RDMA-capable devices. - **Inference tests**: Validate LLM serving performance and generative AI workloads across AMD GPU clusters. - - InferenceMAX benchmarks vLLM inference performance for models like GPT-OSS-120B, measuring throughput, TTFT (Time to First Token), and TPOT (Time Per Output Token). + - InferenceX ATOM benchmarks vLLM inference performance for models like GPT-OSS-120B, measuring throughput, TTFT (Time to First Token), and TPOT (Time Per Output Token). - vLLM single-node tests support multiple models (GPT-OSS-120B, Qwen3-235B, Qwen3-80B, DeepSeek-V3.1) with various workload scenarios on MI355X GPUs. - SGLang disaggregated prefill-decode architecture tests optimize LLM serving by separating prefill and decode phases across different nodes. - Flux.1 text-to-image generation tests validate distributed image generation using xDiT with Ulysses and Ring parallelization. diff --git a/plans/building-a-cvs-test-suite.md b/plans/building-a-cvs-test-suite.md new file mode 100644 index 000000000..e97f70cb8 --- /dev/null +++ b/plans/building-a-cvs-test-suite.md @@ -0,0 +1,270 @@ +**Building a CVS test suite — a reference guide** + +This guide documents the base infrastructure for writing a new **inference** or +**training** test suite in CVS, using the `vllm_single` suite (PR against +`dev/dtni`) as the worked reference implementation. If you are adding a serving +framework, a training benchmark, or any new workload, base your structure on the +patterns here rather than on the older `InferenceBaseJob` / `if_dict` style. + +> **Status note.** CVS is experimental. Flags and config keys change, and the +> built-in pass/fail is not a trustworthy oracle on its own — always verify a +> run's artifacts independently. This guide describes the structure, not a frozen +> API. + +> **Training is not ported yet.** Only `vllm_single` (inference) exists today. +> The library layout, the generic↔domain seam, and the suite skeleton below are +> deliberately framework-neutral so a training suite drops into the same shape — +> this guide is the blueprint for that port, not a record of it. + +### What changed in the restructure + +This PR reorganizes where shared vs. workload-specific code lives, so the next +suite reuses machinery instead of copy-pasting it. + +**`cvs/lib/dtni` → `cvs/lib/utils` (renamed).** `dtni` was a leftover project +codename; the directory actually holds pure, framework-agnostic helpers. The new +name says what it is: utilities any suite (inference, training, …) can call. + +**`cvs/lib/utils/` — the shared/common layer.** Everything here is generic: no +suite knows about vLLM, Megatron, ISL/OSL, or goodput. It holds: +- `config_loader.py` — the generic config skeleton (`BaseVariantConfig`), the + `paths`/`model`/`image`/`container` schema, the 3-pass placeholder + substitution engine, and `substitute_config()` (read a config + its sibling + threshold file, resolve placeholders). +- `verdict.py` — `evaluate_all(actuals, thresholds)`, a metric-name-agnostic + threshold checker (`min`, `max_ms`, `within`, `min_tok_s`, `min_ratio`). + +**`cvs/lib/inference/utils/` — the domain-specific layer.** Helpers only an +*inference* suite needs live here, one level down from the shared dir. It holds: +- `inferencing_config_loader.py` — the inference config schema: the named-combo + sweep selector, `GoodputSlo`, the vLLM bench `Params`, and + `VariantConfig(BaseVariantConfig)` with the ISL/OSL/TP/CONC `cell_key`. It + *imports* `BaseVariantConfig` + `substitute_config` from `cvs/lib/utils` rather + than re-implementing them. +- `vllm_parsing.py` — the `client.*` metric vocabulary (`to_client_metrics`, + `CLIENT_METRICS`): pure transforms from a vLLM benchmark artifact to a + namespaced metric dict. + +**The rule for where a helper goes.** If every workload could use it (config +plumbing, threshold math) → `cvs/lib/utils/`. If it only makes sense for one +domain (sweep shapes, serving metrics) → `cvs/lib//utils/`. A training +port adds `cvs/lib/training/utils/` the same way inference did — subclass +`BaseVariantConfig`, reuse `substitute_config` and `evaluate_all`, add its own +schema and metric vocabulary. The shared layer never grows a dependency on a +specific framework. + +``` +cvs/lib/ + utils/ # shared, framework-agnostic (was: dtni) + config_loader.py # BaseVariantConfig, substitute_config, placeholders + verdict.py # evaluate_all (generic threshold kinds) + inference/ + utils/ # inference-only helpers + inferencing_config_loader.py # sweep selector, Params, VariantConfig, cell_key + vllm_parsing.py # client.* metrics, to_client_metrics + vllm_single.py # the driver (VllmJob) + training/ # (future) same shape: + utils/ # training_config_loader.py, _parsing.py +``` + +### The layered architecture + +A suite is built from six layers. Each has one job; the seams between them are +the reusable surface a second suite plugs into. + +``` + cluster file (node IP + user/key/orchestrator) <- per-environment, OUT of repo + | + variant config (*_config.json + *threshold.json) <- per-workload, IN repo + | + config loader ── cvs/lib/utils/config_loader.py (generic: schema, substitution) + | └ cvs/lib/inference/utils/inferencing_config_loader.py (domain schema + sweep) + | + orchestrator ── cvs/core/orchestrators (ContainerOrchestrator; provided) + | + job / driver ── cvs/lib/inference/vllm_single.py (launch server, run client, fetch artifact) + | + suite / pytest ── cvs/tests/inference/vllm/ (lifecycle-as-tests, parametrization, HTML) + | + parsing + verdict ── vllm_parsing.to_client_metrics + cvs/lib/utils/verdict.evaluate_all +``` + +The **generic↔domain seam** is the key idea: anything every workload shares lives +in `cvs/lib/utils/`; anything specific to *your* workload lives in your own +`cvs/lib//utils/` module that subclasses/reuses it. The config loader +docstring literally calls this the "generalization seam," and this PR executes +the split. + +### The two config files (per workload) + +A workload is described by a pair of JSON files in the same directory: + +- `*_config.json` — the variant: paths, model, container (with `container.image`), + params, sweep. +- `*threshold.json` — the per-cell pass/fail thresholds. + +The loader finds the threshold by **sibling glob** (exactly one `*threshold.json` +next to the config), so the two share a descriptive prefix +(`llama31_70b_fp8_config.json` / `llama31_70b_fp8_threshold.json`). + +#### Placeholders + +Config values use `{...}` placeholders resolved in three passes: +`{user-id}` (from the cluster file) → `{shared_fs}` (self-reference within +`paths`) → `{paths.models_dir}` (cross-block). An unknown placeholder is left +literal — there is no error, so check for a stray brace in a resolved path if +something doesn't mount. + +#### enforce_thresholds + +`enforce_thresholds: false` makes the suite **record-only**: it captures every +metric and skips assertions, and a threshold/sweep mismatch warns instead of +failing the load. Use it for un-calibrated workloads (e.g. throughput +characterization where the published numbers are curves, not tabulated cells). +Flip to `true` once you have real numbers — the coverage check then guarantees +both that every sweep cell has a threshold entry AND that every cell carries a +spec for every gated metric (`GATED_METRICS`), so a green run can't have +silently skipped its verdict. + +### The sweep selector (named combos + runs) + +The sweep enumerates exactly the `(sequence-shape, concurrency)` cells to run. It +replaces the old `sequence_combinations × concurrency_levels` cartesian: + +```json +"sweep": { + "sequence_combinations": [ + { "name": "w1_isl=128_osl=2048", "isl": "128", "osl": "2048", + "goodput_slo": { "ttft_ms": 1000000000.0, "tpot_ms": 1000000000.0, "e2el_ms": 1000000000.0 } } + ], + "runs": [ + { "combo": "w1_isl=128_osl=2048", "concurrency": 16 } + ] +} +``` + +- Combos are named once; `runs` cherry-picks `(combo, concurrency)` pairs. No NxM + explosion — you list precisely the cells you want. +- Load-time validation rejects duplicate combo names and any `run.combo` that + names no combo. (This is mirrored in the suite's `pytest_generate_tests`, which + reads raw JSON at collection time before the typed loader runs.) +- `goodput_slo` is an **input** to the run (passed to `vllm bench serve + --goodput`), not a threshold — that's why it lives in the sweep. + +Each cell's threshold key is produced by `VariantConfig.cell_key()` +(`ISL=…,OSL=…,TP=…,CONC=…`). It is the single source of truth shared by the +coverage check and the verdict lookup, so threshold.json keys must match it +exactly. + +### The job/driver (self-contained, no external .sh) + +`vllm_single.VllmJob` is the reference driver. It talks only to an injected +orchestrator (`orch.exec`, which routes into the running container) and a typed +`VariantConfig`. Lifecycle highlights to copy: + +- **The server command is built in Python** (`_server_argv`), not cloned from an + external `.sh` repo. A run is self-contained. Per-model quirks come from + `roles.server.serve_args` (a `{flag: value}` map) / `roles.server.env` in + config. +- **`/tmp/server_env_script.sh`** is written by `build_server_cmd` and **sourced + by both server and client**, so the two share one environment (HF token, cache + pin, AITER flags). Each value is `shlex.quote`d. +- **`--max-model-len` is derived per cell** from isl/osl/random_range_ratio so a + sweep change stays self-consistent. +- **Readiness/completion are detected by scanning the whole log**, with narrow + failure markers (don't match bare `error:` — ROCm/vLLM logs benign ones). +- **Results come from the stock `results` artifact**, not console-regex. + `parse_results` fetches the extensionless JSON `vllm bench serve` writes to + `--result-dir` and hands it to the pure `to_client_metrics`. Missing/empty/ + unparseable → hard-fail the cell (never a silently-green empty row). + +The fetch lives in the job (artifact layout is job-specific); the transform lives +in `inference/utils` (so distributed/disagg/InferenceMax reuse it). + +### The suite (lifecycle-as-tests) + +In `cvs/tests/inference/vllm/`, each lifecycle stage is its own pytest test so it +shows up as a timed, pass/fail row in the HTML report: + +``` +test_launch_container → test_setup_sshd → test_model_fetch + → test_vllm_inference (per cell) → test_metric (per metric per cell) + → test_print_results_table → test_teardown +``` + +Patterns to copy: + +- **`pytest_generate_tests`** (in the suite module, not conftest) parametrizes + from the sweep selector. It runs at collection time and reads raw JSON, so it + re-validates combos by hand (mirroring the typed loader). +- **`test_vllm_inference` runs the benchmark once per cell** and stashes results + in a module-scoped `inf_res_dict`. **`test_metric` reads one cached metric** and + is one HTML row per metric per cell — no GPU work, asserts only when + `enforce_thresholds` is true and a spec exists. +- **`_Lifecycle`** (`conftest`) carries cross-test state: `failed` lets a broken + stage skip the rest instead of cascading; `torn_down` lets explicit teardown + suppress the fixture leak-guard finalizer. +- **The `orch` fixture owns only the teardown safety net** — launch/sshd happen + in tests so they're timed rows. A mid-sweep failure still tears the container + down via the finalizer. +- **Single-node guards**: `test_setup_sshd` only probes port 2224 when + `len(orch.hosts) > 1` (in-container sshd exists only for inter-node MPI). +- **HTML Value/Unit columns** come from `pytest_html_results_table_header` / + `_row` hooks scoped to this conftest, populated from + `metric_value`/`metric_unit` user-properties. + +### Parsing + verdict + +- **`to_client_metrics(raw, *, tp, isl)`** — pure: stock keys namespaced + `client.*` 1:1, plus derived metrics (`per_gpu_throughput`, + `decode_throughput_p50`, `success_rate`, …) via `_safe_div` (degrades to + `None`, never crashes). `CLIENT_METRICS` is the ordered display surface. +- **`evaluate_all(actuals, thresholds)`** — generic, framework-neutral. Kinds: + `min`, `max`, `max_ms`, `within`, `min_tok_s`, `min_ratio`. Raises + `ThresholdViolation` listing every failure. A `None` actual is a loud + violation, not a TypeError. +- **`GATED_METRICS`** (in `vllm_parsing`) — the asserted SLO subset of + `CLIENT_METRICS`. The loader's coverage check requires a threshold spec for + every gated metric in every present cell, so a gated metric can't silently + fall through to a zero-assertion record-only row. A new metric is record-only + until added to the set. + +### To add your own suite — checklist + +1. **Schema.** If serving: reuse `inferencing_config_loader` (subclass `Params` + for a new framework's flags). If a new domain (training): create + `cvs/lib//utils/_config_loader.py`, subclass + `BaseVariantConfig`, reuse `substitute_config` + `evaluate_all` from + `cvs/lib/utils/`. Define your own `cell_key` + coverage check. +2. **Config pair.** Write `*_config.json` + `*threshold.json` under + `cvs/input/config_file////`. Start + `enforce_thresholds: false` until calibrated. +3. **Driver.** Write a `Job` class that takes an `orch` + typed config and owns + launch → run → fetch-artifact → `parse`. Build commands in Python; keep the + pure transform in your `utils`. Hard-fail on missing artifacts. +4. **Suite.** Under `cvs/tests///`: `conftest.py` (fixtures + + lifecycle + HTML hooks), the suite module (`pytest_generate_tests` + + lifecycle-as-tests), `_shared.py` (the results table). Pin test order in + `pytest_collection_modifyitems`. +5. **Metrics.** Define your metric vocabulary + units list once in your `utils`, + the way `CLIENT_METRICS` does — don't re-list rows per suite. +6. **Cluster file.** Keep it minimal and OUT of the repo: node IP + user + key + + orchestrator. The variant config supplies the container block; don't ship a + bespoke per-suite cluster file. +7. **Verify independently.** After a run, read the artifact + the HTML cells + yourself. CVS's PASS is not a trustworthy oracle. + +### Reference files (this PR) + +| Layer | File | +|---|---| +| generic schema + substitution | `cvs/lib/utils/config_loader.py` | +| generic verdict | `cvs/lib/utils/verdict.py` | +| inference schema + sweep | `cvs/lib/inference/utils/inferencing_config_loader.py` | +| client metric vocabulary | `cvs/lib/inference/utils/vllm_parsing.py` | +| driver | `cvs/lib/inference/vllm_single.py` | +| suite | `cvs/tests/inference/vllm/{vllm_single,conftest,_shared}.py` | +| config pair | `cvs/input/config_file/inference/vllm_single/w1_llama31_70b_fp8kv/` | + +Agent-facing summaries (entry points + gotchas) live in each package's +`AGENTS.md`. diff --git a/plans/dtni-dev-guide.md b/plans/dtni-dev-guide.md new file mode 100644 index 000000000..04b48240e --- /dev/null +++ b/plans/dtni-dev-guide.md @@ -0,0 +1,352 @@ +# DTNI suite developer guide + +> **⚠️ SUPERSEDED — design doc, not current state.** This guide predates the +> restructure that shipped. It still references the old `cvs/lib/dtni/` and +> `cvs/input/dtni/` paths (now `cvs/lib/utils/` + `cvs/lib/inference/utils/`), +> the `concurrency_levels × sequence_combinations` cartesian (now named combos + +> a `runs[]` selector), the top-level `image` block (now `container.image`), +> `OrchestratorConfig.from_dicts` / `container.enabled|launch`, and other +> pre-merge shapes. For the as-built architecture, read +> `plans/building-a-cvs-test-suite.md` and each package's `AGENTS.md`. Kept for +> the design rationale (the §5–§7 *why* behind the seam, the Job split, and the +> config/threshold split), which is still accurate. + +## 1. Intro and scope + +This guide is for CVS developers who already run `cvs run` regularly and have edited a test wrapper or a lib helper, but who haven't worked under the new DTNI (data-center training and inference) layout. The goal is to give you the mental model and the concrete skeleton needed to port an existing suite or author a new one. + +The framing: today every suite is a hand-written pytest module that ships its own container lifecycle, its own config parsing, and its own threshold checks inline. Under DTNI, those concerns move out of the test module into shared machinery — a typed config loader, an `orch` (orchestrator) fixture that owns the container, and a per-framework Job class that bundles the framework-specific verbs — so the test module shrinks to a few phases: load → setup → generated tests → custom tests. + +`vllm_single` (inference) and `megatron_*` (training) appear as running examples. The same shape applies to sglang, inferencex_atom, pytorch_xdit, jax. + +## 2. Old lifecycle: `cvs run` to HTML report + +```mermaid +flowchart TD + A[cvs run suite --cluster_file --config_file] --> B[cvs/cli_plugins/run_plugin.py] + B --> C[pytest invocation] + C --> D[wrapper module imports] + D --> E[6 module fixtures
    cluster_file, *_dict, hf_token, phdl, gpu_type] + E --> F[test_cleanup_stale_containers
    docker_lib.kill + delete_all] + F --> G[test_launch_*_containers
    docker_lib.launch_docker_container] + G --> H[parametrized workload test
    Job.start/poll/verify] + H --> I[test_print_results_table
    tabulate inf_res_dict] + I --> J[pytest HTML report + exit code] +``` + +Concrete trace using `cvs/tests/inference/vllm/vllm_qwen3_80b_single.py` (480 LOC) and `cvs/tests/training/megatron/megatron_llama3_1_8b_single.py` (315 LOC): + +1. **CLI entry.** `cvs/cli_plugins/run_plugin.py` parses args and invokes pytest against the resolved test module. `cvs list ` uses `cli_plugins/list_plugin.py` (which calls `pytest --collect-only -q`). +2. **Fixtures.** Each wrapper declares ~6 module-scoped fixtures that re-implement the same shape: `cluster_file`, `_config_file`, `cluster_dict`, `_dict`, helper dicts (`benchmark_params_dict` for inference, `model_params_dict` for training), `hf_token`, and a Pssh handle (`s_phdl`/`c_phdl` for inference, `phdl` for training). Dicts are loaded via raw `json.load` and run through `resolve_test_config_placeholders`. The training wrapper also probes `rocm-smi -a` live to derive `gpu_type`. +3. **Container lifecycle as ordered pytest functions.** `test_cleanup_stale_containers` calls `cvs/lib/docker_lib.py` (`kill_docker_container`, `delete_all_containers_and_volumes`). `test_launch__containers` calls `docker_lib.launch_docker_container` with device/volume/env/shm-size pulled from the loaded dict. Distributed training wrappers add a third lifecycle test (`test_disable_firewall`) that shells out via `phdl.exec('sudo service ufw stop')` to work around torchrun rendezvous timeouts. Inference wrappers add an autouse `cleanup_on_exit` fixture that kills the container again on module teardown. +4. **Workload.** Inference: parametrized `test__inference[-conc]` builds a `VllmJob` (`cvs/lib/inference/vllm.py`, subclass of `InferenceBaseJob` in `cvs/lib/inference/base.py`, 711 LOC), mutates a shared `benchmark_params_dict` with the current cell's params, calls `build_server_inference_job_cmd → start_inference_server_job → wait_for_health → start_inference_client_job → verify_inference_results`, parses into module-level `inf_res_dict`. Training: a single non-parametrized test builds a `MegatronLlamaTrainingJob` (`cvs/lib/megatron_training_lib.py`, 834 LOC, no shared base) and calls `exec_nic_setup_scripts → build_training_job_cmd → start_training_job → poll_for_training_completion → verify_training_results`. Both flows accumulate errors into the global `globals.error_list` and call `update_test_result()` at the end. +5. **Output.** Inference: `test_print_results_table` tabulates `inf_res_dict`. Training: no separate printer; verification is inline. Both emit the pytest HTML report and exit with the pytest exit code. + +The seams that hurt: + +- 4 wrappers per suite, byte-similar with one knob different (model id for inference, `distributed_training=False/True` for training). Any cluster-config schema change must be reapplied 4 times per suite. +- Container lifecycle encoded as ordered pytest functions plus an autouse fixture. Test ordering matters; shared state in `inf_res_dict`/`globals.error_list` is implicit. +- 700-800 LOC Job classes that mix command building, remote execution, output parsing, and pass/fail thresholds. `InferenceBaseJob` has vllm-shaped env vars leaked into the base, a dead distributed branch, the `random_range_ration` typo, and a silent skip in `verify_inference_results`. New suites that subclass it inherit the bugs. +- Cluster probes (`rocm-smi`, firewall status) live inside fixtures or workaround tests, called directly through `phdl.exec`. No consistent way to "ask the cluster something." +- Configuration is a single JSON blob mixing "what to run" with "did it pass." No typing, no schema, no separable lifecycle. + +## 3. New lifecycle under DTNI + +The CLI surface and pytest invocation are unchanged. What changes is what the test module looks like and where the work lives. + +```mermaid +flowchart TD + A[cvs run suite --cluster_file --config_file] --> B[cvs/cli_plugins/run_plugin.py] + B --> C[pytest invocation] + C --> D[wrapper module imports] + D --> E[Phase 1: load
    config_loader.load_variant
    returns typed config + thresholds] + E --> F[Phase 2: setup
    orch fixture builds OrchestratorConfig
    orch.setup_containers] + F --> G[Phase 3: generated tests
    pytest_generate_tests parametrizes
    over benchmark_params / model_params] + G --> H[Phase 4: custom tests
    suite-specific assertions
    e.g. firewall, NIC setup, print_results] + H --> I[orch.teardown_containers] + I --> J[pytest HTML report + exit code] +``` + +Phase-by-phase: + +1. **Load.** `cvs/lib/dtni/config_loader.py` reads `cvs/input/dtni///config.json` and `threshold.json`, validates through Pydantic models (`extra="forbid"` everywhere except the runtime args passthrough), runs placeholder substitution in fixed order (cluster → self-reference → cross-block), and returns typed objects. One wrapper per suite, parametrized across all variant directories. +2. **Setup.** A pytest fixture builds an `OrchestratorConfig` (`cvs/core/orchestrators/factory.py`) from the loaded config and yields an `orch`. The fixture calls `orch.setup_containers()` on entry and `orch.teardown_containers()` on exit. No `test_cleanup_stale_containers` or `test_launch_*_containers` in suite code — those concerns are gone. +3. **Generated tests.** `pytest_generate_tests` (in the suite's `conftest.py`) reads sweep dimensions from the typed config — `benchmark_params.concurrency_levels × sequence_combinations` for inference, `model_params` presets for training — and parametrizes the workload test. Each parametrize cell constructs a Job, calls its verbs, gets back a flat `actuals` dict, and runs `evaluate_all(actuals, thresholds)`. +4. **Custom tests.** Suite-specific assertions that aren't part of the sweep grid: firewall disable, NIC setup probe, results-table printer, smoke checks. These live in the wrapper as plain `def test_*` functions and use `orch.exec`/`orch.exec_on_head` to talk to the cluster — never `phdl.exec` or `docker_lib` directly. + +Output: same pytest HTML report, same exit code. Reportable rows are built from `actuals`, not from a global dict mutated by ordered tests. + +## 4. Test file skeleton (the phases, concretely) + +This is the base layout. Copy and replace the framework-specific bits. + +```python +# cvs/tests///.py +""" — DTNI layout. Phases: load → setup → generated → custom.""" + +import pytest +from cvs.lib.dtni.config_loader import load_variant, enumerate_variants +from cvs.lib.dtni.verdict import evaluate_all +from cvs.lib.._orch import Job + + +# -------- Phase 1: load (delegated to conftest.py fixtures) -------- +# variant_config + thresholds come from the conftest's load_variant fixture. +# This module does not call json.load. + +# -------- Phase 2: setup (delegated to conftest.py fixtures) -------- +# orch comes from the conftest's orch fixture. Container lifecycle is +# owned by the fixture (setup_containers on entry, teardown on exit). + +# -------- Phase 3: generated tests -------- +# Parametrization is driven by pytest_generate_tests in conftest.py, +# walking variant_config.benchmark_params (or model_params for training). + +def test_(orch, variant_config, hf_token, cell, inf_res_dict): + job = Job(orch=orch, config=variant_config, hf_token=hf_token) + job.stop() # idempotent pre-clean + job.start(cell) # framework verbs + job.wait_ready() + job.run(cell) + job.wait_complete() + actuals = job.parse_results() + inf_res_dict[cell.id] = actuals + evaluate_all(actuals, variant_config.thresholds, prefix=cell.id) + + +# -------- Phase 4: custom tests -------- +# Suite-specific assertions outside the sweep grid. Use orch, not phdl. + +def test_print_results_table(inf_res_dict): + from cvs.tests..._shared import print_table + print_table(inf_res_dict) + +# Training-flavored example (distributed quirk): +# def test_disable_firewall(orch): +# out = orch.exec("sudo service ufw stop || true") +# out = orch.exec("sudo ufw status") +# for node, text in out.items(): +# assert "inactive" in text.lower() or "disabled" in text.lower(), node +``` + +And the conftest that backs it: + +```python +# cvs/tests///conftest.py +import pytest +from cvs.lib.dtni.config_loader import load_variant, enumerate_variants +from cvs.core.orchestrators.factory import OrchestratorConfig, build_orchestrator + +def pytest_generate_tests(metafunc): + if "variant_config" in metafunc.fixturenames: + variants = enumerate_variants("cvs/input/dtni/") + metafunc.parametrize("variant_config", variants, ids=[v.id for v in variants], indirect=True) + if "cell" in metafunc.fixturenames: + # Cross-product of sweep dims from the already-resolved variant_config. + ... + +@pytest.fixture(scope="module") +def cluster_dict(pytestconfig): ... + +@pytest.fixture +def variant_config(request, cluster_dict): + return load_variant(request.param, cluster_dict) + +@pytest.fixture +def orch(variant_config, cluster_dict): + oc = OrchestratorConfig.from_dicts(cluster_dict, variant_config.container.dict()) + o = build_orchestrator(oc) + o.setup_containers() + try: + yield o + finally: + o.teardown_containers() + +@pytest.fixture(scope="session") +def inf_res_dict(): + return {} +``` + +Conventions baked into this skeleton: + +- The wrapper does not import `docker_lib`, `parallel_ssh_lib`, or `globals`. Anything those modules did is now reachable through `orch` or `evaluate_all`. +- The wrapper does not `json.load` anything. Config IO lives in `config_loader`. +- Job verbs are framework-specific but consistent in *shape*: a small constructor, a few verbs that drive remote work through `orch`, and a `parse_results()` that returns a flat dict. The exact verbs differ by domain (inference: `start_server/run_client`; training: `start_training/poll_for_completion`). +- `pytest_generate_tests` is the only place parametrization lives. No ad-hoc `@pytest.mark.parametrize` on the workload test. + +## 5. The three new concepts + +### `orch` (the orchestrator fixture) + +**What.** An object from `cvs/core/orchestrators/factory.py` that abstracts "run a command on cluster nodes" and, when `container.enabled=true`, "run that command inside a container managed by me." Methods you'll touch: `setup_containers`, `teardown_containers`, `exec`, `exec_on_head`. Routing between baremetal and container is automatic. + +**Why.** Today every suite re-implements container launch and cleanup as ordered pytest functions plus an autouse fixture, calling `docker_lib` directly, with bonus shell workarounds (firewall, NIC scripts) reaching past it into `phdl.exec`. That couples test order to lifecycle order, duplicates teardown, and forces every wrapper to know about Docker flags. `orch` collapses this to one fixture: the test sees a ready environment when it starts and a clean one when it ends. + +### The Job class + +**What.** A standalone Python class, one per framework, under `cvs/lib//_orch.py`. It bundles the framework-specific verbs and uses an injected `orch` for all remote execution. Domain shapes verb names: inference Jobs expose `build_server_cmd / start_server / wait_ready / run_client / parse_results`; training Jobs expose `build_training_cmd / start_training / poll_for_completion / parse_results`; pre-workload shell workarounds (NIC setup, firewall) become methods on the Job rather than ordered tests. + +**Why.** `InferenceBaseJob` (711 LOC) tried to be a shared base for every inference framework and ended up tangling vllm-shaped env vars into the base, with a dead distributed branch and silent skips in result verification. `MegatronLlamaTrainingJob` (834 LOC) avoided the base-class trap but mixed config parsing, command building, remote execution, output parsing, and pass/fail thresholds in one file. The new shape: small, flat, framework-specific Job; cluster talk via `orch`; thresholds via `evaluate_all`. No inheritance, no `globals.error_list`. + +### The config / threshold split + +**What.** The single suite config JSON splits into two files per variant directory: `config.json` (what to run — identity, paths, model, image, container, framework params, sweep dimensions) and `threshold.json` (did it pass — flat map of `.` to typed predicates). + +**Why.** Different churn rates (config flips with new models or images; thresholds drift with hardware/kernel/version moves), different ownership (suite author vs perf/release), different lifecycles (re-baseline thresholds without re-reviewing the whole suite). Splitting also makes `cvs list` granular at the variant level and removes v1's anti-pattern of encoding non-metric checks ("did the server start") as a numeric threshold. + +## 6. What the current lib/Job files try to do — and what to lift out + +Read this as a refactor map for the legacy files, not a critique. Use it to decide what lands in the new Job, what lands in shared machinery, and what stays in the old file for legacy suites. + +### `cvs/lib/inference/base.py` — `InferenceBaseJob` (711 LOC) + +Concerns currently mixed: + +| Concern | Current home | DTNI home | +|---|---|---| +| Container launch flags | base init pulls from `inference_dict['container_config']` | **orch** (passed through `container` block) | +| Server command build | `build_server_inference_job_cmd` | **Job** (framework-specific) | +| Remote process start | `start_inference_server_job` → `phdl.exec` | **Job** uses `orch.exec_on_head` | +| Health wait | `wait_for_inference_server_health` | **Job** (verb on the Job) | +| Client command build | `build_client_inference_job_cmd` | **Job** | +| Result parsing | `parse_inference_results` → `inf_res_dict` | **Job.parse_results** returns a flat dict | +| Threshold check | `verify_inference_results` (silent-skip bug) | **evaluate_all** (shared, predicate-typed) | +| Error accumulation | `globals.error_list` | plain `assert` + `evaluate_all` | + +What to lift out: container concerns (to `orch`), threshold checks (to `evaluate_all`), the global error list (delete entirely). What stays on the Job: framework verbs. Net: the new `VllmJob` is in the 200–300 LOC range instead of 711. + +### `cvs/lib/megatron_training_lib.py` — `MegatronLlamaTrainingJob` (834 LOC) + +Same split, training-flavored: + +| Concern | Current home | DTNI home | +|---|---|---| +| Per-model presets (`single_node`/`multi_node` × `gpu_type`) | `model_params_dict` indexed inside Job | **variant config** (one variant per preset) | +| NIC setup script execution | `exec_nic_setup_scripts` | **Job method**, calling `orch.exec` | +| Training command build | `build_training_job_cmd` | **Job** | +| Launch + poll | `start_training_job` + `poll_for_training_completion` | **Job** (drives via `orch`) | +| Log scan for errors | `scan_for_training_errors` | **Job.parse_results** returns flat metrics dict; **evaluate_all** gates pass/fail | +| Threshold check | `verify_training_results` | **evaluate_all** | +| Distributed-only workarounds (firewall) | extra ordered pytest function in distributed wrappers | **Job method or custom Phase-4 test using orch** | + +Optimizations specific to training: + +- The "single vs distributed" axis is a config dimension, not a separate suite. With one wrapper parametrized by variant directory, `llama3.1_8b_fp8_single` and `llama3.1_8b_fp8_distributed` are sibling variant dirs sharing one wrapper. +- `gpu_type` derived by `rocm-smi` probe in a fixture today; in DTNI it's either declared in the variant (`gpu_arch: mi300x`) or queried once via `orch.exec_on_head` and cached on `orch`. + +### `cvs/lib/docker_lib.py` and `cvs/lib/parallel_ssh_lib.py` + +Both are reachable from DTNI suites via `orch` indirection, but DTNI Job code should never import them directly. `parallel_ssh_lib` is a deprecated shim around `cvs/lib/parallel/pssh.py`; the orch already uses the new path. New suites referencing either of these by name should fail review. + +## 7. Config vs threshold: philosophy and concrete shape + +**Why split.** + +- Different churn rates. Configs change when you bring up a new model, image, or container layout. Thresholds change when hardware, kernels, or framework versions move performance characteristics. +- Different ownership. Suite author owns config; perf or release engineering often owns thresholds. +- Different lifecycle. Re-baselining thresholds for a new MI generation should not require touching what the run does. +- Granular `cvs list`. One variant directory = one collected suite row, regardless of how many metrics it gates. +- Rejects v1's anti-pattern of encoding "did the thing start" as `min: 1` against some token counter. Liveness belongs in `assert` inside the test; thresholds are for numeric pass/fail on real measurements. + +### Example: `config.json` for a `vllm_single` variant + +```json +{ + "schema_version": 1, + "framework": "vllm_single", + "gpu_arch": "mi355x", + "paths": { + "shared_fs": "/mnt/dtni/{user-id}/cvs", + "models_dir": "/mnt/dtni/{user-id}/models", + "datasets_dir": "{shared_fs}/datasets", + "artifacts_dir": "{shared_fs}/artifacts" + }, + "model": {"id": "Qwen3-Next-80B-A3B-Instruct", "remote": 0, "precision": "bf16"}, + "image": {"tag": "rocm/vllm:latest", "remote": 1}, + "container": { + "enabled": true, + "launch": true, + "name": "vllm_inference_rocm", + "runtime": { + "name": "docker", + "args": { + "volumes": {"{paths.models_dir}": "/models"}, + "env": {"VLLM_USE_TRITON_FLASH_ATTN": "0"}, + "shm_size": "64G" + } + } + }, + "params": {"tensor_parallelism": 8, "max_model_len": 8192, "gpu_memory_utilization": 0.85}, + "benchmark_params": { + "concurrency_levels": [16], + "sequence_combinations": [{"name": "balanced", "isl": 1024, "osl": 1024}] + } +} +``` + +Block walkthrough: + +- `schema_version`, `framework`, `gpu_arch` — identity. Loader picks the right Pydantic model and Job class. +- `paths` — substituted in fixed order: cluster (`{user-id}`) → self-reference (`{shared_fs}`) → cross-block (`{paths.X}` used elsewhere). +- `model`, `image` — typed objects with explicit `remote` flags. +- `container` — passed through to `OrchestratorConfig.container`. `launch: true` hands lifecycle to orch. +- `params` — framework server flags. Passthrough; the framework's Pydantic model decides what's allowed. +- `benchmark_params` — sweep dimensions. `pytest_generate_tests` reads these to parametrize. + +### Example: `threshold.json` for the same variant + +```json +{ + "smoke_request_latency_ms": {"kind": "max_ms", "value": 600000}, + "smoke_completion_tokens": {"kind": "min", "value": 1}, + + "balanced_conc16.request_throughput": {"kind": "min", "value": 0.5}, + "balanced_conc16.output_throughput": {"kind": "min_tok_s", "value": 50.0}, + "balanced_conc16.ttft_p95_ms": {"kind": "max_ms", "value": 60000}, + "balanced_conc16.tpot_p95_ms": {"kind": "max_ms", "value": 5000} +} +``` + +Walkthrough: + +- Flat namespace. Keys are `.` where `` matches the parametrize id or a synthetic name like `smoke`. +- Five predicate kinds: `min`, `max_ms`, `within`, `min_tok_s`, `min_ratio`. Each entry is `{kind, value, tolerance?}`. +- Missing entries are logged by `evaluate_all` but do not gate the run — adding new metrics stays cheap. + +### Anti-patterns to reject in review + +- A `config.json` key whose value is a pass/fail threshold (move it). +- A `threshold.json` entry that is actually a config flag (move it). +- A threshold that branches on hardware. Split into separate variant directories per `gpu_arch`. +- Placeholder substitution across files (a threshold value pulled from a config block). They have different lifecycles; do not couple them. +- A "smoke" threshold of `min: 1` standing in for "did the thing start." Liveness belongs in `assert` inside the test. +- Globbing multiple models into one config with `models: [...]`. Drops `cvs list` granularity. One model per variant directory. + +## 8. Porting checklist (suite-agnostic) + +1. **Inventory the source wrapper.** List every fixture, every test function, every key read from the config JSON, and every reach into `docker_lib`/`parallel_ssh_lib`/`globals`. Note which keys gate behavior vs gate pass/fail. +2. **Identify framework verbs.** Pull out the framework-specific calls in the legacy lib (`cvs/lib/inference/.py`, `cvs/lib/_training_lib.py`). These become the new Job's public surface. +3. **Classify each config key.** Each key is "what we run" (config), "did it pass" (threshold), or "cluster scaffolding" (already in the cluster file). If undecided, default to config; thresholds are only for numeric measurements with predicates. +4. **Lay out variant directories.** One directory per `(model, purpose)` or `(model, mode, purpose)` under `cvs/input/dtni//__/`. Full model IDs, no abbreviations. Single-vs-distributed is a *mode*, not a separate wrapper. +5. **Write the Job.** Standalone class under `cvs/lib//_orch.py`. Constructor takes the typed config and an `orch`. Do not inherit `InferenceBaseJob`. Do not import `globals`. +6. **Write the wrapper + conftest** following the skeleton in §4. One wrapper per suite. +7. **Verify against the pre-port baseline.** Run the old wrapper and the new one on the same hardware with the same model and confirm metrics are within noise. + +## 9. Verification template + +Every port PR should include: + +- `pytest --collect-only` on the new wrapper, expected count matches the variant × sweep grid. +- `cvs list ` showing the variant rows (run with `--config_file=dummy` if needed, mirroring `list_plugin.py`). +- An end-to-end hardware run that produces the HTML report and exits 0. +- Container lifecycle observation: `docker ps` before, during, and after the run, confirming the container is created by `orch.setup_containers()` and removed by `orch.teardown_containers()` — no stale containers remain. +- A negative test: deliberately tighten one threshold so it fails; confirm the HTML report flags the right cell and the exit code is non-zero. +- A diff against the pre-port baseline metrics, attached to the PR. + +## 10. Out of scope for DTNI v-PoC + +- Refactoring or fixing `cvs/lib/inference/base.py` (`InferenceBaseJob`) or `cvs/lib/megatron_training_lib.py`. Older suites still depend on them; leave them alone. +- `model.remote=1` HuggingFace download path. Only `remote=0` (pre-staged on shared FS) is wired up. +- Accuracy variants. Only `_perf` is in scope; `_accuracy` directories may exist but their evaluation pipeline is a follow-up. +- Sweep rework. The current `pytest_generate_tests`-style parametrization is preserved; a declarative sweep grammar is a follow-up. +- `globals.error_list` deletion. The new suites stop using it; the legacy import stays until the last suite ports. diff --git a/plans/inferencex-atom-cvs-automation-plan.md b/plans/inferencex-atom-cvs-automation-plan.md new file mode 100644 index 000000000..da2d87a24 --- /dev/null +++ b/plans/inferencex-atom-cvs-automation-plan.md @@ -0,0 +1,1086 @@ +# InferenceX ATOM — CVS automation implementation plan (DTNI-first) + +## 0. Branch state (`hnimrama/ix-atom-multinode`) — read this first + +This section records **what exists on the branch today** vs **what this plan targets**. Refresh when landing major phases. + + +| Area | Current on branch | Target (this plan) | +| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Suite name** | `inferencex_atom` | Same | +| **Driver** | `InferenceXAtomJob`: `params.driver=atom` → standalone ATOM; `vllm_atom` → vLLM PP coordinator + ATOM ROCm kernels; `sglang` → SGLang PP coordinator; `vllm` → interim uplift only | Same; **multinode PP=2 validation uses `vllm_atom` or `sglang`, not `atom`** | +| **Config layout (canonical)** | `cvs/input/config_file/inference/inferencex_atom/` — flat `.json` + `_threshold.json` (vLLM-style); inline `atom_args` / `serve_args` / `sglang_args` | **Source of truth** for lab + `cvs copy-config`. | +| **Cluster files** | `cvs/input/cluster_file/inferencex_atom_cluster.json` | Per run; 2+ hosts for multinode PP; container image pinned in variant config | +| **Shipped W1 variants** | Single-node: `mi300x/mi355x_inferencex-atom_deepseek-r1_fp8_{single,mtp3,baseline_sweep}`. Multinode PP=2: `*_distributed`, `*_baseline_sweep_distributed` (`driver=vllm_atom`); `*_sglang_distributed` (`driver=sglang`) | + remaining W1–W18 stems (Section 3.1); single-node parity triple (M4) | +| **Interim uplift** | `mi300x/mi355x_inferencex-atom_gpt-oss-120b_bf16` (`driver: vllm`, record-only) | Replaced by W2 ATOM stems in M3 | +| **Thresholds** | MI300X W1 perf + multinode PP keys (`PP=2,NNODES=2`); baseline sweep multinode seeds need **lab recalibration** after true PP runs. MI355X: `enforce_thresholds: false` until lab confirm | Per-arch lab calibration; never cross-arch copy | +| **Accuracy (gsm8k)** | Not implemented | M2 — Section 5 (ACC-1..7) + Phase D | +| **Platform metrics** | `lifecycle.record` only (server_ready, client_complete) | `server.*` + sweep summary — Section 6.1, CVS-2/10 | +| **MTP** | W1 `*_mtp3` flat stems + thresholds seeded; MTP3 `atom_args` and `bench_extra_args` inline in config | Speculative-token flags preserved on serve restart via inline config | +| **Shared suite helpers** | `inference_suite_lifecycle.py`, `inference_suite_results_table.py`, `unittests/fake_orch.py` | Documented in variant `README.md` | +| **Multi-node (M5)** | **In repo:** `test_setup_sshd`, container sshd fail-fast, `params.nnodes` + PP orchestration for `vllm_atom`/`sglang`, W1 multinode configs + `scaling.efficiency_pct`. **Lab:** recalibrate thresholds on true PP=2 runs | Extend to N-node + single-node parity triple (M4); ATOM SPMD DP path for scale-out without PP remains optional on `driver=atom` | + + +**Branch implication:** Phase **R** and Phase **0** are **largely done** (ATOM serve + bench + W1 dirs + cluster JSON). **M5 multinode PP configs and Job hooks are landed** on `hnimrama/ix-atom-multinode`; lab must set `container.image`, `roles.server.ib_netdev`, and `params.master_addr` before enforcing gates. Active work: **lab recalibration** on true PP=2 → **M4 parity** (single-node vLLM/SGLang triple) → gsm8k (M2). + +--- + +## 1. Purpose and scope + +This document is the **implementation and action-item plan** for **InferenceX ATOM** automation in CVS. Work is tracked against the **DTNI Validation Tracker (IX ATOM)** spreadsheets — not the older W1–W16 Qwen/GLM/Kimi list in earlier drafts of this plan. + +**Normative references** + +- `plans/dtni-dev-guide.md` — pytest phases, `orch`, Job shape, `load_variant`, `evaluate_all`. +- **DTNI Validation Tracker (IX ATOM)** — framework paths, workload list, priorities, automation status (39 framework tests; **192 workload cases** in the matrix). +- **DTNI Validation Tracker (IX ATOM Matrix)** — workload legend **W1–W18** × performance metric coverage (`Y/P` = yes / planned for every cell). + +**In scope (InferenceX focus)** + +- **IX paths:** vLLM (ROCm) baseline, SGLang (ROCm) baseline, **ATOM**, **ATOM + MTP**, **ATOM-Disagg** (when orchestration allows). +- **Workloads:** W1–W18 recipes aligned with `amd-master.yaml` / InferenceX ATOM (Section 3). +- **Metrics:** Per-GPU throughput, output throughput per GPU, TTFT/TPOT (mean + tails), prefill/E2E, sweep curves, goodput, scaling — Section 6 + **Section 6.1** tiers. +- **Quality:** gsm8k and MTP accuracy tests — Section 5; optional quant parity (P2). +- **Platform:** CVS enhancements from inferencex_atom — Section 1.6. +- **Multi-node scaling:** **P1 milestone M5** — immediately after framework parity (M4), before broad MTP+P2 widen (M6), whenever cluster hardware and the suite’s upstream recipe support `nnodes>1` (Section 1.7). +- **Lab:** **Thor2 NIC first**; AINIC documented when available. See **Section 3.1** — **MI300X and MI355X are both in scope** even though the validation tracker rows are mostly MI355X-labelled. + +**GPU platforms (MI300X + MI355X)** + +The DTNI Validation Tracker names many recipes with **MI355X** in the title (e.g. W1 `dsr1-fp8-mi355x-atom`). **This plan still requires MI300X automation** for the same workload cards wherever the model fits on 8× MI300X. Tracker omission is **not** an out-of-scope signal for MI300X. + +- **Variant naming:** Same flat stem as `vllm_single`: `{gpu}_inferencex-atom_{model}_{precision}[_{mode}]` (e.g. `mi300x_inferencex-atom_deepseek-r1_fp8_single`). +- `**gpu_arch`:** `mi300x` or `mi355x` in config; **separate `threshold.json` per arch** — never share thresholds across GPUs. +- **Cluster files:** `input/cluster_file/mi300x_*.json` and `mi355x_*.json` (or equivalent) matched to variant `gpu_arch`. +- **Implementation:** Ship `_mi300x_` and `_mi355x_` variant dirs together in code/config PRs when possible. **Lab validation** follows hardware: MI300X runs gate milestones; MI355X lab runs are **pending when hardware is available** and do not block the MI300X spine (see **Section 1.2**). +- **Calibration / lab:** MI300X lab numbers in Section 4.1–4.2; **MI355X W1** numbers from upstream **ROCm/ATOM** nightly benchmark run in Section 4.3. **Never copy MI300X → MI355X** (or vice versa) for thresholds. + +**Current milestone scope (M1):** Phase **0 done** on branch; **Phase A** W1 perf on `*_mi300x_*` variant dirs (Sections 4.1–4.2, `enforce_thresholds: true` after lab confirm with branch code — Section 1.5). `*_mi355x_*` dirs ship with Section 4.3 threshold **seeds** and `enforce_thresholds: false` until MI355X lab is available — **not a blocker for M1 close or M2+ on MI300X**. + +### 1.2 Lab hardware policy — MI355X pending (non-blocking) + +When MI355X nodes are **not** available in the lab: + + +| Track | Policy | +| --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **MI300X (active)** | Gates milestones: Phase A lab confirm → M1 close on MI300X → M2 gsm8k on MI300X → M3 P1 workloads on MI300X → M4 parity → **M5 multi-node when hardware available**. | +| **MI355X (pending)** | Keep variant dirs, cluster JSON, and CI-seeded `threshold.json` in tree. Leave `enforce_thresholds: false`. No lab run required to merge PRs or advance M2/M3 on MI300X. | +| **When MI355X lands** | Run confirming CVS per variant → flip `enforce_thresholds: true` per arch → attach HTML/logs to PR. Does not require re-doing MI300X work. | + + +**Repo rule:** MI355X configs must never block CI or pytest collection on a machine without MI355X — only the variant you pass to `cvs run` is exercised. + +### 1.3 Config layout — canonical paths + + +| Path | Role | +| ------------------------------------------------------------------------ | ---------------------------------------------------------------------- | +| `cvs/input/config_file/inference/inferencex_atom/` | **Canonical** flat `*.json` + `*_threshold.json` pairs for lab and `cvs copy-config` | +| `cvs/input/config_file/inference/inferencex_atom/README.md` | Smoke vs perf vs multinode PP runbook, driver matrix, MI355X pending note | +| `cvs/input/cluster_file/inferencex_atom_cluster.json` | Example cluster (edit `node_dict` to 1 or 2+ hosts per variant) | + + +Legacy InferenceMax configs, nested variant subdirs, and the deprecated `inferencemax` suite are **removed**; all work uses flat `inferencex_atom/` stems (filename pattern `{gpu}_inferencex-atom_{model}_{precision}[_{mode}]`). + +### 1.4 ATOM benchmark artifact → CVS metrics contract + +ATOM `benchmark_serving` writes a stock JSON results file. CVS maps it through `to_client_metrics` into the `client.*` namespace used by `test_cell_metrics` and `evaluate_all`. + + +| Topic | Behavior | +| ----------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Namespace (interim)** | All ATOM perf scalars are `client.` today (Phase B may add IX-native keys for baselines only). | +| `**metric_percentiles`** | W1 configs use `"99"`. Benchmark emits **p99** (and mean/median) for ttft/tpot/itl/e2el — **not** p90/p95 unless percentiles string is expanded. | +| **GATED_METRICS vs artifact** | Loader requires a threshold spec for every `GATED_METRICS` member per cell. `test_cell_metrics` batches enforcement by tier (throughput, ttft, tpot, health); `evaluate_all` fails loudly on missing scalars when enforcing. | +| **Health gates (W1 perf)** | MI300X perf: `success_rate ≥ 1`, `failed ≤ 0` when `enforce_thresholds: true` (pairs with `bench_max_failed_requests: 0`). | +| `**failed` / `success_rate`** | ATOM JSON often omits `failed` when all prompts succeed. Parser derives `failed = num_prompts - completed` and then `success_rate`. | +| **Primary M1 gates** | `client.output_throughput`, `client.mean_ttft_ms`, `client.mean_tpot_ms` (+ p99 tails where emitted). | + + +### 1.5 Lab operations (not optional for valid results) + + +| Step | Why | +| ------------------------------------------------------------- | ------------------------------------------------------------------------------------ | +| `pip install -e .` (or editable install of branch) on runner | Installed `site-packages/cvs` shadows repo fixes; lab must run the branch under test | +| `cvs copy-config` variant + threshold + cluster to `~/input/` | Resolves `{user-id}` placeholders; edit cluster IPs locally | +| Archive `--html`, `--log-file`, and per-test HTML bundle | PR evidence for IX-atom; run card fields in Section 8 A-3 | +| Rotate HF token if captured in logs | Server env export may appear in verbose pytest capture | + + +### 1.6 CVS platform enhancements (inferencex_atom backlog) + +Work below improves **CVS as a validation platform**, not only W1. Prioritize items that unblock M2/M3 lab velocity and parity with upstream ATOM CI. + + +| ID | Enhancement | CVS benefit | Phase | +| ---------- | ------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------ | +| **CVS-1** | `**accuracy.*` metric namespace** | Separate quality scalars from `client.*` perf; reuse `evaluate_all` with new threshold kinds (`min_ratio`, `min_exact_match`) | D | +| **CVS-2** | `**server.*` lifecycle metrics** | Emit `server.time_to_ready_s`, `server.warmup_s`, `server.model_cache_bytes` from existing `lifecycle.record` + model `du` probe — gate regressions in load path | B | +| **CVS-3** | **Run card in HTML report** | Surface `gpu_arch`, `atom_args` summary, `atom_image_pin`, `upstream_run_url` as pytest metadata (today: log-only via `_log_variant_run_card`) | B | +| **CVS-4** | **Default `params.driver=atom`** | Schema default still `vllm`; flip default to `atom` once interim uplift variants are isolated | 0+ | +| **CVS-5** | **Inline config validation** | Schema requires `roles.server.atom_args` when `driver=atom`; extend to warn on image pin mismatch vs `run_card.atom_image_pin` | 0-1 | +| **CVS-6** | **MTP orch wiring** | MTP3 `atom_args` / `bench_extra_args` inline in variant config; orch must not drop speculative-token flags on serve restart | G | +| **CVS-7** | **Artifact bundle export** | Zip `results.json`, server log tail, run card JSON per cell into CVS HTML bundle for PR diff vs ATOM CI | A-3 | +| **CVS-8** | **Upstream parity diff** | Script: compare CVS `client.`* per cell to Section 4 reference within margin; flags threshold drift before merge | A | +| **CVS-9** | `**placeholder_gated_threshold_cell` generator** | CLI or doc recipe to mint threshold skeletons for new W2–W18 dirs (already in `inferencex_atom_config_loader.py`) | C | +| **CVS-10** | **Sweep curve aggregation** | Post-run table/chart: throughput vs concurrency per ISL/OSL (tracker metric #30); no new pytest per point | B | +| **CVS-11** | **Baseline variant pairing** | Same sweep cell for `*_atom_perf` vs `*_vllm_baseline` → parity HTML section (M4) | C / M4 | +| **CVS-12** | **Secret redaction** | Strip `HF_TOKEN` from captured server env / verbose logs in pytest hooks | 1.5 | +| **CVS-13** | **Percentile policy switch** | Config `metric_percentiles: "90,95,99"` when tracker gates p90/p95; else keep skip-when-absent (Section 1.4) | B-4 | +| **CVS-14** | **Multi-node `nnodes` in Job** | Extend `InferenceXAtomJob` (then parity Jobs) for M5 scaling without new suite id; `test_setup_sshd` + `scaling.*` gates | **M5** | +| **CVS-15** | **DTNI mirror sync** | Single copy step from `config_file/inferencex_atom/` → `dtni/` when packaging converges | DOC | + + +```mermaid +flowchart LR + subgraph perf["Perf path today"] + BENCH["benchmark_serving"] + CM["client.*"] + TM["test_cell_metrics"] + end + subgraph add["CVS additions"] + SV["server.* lifecycle"] + AC["accuracy.* gsm8k"] + RC["run_card HTML"] + BD["bundle + CI diff"] + end + BENCH --> CM --> TM + SV --> TM + AC --> EVAL["evaluate_all"] + CM --> EVAL + RC --> HTML["pytest HTML"] + BD --> PR["PR evidence"] +``` + + + +**Explicitly out of scope for early waves** + +- Full **Optimus / KVMGR / NIXL / hipFile / MaaS / Gateway** automation — **Appendix B** only. +- New gates via legacy `InferenceBaseJob.verify_inference_results`. + +### 1.7 Multi-node priority — PP=2 via framework coordinators (M5) + +Multi-node **pipeline parallel** (PP=2) is a **P1 M5** deliverable. Standalone ATOM has **no native PP engine**; multinode PP validation uses a **framework coordinator** while ATOM (or SGLang) accelerates local kernels. + +| Track | Policy | +| ----- | ------ | +| **Architecture** | **True PP=2:** `params.driver=vllm_atom` (`vllm serve` + `--pipeline-parallel-size` + `--node-rank`) or `params.driver=sglang` (`sglang.launch_server` + `--pp-size` + `--dist-init-addr`). **Not PP:** `driver=atom` multinode uses ATOM SPMD **data parallel** (`-dp`, cell keys `DP=`) for scale-out / P-D disagg — do not label as pipeline parallel. | +| **When to start** | Configs + Job hooks **landed** on `hnimrama/ix-atom-multinode`. Lab confirm + threshold recalibration required before `enforce_thresholds: true` on multinode stems. | +| **Hardware gate** | Cluster file with **2+ nodes**, inter-node SSH (`test_setup_sshd`, sshd in container image — no runtime `apt-get`), `roles.server.ib_netdev`, vLLM+ATOM or SGLang container image. | +| **Suite scope (shipped)** | `mi300x_inferencex-atom_deepseek-r1_fp8_distributed` + `*_baseline_sweep_distributed` (`driver=vllm_atom`, `PP=2`); `mi300x_inferencex-atom_deepseek-r1_fp8_sglang_distributed` (`driver=sglang`). MI355X multinode: `enforce_thresholds: false` until lab. | +| **Does not block** | M1–M4 single-node work, gsm8k (M2), or P1 workload stems (M3). | +| **Deliverables (M5)** | Done in repo: `params.nnodes`, PP orchestration, multinode configs, `scaling.efficiency_pct`, sshd fail-fast. Pending lab: recalibrated `threshold.json`, run card fabric metadata (F-7). | + +**Repo rule:** Single-node cluster JSON and pytest collection must keep working when `nnodes=1` — only the variant + cluster file passed to `cvs run` exercises distributed paths. + +### 1.1 Diagrams — CVS entry and DTNI inputs + +```mermaid +flowchart LR + subgraph entry["Entry"] + CLI["cvs run inferencex_atom"] + CLUSTER["cluster JSON"] + end + subgraph variant["Variant (flat)"] + DIR["inferencex_atom/"] + CONFIG[".json"] + THRESH["_threshold.json"] + end + subgraph pytest["Pytest"] + CONFT["conftest: orch + load_variant"] + GEN["pytest_generate_tests"] + TEST["per-cell workload test"] + end + subgraph gate["Gate"] + JOB["InferenceXAtomJob → ATOM backend"] + PARSE["parse_results"] + EVAL["evaluate_all"] + end + CLI --> CONFT + CLUSTER --> CONFT + DIR --> CONFIG + DIR --> THRESH + CONFT --> JOB + GEN --> TEST + TEST --> JOB + JOB --> PARSE + PARSE --> EVAL + THRESH --> EVAL +``` + + + +InferenceX paths under automation: + +```mermaid +flowchart TB + subgraph baselines["Baselines P1"] + VLLM["vLLM ROCm"] + SGL["SGLang ROCm"] + end + subgraph atom["ATOM track"] + ATOM["ATOM serve"] + MTP["ATOM + MTP"] + DIS["ATOM disagg"] + end + VLLM --> CMP["parity tables same ISL/OSL/conc"] + SGL --> CMP + ATOM --> CMP + MTP --> MTPQ["chat template + perf uplift"] + DIS --> BL["blocked: SLURM or orch spike"] +``` + + + +--- + +## 2. DTNI alignment (non-negotiable for new work) + + +| DTNI guide concept | InferenceX ATOM application | +| ----------------------- | -------------------------------------------------------------------------------------------------------------------------- | +| **Load** | Each variant = typed config via `**load_variant`** (`InferenceXAtomVariantConfig` or DTNI Pydantic equivalent). | +| **Setup** | Module-scoped `**orch`**: `setup_containers` on entry, `teardown_containers` on exit. | +| **Generated tests** | `**pytest_generate_tests`** builds sweep cells (`sequence_combinations` + explicit `runs[]`). | +| **Workload test** | `**InferenceXAtomJob(orch, variant, hf_token)`** — verbs then `**parse_results()`** → flat metrics for `**evaluate_all**`. | +| **Verification** | `**evaluate_all`** against `**threshold.json`** per cell (`ISL=…,OSL=…,TP=…,CONC=…`). | +| **Config vs threshold** | **Run recipe** in config; **pass/fail only** in threshold. | +| **Job class** | Standalone job using `**orch` only** — no `InferenceBaseJob` for new ATOM gates. | + + +### 2.1 Execution backends — `atom`, `vllm_atom`, `sglang`, `vllm` + +| `params.driver` | Server | Client | Multinode PP | +| --------------- | ------ | ------ | ------------ | +| **`atom`** | `atom.entrypoints.openai_server` | `atom.benchmarks.benchmark_serving` | **No native PP.** Optional SPMD DP (`-dp` + `ATOM_DP_*`; cell keys `DP=`). | +| **`vllm_atom`** | `vllm serve` + ROCm ATOM env | `vllm bench serve` | **Yes** — vLLM injects `--pipeline-parallel-size`, `--node-rank`, `--headless` on workers. | +| **`sglang`** | `sglang.launch_server` | `sglang.bench_serving` | **Yes** — SGLang injects `--pp-size`, `--nnodes`, `--dist-init-addr`. | +| **`vllm`** | `vllm serve` (uplift) | `vllm bench serve` | Same coordinator flags as `vllm_atom` without ATOM-specific env block. | + +```mermaid +flowchart LR + subgraph today["W1 single-node"] + ATOM["driver=atom"] + BENCH["benchmark_serving"] + CM["client.* via to_client_metrics"] + end + subgraph m5["M5 multinode PP=2"] + VA["driver=vllm_atom"] + SG["driver=sglang"] + PP["PP=2 cell keys"] + end + subgraph uplift["Interim only"] + VJ["driver=vllm GPT-OSS uplift"] + end + ATOM --> BENCH --> CM + VA --> PP + SG --> PP +``` + +**Default for W1 single-node perf:** `params.driver=atom`. **Default for multinode PP validation:** `params.driver=vllm_atom` or `sglang` — never `atom` with `pipeline_parallel_size>1`. + + + +--- + +## 3. Workload legend (W1–W18) — from Validation Tracker + +Authoritative **model / ISL / OSL / precision** mapping from **DTNI Validation Tracker (IX ATOM Matrix)**. Each workload becomes **variant directories per GPU** (Section 3.1): mode suffixes `_atom`, `_atom_mtp`, `_vllm_baseline`, etc. + + +| ID | Model / recipe | HF id (tracker) | TP | Precision | Tracker ISL/OSL | Priority | +| ------- | ------------------ | --------------------------------------------------------------------------------------------------- | --- | --------- | --------------- | -------- | +| **W1** | DeepSeek R1 FP8 | `dsr1-fp8-mi355x-atom` (tracker); **MI300X:** `dsr1-fp8-mi300x-atom` (IX sibling — confirm in repo) | 8 | FP8 | 1K / 1K | **P1** | +| **W2** | GPT-OSS-120B | `openai/gpt-oss-120b` | 4 | MXFP4 | 8K / 1K | **P1** | +| **W3** | GLM 5.1 | `zai-org/GLM-5.1` | 8 | BF16 | 1K / 8K | **P1** | +| **W4** | GLM 5.1 FP8 | `zai-org/GLM-5.1-FP8` | 8 | FP8 | 1K / 4K | P2 | +| **W5** | DeepSeek V4 Pro | `deepseek-ai/DeepSeek-V4-Pro` | 8 | FP4+FP8 | 5000 / 1024 | P2 | +| **W6** | DeepSeek V4 Flash | `deepseek-ai/DeepSeek-V4-Flash` | 4 | FP4+FP8 | 1K / 1K | P2 | +| **W7** | Kimi K2.6 Thinking | `uniquealexx/Kimi-K2.6-Thinking-200x` | 4 | INT4 | 1K / 1K | P2 | +| **W8** | GLM 5 MXFP4 | `amd/GLM-5-MXFP4` | 8 | MXFP4 | 1K / 1K | P2 | +| **W9** | Kimi K2.5 MXFP4 | `amd/Kimi-K2.5-MXFP4` | 4 | MXFP4 | 1K / 1K | P2 | +| **W10** | Qwen 3.5 397B | `Qwen/Qwen3.5-397B-A17B` | 8 | BF16 | 1K / 1K | P2 | +| **W11** | GLM 5.2 FP8 | `zai-org/GLM-5.2-FP8` | 8 | FP8 | 1K / 1K | P2 | +| **W12** | GLM 5.2 | `zai-org/GLM-5.2` | 8 | BF16 | 1K / 1K | P2 | +| **W13** | Kimi K2.7 Code | `moonshotai/Kimi-K2.7-Code` | 8 | BF16 | 1K / 1K | **P1** | +| **W14** | MiniMax M3 | `MiniMaxAI/MiniMax-M3` | — | BF16 | 1K / 1K | P2 | +| **W15** | Qwen 3.5 MXFP4 | `amd/Qwen3.5-397B-A17B-MXFP4` | 8 | MXFP4 | 1K / 1K | P2 | +| **W16** | Mistral Large 3 | `mistralai/Mistral-Large-3-675B-Instruct-2512` | 8 | FP8 | 1K / 1K | P2 | +| **W17** | DeepSeek R1 MXFP4 | `amd/DeepSeek-R1-0528-MXFP4` | 8 | MXFP4 | 1K / 1K | **P1** | +| **W18** | MiMo v2.5 Pro | `XiaomiMiMo/MiMo-V2.5-Pro` | 8 | BF16 | 1K / 1K | P2 | + + +**P1 workloads for first automation wave:** W1, W2, W3, W13, W17 (five models) plus framework paths (ATOM, ATOM+MTP, ATOM-Disagg, **`params.driver=vllm_atom`**, **`params.driver=sglang`**). + +**MTP variants:** For workloads that have `*-atom-mtp` recipes in InferenceX, treat **FP8 + MTP3** (and similar) as **sibling variant dirs** or `roles`/recipe flags — not a different suite id. Chat-formatted prompts required per InferenceX AGENTS.md. + +### 3.1 GPU platform coverage (MI300X + MI355X) + +The tracker matrix does **not** list MI300X explicitly. **CVS automation does.** Every workload in scope ships as **one or more variant dirs per `gpu_arch`** when the model is supported on that hardware. + +**Implementation priority — MI300X leads lab; MI355X code ships in parallel** + + +| Platform | Code / config (M1+) | Thresholds / lab | Lab status | +| ---------- | -------------------------------------------- | ------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------- | +| **MI300X** | Ship with every P1 workload | Section 4.1–4.2 (internal lab reference) | **Active** — gates M1/M2/M3 on available hardware | +| **MI355X** | Ship variant dirs + cluster JSON with MI300X | Section 4.3 ([ROCm/ATOM run 27912164002](https://github.com/ROCm/ATOM/actions/runs/27912164002)) | **Pending** — CI seeds only until nodes available; does not block MI300X milestones | + + +**P1 target — dual variant dirs per workload (MI300X + MI355X)** + + +| Workload | MI300X variant | MI355X variant | Notes | +| ------------------------- | -------------------------------------------------------- | ---------------------------------------------- | --------------------------------------------------------------------------------- | +| **W1** DeepSeek R1 FP8 | `mi300x_inferencex-atom_deepseek-r1_fp8_{single,mtp3}` | `mi355x_inferencex-atom_deepseek-r1_fp8_{single,mtp3}` | MTP3: **post-M1** optional on MI300X | +| **W2** GPT-OSS MXFP4 | `mi300x_inferencex-atom_gpt-oss-120b_mxfp4` (target) | `mi355x_inferencex-atom_gpt-oss-120b_mxfp4` (target) | Interim `mi300x_inferencex-atom_gpt-oss-120b_bf16` is **not** final W2 | +| **W3** GLM 5.1 BF16 | `glm51_mi300x_atom` | `glm51_mi355x_atom` | Same ISL/OSL as tracker | +| **W13** Kimi K2.7 Code | `kimi_k27_code_mi300x_atom` | `kimi_k27_code_mi355x_atom` | | +| **W17** DeepSeek R1 MXFP4 | `deepseek_r1_mxfp4_mi300x_atom` | `deepseek_r1_mxfp4_mi355x_atom` | gsm8k ≥ 0.93 on MXFP4 | + + +**P2 workloads:** `_mi300x_` and `_mi355x_` dirs together when each workload is automated. + +**Baselines (parity engines):** Per workload × `gpu_arch` — same `inferencex_atom` framework with `params.driver` = `atom` / `vllm_atom` / `sglang` (Section 12.3), not legacy `vllm_single` / SGLang disagg. + +**Run card fields:** `gpu_arch`, GPU count, IX recipe id, image tag, NIC, IX SHA — comparable dashboards, separate thresholds per arch. + +--- + +## 4. Reference performance (calibration seeds) + +W1 (**DeepSeek R1 FP8**, ISL=OSL=1024, TP8, FP8 KV cache). Use to seed per-arch `threshold.json` after margin policy is agreed (typically reference × guard band, not raw copy). + +**ATOM bench JSON → CVS threshold keys** (today: all prefixed `client.` in threshold files; see Section 1.4): + + +| ATOM artifact field | CVS threshold key (current) | M1 gate? | +| ------------------------------- | -------------------------------------- | -------------------------------------------- | +| `output_throughput` | `client.output_throughput` | **Yes** | +| `total_token_throughput` | `client.total_token_throughput` | Loose / placeholder | +| `mean_ttft_ms` | `client.mean_ttft_ms` | **Yes** | +| `mean_tpot_ms` | `client.mean_tpot_ms` | **Yes** | +| `p99_ttft_ms`, `p99_tpot_ms`, … | `client.p99_`* | Emitted when `metric_percentiles: "99"` | +| `p90_*`, `p95_*` | `client.p90_*`, `client.p95_*` | Placeholder only unless percentiles expanded | +| (derived) | `client.failed`, `client.success_rate` | Derived when `failed` omitted | + + +### 4.1 MI300X — FP8 (lab reference) + +8× MI300X, ATOM, DeepSeek R1 FP8, TP8, FP8 KV cache. + + +| Concurrency | Output throughput (tok/s) | Total throughput (tok/s) | Mean TPOT (ms) | +| ----------- | ------------------------- | ------------------------ | -------------- | +| 128 | 4,274 | 8,558 | 28.8 | +| 256 | 6,039 | 12,071 | 40.8 | + + +### 4.2 MI300X — FP8 + MTP3 (lab reference) + +8× MI300X, ATOM, DeepSeek R1 FP8 + MTP3, TP8, FP8 KV cache, 3 speculative tokens. + + +| Concurrency | Output throughput (tok/s) | Total throughput (tok/s) | Mean TPOT (ms) | +| ----------- | ------------------------- | ------------------------ | -------------- | +| 128 | 6,913 | 13,856 | 17.5 | +| 256 | 7,284 | 14,583 | 33.0 | + + +### 4.3 MI355X — from ROCm/ATOM CI (W1 seeds) + +Source: [ROCm/ATOM ATOM Benchmark run 27912164002](https://github.com/ROCm/ATOM/actions/runs/27912164002) (also mirrored on [benchmark dashboard](https://rocm.github.io/ATOM/benchmark-dashboard/)). Job summary: [summarize step raw markdown](https://github.com/ROCm/ATOM/actions/runs/27912164002/jobs/65963327389/summary_raw) (GitHub login required). + + +| Field | Value | +| ----------- | ------------------------------------------------------------------- | +| Model | `deepseek-ai/DeepSeek-R1-0528` (ATOM display: **DeepSeek-R1-0528**) | +| GPU | **AMD Instinct MI355X**, 8× GPU, TP8 | +| Image | `rocm/atom-dev:nightly_202606211542` | +| ROCm | 7.2.4 | +| ATOM commit | `ea08015` | + + +#### 4.3.1 FP8 — ISL=1024, OSL=1024 + + +| Concurrency | Output throughput (tok/s) | Total throughput (tok/s) | Mean TPOT (ms) | Mean TTFT (ms) | +| ----------- | ------------------------- | ------------------------ | -------------- | -------------- | +| 128 | 4,449.62 | 8,909.01 | 27.64 | 329.25 | +| 256 | 6,249.73 | 12,493.43 | 39.46 | 551.66 | + + +#### 4.3.2 FP8 + MTP3 — ISL=1024, OSL=1024 + + +| Concurrency | Output throughput (tok/s) | Total throughput (tok/s) | Mean TPOT (ms) | Mean TTFT (ms) | +| ----------- | ------------------------- | ------------------------ | -------------- | -------------- | +| 128 | 5,101.99 | 10,208.96 | 23.77 | 570.42 | +| 256 | 7,168.43 | 14,321.35 | 34.22 | 606.67 | + + +**Planning notes** + +- These four cells are the **MI355X W1 threshold candidates** (`mi355x_inferencex-atom_deepseek-r1_fp8_single` and `_mtp3` sibling). +- Re-pull from a newer ATOM nightly when image or `ea08015`+ moves; pin the run URL + docker tag in variant README / run card. +- MI300X (Sections 4.1–4.2) and MI355X (Section 4.3) numbers are **close but not identical** — keep separate `threshold.json` per `gpu_arch`. +- As other P1 workloads appear in ATOM CI, add sibling subsections here before enabling `enforce_thresholds: true` on those variants. + +--- + +## 5. Accuracy gates and quality tests + +Reference accuracy on **8 GPUs**, FP8, FP8 KV cache (W1 DeepSeek R1): + + +| Task | Version | Filter | n-shot | Metric | Value | Stderr | +| ----- | ------- | ---------------- | ------ | ----------- | ------ | ------ | +| gsm8k | 3 | flexible-extract | 5 | exact_match | 0.9553 | 0.0057 | +| gsm8k | 3 | strict-match | 5 | exact_match | 0.9538 | 0.0058 | + + +**CI thresholds (tracker policy)** + + +| Precision path | gsm8k flexible-extract minimum | +| -------------- | ------------------------------ | +| FP8 | **≥ 0.94** | +| MXFP4 | **≥ 0.93** | + + +### 5.1 Accuracy test catalog (what CVS should run) + +Accuracy is **not** a perf sweep cell. Each row is a **separate pytest stage** (or dedicated variant dir) that reuses the same ATOM server container after perf gates pass (or cold-starts once per accuracy job). + + +| Test id | Task / benchmark | When | Workloads | Gate? | Metric key (proposed) | +| --------- | ---------------------------------- | ----------------------------- | --------------------------------------- | ------------------- | ----------------------------------------- | +| **ACC-1** | **gsm8k** flexible-extract, 5-shot | **M2** — after M1 MI300X perf | W1 FP8, W17 MXFP4, other P1 quant paths | **Yes** | `accuracy.gsm8k_exact_match` | +| **ACC-2** | **gsm8k** strict-match, 5-shot | Same run as ACC-1 | W1+ | Record-only | `accuracy.gsm8k_strict_match` | +| **ACC-3** | **gsm8k stderr bound** | Optional nightly | W1 | Record-only | `accuracy.gsm8k_stderr` (flag if > 0.02) | +| **ACC-4** | **MTP acceptance rate** | Post-M1 MTP3 lab | W1 `*_mtp3` | P2 gate | `mtp.acceptance_rate` (min floor TBD) | +| **ACC-5** | **Degenerate decode check** | MTP variants | W1 MTP3+ | P2 gate | `mtp.empty_or_repeat_ratio` (max ceiling) | +| **ACC-6** | **MMLU** (5-shot, subset) | P2 nightly | W2–W3 code/reasoning models | Record → gate later | `accuracy.mmlu_acc` | +| **ACC-7** | **Quant logit parity** vs BF16 ref | P2 optional | FP8/MXFP4 paths | Record-only | `accuracy.logit_max_delta` | + + +**Per-workload gsm8k floors (tracker-aligned)** + + +| Workload | Precision | ACC-1 minimum (`flexible-extract`) | +| --------------------- | --------- | ---------------------------------------- | +| W1 DeepSeek R1 FP8 | FP8 | **0.94** | +| W17 DeepSeek R1 MXFP4 | MXFP4 | **0.93** | +| W2 GPT-OSS MXFP4 | MXFP4 | **0.93** (confirm with IX when W2 lands) | +| W3 GLM 5.1 BF16 | BF16 | **0.94** (BF16 reference path) | + + +### 5.2 Accuracy harness design (CVS integration) + +**Reference pattern:** SGLang disagg already runs gsm8k via `run_gsm8k_benchmark_test` (`sglang_disagg_lib.py`). InferenceX ATOM should follow the same **DTNI shape**: one job method + one pytest test, not perf parametrization. + + +| Step | Implementation | +| ---- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| 1 | Add `mi300x_inferencex-atom_deepseek-r1_fp8_accuracy` variant: single cell, `enforce_thresholds: true`, `num_prompts` N/A (accuracy-only). | +| 2 | Extend `InferenceXAtomJob` (or sibling `InferenceXAtomAccuracyJob`) with `run_gsm8k_eval()` — invoke **lm-eval** or ATOM-shipped eval inside container against `http://localhost:{port}`. | +| 3 | Parse eval JSON → flat `accuracy.`* dict; attach to `inf_res_dict` under a fixed accuracy key (not per-conc sweep). | +| 4 | Add `test_gsm8k_accuracy` in `inferencex_atom.py` (or `inferencex_atom_accuracy.py` module) — runs **after** perf tests when chained, or standalone via `--config_file` accuracy variant. | +| 5 | Threshold file: one global cell or `"accuracy"` key with `accuracy.gsm8k_exact_match: {kind: min, value: 0.94}`. | +| 6 | HTML row: add `ACCURACY_METRICS` list beside `CLIENT_METRICS` in `vllm_parsing.py` (or new `accuracy_parsing.py`). | + + +**CI job split** + + +| Job | Variant | Duration | Blocks merge? | +| -------- | ----------------- | ------------------------ | ------------- | +| Perf | `*_atom_perf` | Long (full sweep) | M1 | +| Smoke | `*_atom_smoke` | Short | Pre-gate only | +| Accuracy | `*_atom_accuracy` | Medium (~gsm8k full set) | M2 | + + +### 5.3 Accuracy metrics namespace + + +| Metric | Unit | Source | Threshold kind | +| ----------------------------- | --------- | --------------------------- | -------------------- | +| `accuracy.gsm8k_exact_match` | ratio 0–1 | lm-eval / ATOM eval | `min` | +| `accuracy.gsm8k_strict_match` | ratio 0–1 | same run, second filter | record-only | +| `accuracy.gsm8k_stderr` | ratio | eval stderr | `max` (optional) | +| `accuracy.samples_completed` | count | eval progress | `min` (= expected N) | +| `accuracy.eval_duration_s` | s | wall clock | record-only | +| `mtp.acceptance_rate` | ratio | ATOM MTP stats / log scrape | `min` (P2) | +| `mtp.speculative_tokens_avg` | count | MTP telemetry | record-only | + + +**Automation plan (summary)** — action items in **Phase D** (Section 8) and **CVS-1** (Section 1.6). + +- Accuracy is a **separate pytest stage** (not mixed into perf `test_cell_metrics` rows). +- Run after **M1** MI300X perf is green; MI355X accuracy pending with hardware (Section 1.2). +- Use a **dedicated variant stem** (e.g. `mi300x_inferencex-atom_deepseek-r1_fp8_accuracy`) with low concurrency / fixed eval split to limit wall time. +- Workload-specific ACC rows (W13 code, W2 long-context, etc.) — **Section 12.2**. + +--- + +## 6. Master metric matrix (framework + workloads) + +From **IX ATOM Matrix**: every workload row W1–W18 is marked **Y/P** for all core performance metrics below. CVS automation should eventually emit and gate (where P1) each metric per cell. + + +| # | Category | Test / Metric | Priority | Automation status | Notes | +| ---- | ----------- | ------------------------------------------------- | ------------ | ------------------- | ------------------------------------------- | +| 1 | IX Path | vLLM (ROCm) baseline | P1 | Not started | M4; interim GPT-OSS uplift only | +| 2 | IX Path | SGLang (ROCm) baseline | P1 | Not started | M4 | +| 3 | IX Path | ATOM (`params.driver=atom`) | P1 | **W1 in lab** | `inferencex_atom_orch.py` | +| 4 | IX Path | ATOM + MTP | P1 | **Configs shipped** | W1 `*_mtp3` dirs; orch recipe TBD | +| 5 | IX Path | ATOM-Disagg | P1 | Blocked | PD pools; SLURM spike | +| 6–23 | Workload | W1–W18 (Section 3) | P1/P2 | **W1 only** | 192 matrix cells total | +| 24 | Performance | Throughput per GPU (`tput_per_gpu`) | P1 | **W1 gated** | `client.per_gpu_throughput` = total/TP | +| 25 | Performance | Output throughput per GPU (`output_tput_per_gpu`) | P1 | **W1 gated** | `client.output_tput_per_gpu` = output/TP | +| 26 | Performance | TTFT mean & p99 | P1 | **W1 gated** | p99 via `metric_percentiles: "95,99"` | +| 27 | Performance | TPOT mean & p95 | P1 | **W1 gated** | p95 via `metric_percentiles: "95,99"` | +| 28 | Performance | Prefill latency p50 / p95 | P2 | Not started | | +| 29 | Performance | E2E mean / p95 / p99 | P2 | Partial | p99 emitted; p90/p95 record-only | +| 30 | Performance | Latency vs load (per sweep step) | P2 | Not started | | +| 31 | Performance | Goodput | P2 | Not started | | +| 32 | Performance | Scaling efficiency % | **P1** | Not started | **M5** after M4 parity; Section 1.7, 6.1 Tier 5 | +| 33 | Performance | Peak GPU memory | P2 | Not started | | +| 34 | Performance | KV cache footprint | P2 | Not started | | +| 35 | Performance | Request success rate & error mix | P2 | Partial | Derived `failed` / `success_rate` | +| 36 | Performance | Model load time + memory | P2 | Not started | | +| 37 | Performance | Time-to-ready | P2 | Partial | `wait_ready` + server warmup timing | +| 38 | Quality | MTP acceptance / degenerate decode | P2 | Not started | MTP workloads only | +| 39 | Quality | Quant / logit parity vs BF16 | P2 | Not started | Nightly optional | +| 40 | Quality | **gsm8k accuracy** | P1 (W1 gate) | Not started | M2 — Section 5 + Phase D | + + +**Tracker rollup (IX ATOM tab):** 39 framework tests — 14 P1, 25 P2; **W1 ATOM perf path automated on branch**; gsm8k and remaining workloads not yet automated; 192 workload cases in matrix. + +### 6.1 Recommended metrics for CVS (tiers and namespaces) + +Beyond the tracker matrix above, this is the **practical metric set** CVS should emit, display in HTML, and eventually gate. Maps to code in `vllm_parsing.py` (`CLIENT_METRICS`, `GATED_METRICS`, `to_client_metrics` derivations) and planned namespaces from Section 1.6. + +#### Tier 1 — Gate on every perf cell (P1, M1+) + + +| Metric | Namespace key | Producer today | Gate policy | +| ----------------- | ------------------------------------------ | ------------------------------------ | --------------------------------------------------- | +| Output throughput | `client.output_throughput` | ATOM `results.json` | **min_tok_s** — primary SLO | +| Per-GPU throughput | `client.per_gpu_throughput`, `client.output_tput_per_gpu` | derived `total/TP`, `output/TP` | **min_tok_s** on W1 perf cells | +| Mean TTFT | `client.mean_ttft_ms` | ATOM | **max_ms** | +| Mean TPOT | `client.mean_tpot_ms` | ATOM | **max_ms** | +| P99 TTFT / P95 TPOT | `client.p99_ttft_ms`, `client.p95_tpot_ms` | ATOM when `metric_percentiles: "95,99"` | **max_ms** when emitted | +| Run health | `client.failed`, `client.success_rate` | derived if `failed` omitted | **max** failed, **min** success_rate when enforcing | + + +#### Tier 2 — Record on every cell; gate when calibrated (P1/P2) + + +| Metric | Namespace key | Value to CVS | Why useful | +| ---------------------- | ------------------------------------------------------------- | ------------------------- | ----------------------------------------------- | +| Total token throughput | `client.total_token_throughput` | ATOM | Prefill+decode capacity; parity vs ATOM CI | +| Request throughput | `client.request_throughput` | ATOM if present | Goodput proxy at fixed conc | +| Goodput | `client.goodput` | ATOM `request_goodput` | SLA under rate limits | +| Median latencies | `client.median_ttft_ms`, `client.median_tpot_ms` | ATOM | Robust center vs mean skew | +| P99 ITL / E2E | `client.p99_itl_ms`, `client.p99_e2el_ms` | ATOM | Decode jitter + end-to-end tail | +| Decode diagnostics | `client.decode_latency_ratio`, `client.decode_throughput_p50` | derived | Spot unstable decode (p99/p50 ITL, median TPOT) | +| Normalized TTFT | `client.normalized_ttft_ms_per_tok` | derived `mean_ttft / ISL` | Compare across ISL sweep steps | +| Bench duration | `client.duration` | ATOM | Wall time per cell for CI budgeting | +| Token totals | `client.total_input_tokens`, `client.total_output_tokens` | ATOM | Sanity vs `num_prompts` × ISL/OSL | + + +#### Tier 3 — Server / platform metrics (implement CVS-2) + + +| Metric | Namespace key | Source | Gate? | +| ---------------------- | --------------------------- | ------------------------------------------ | -------------------------- | +| Time to ready | `server.time_to_ready_s` | `lifecycle.record` after `wait_ready` | P2 max regression | +| Client bench wall time | `server.client_wall_s` | lifecycle `client_complete` | record-only | +| Model cache size | `server.model_cache_bytes` | `_du_bytes` on `HF_HUB_CACHE` path | record-only | +| Container launch | `server.container_launch_s` | existing `test_launch_container` lifecycle | P2 | +| Image / recipe | (metadata) | `run_card` + config | PR audit, not numeric gate | + + +#### Tier 4 — Accuracy and MTP quality (Section 5) + + +| Metric | Namespace key | Test id | Gate? | +| ------------------ | ----------------------------- | ------- | --------- | +| gsm8k exact match | `accuracy.gsm8k_exact_match` | ACC-1 | **M2 P1** | +| gsm8k strict match | `accuracy.gsm8k_strict_match` | ACC-2 | record | +| MTP acceptance | `mtp.acceptance_rate` | ACC-4 | P2 | + + +#### Tier 5 — Multi-node / scaling (Milestone M5 — P1 after parity) + + +| Metric | Namespace key | Notes | +| --------------------- | ----------------------------------- | ------------------------------------ | +| Scaling efficiency % | `scaling.efficiency_pct` | actual tput / (single-node × nnodes) | +| Per-node throughput | `client.output_throughput` per rank | requires multi-node orch | +| Fabric / NIC metadata | run_card fields | Thor2 vs AINIC comparability | + + +#### Percentile and gating policy (summary) + + +| Config | Emitted percentiles | CVS behavior | +| ----------------------------------------- | -------------------------- | ------------------------------------------------------------------------------------ | +| `metric_percentiles: "95,99"` (W1 perf) | mean + **p95/p99** tails for TPOT/TTFT | Tier gates: `p99_ttft_ms`, `p95_tpot_ms` enforced in `health` / `ttft` / `tpot` tiers | +| `metric_percentiles: "90,95,99"` (future) | full tails | Can gate p90/p95 per tracker rows #26–27 | +| Accuracy | N/A | Never mixed into perf `GATED_METRICS` | + + +#### Sweep-level analytics (CVS-10) + +For each variant run, CVS should also produce **one summary row per ISL/OSL pair**: + +- `summary.max_output_throughput` — best conc in sweep +- `summary.conc_at_max_tput` — argmax concurrency +- `summary.ttft_at_max_tput` — TTFT at that point (latency vs load knee) + +These are **post-processing** over `inf_res_dict`, not new container benchmarks. + +See **Section 12** for perf variant modes (PERF-2..8), supplemental metrics (12.4), MTP (12.5), and CI compare keys (12.6). + +--- + +## 7. Phased implementation strategy (revised for `hnimrama/IX-atom`) + + +| Phase | Name | Goal | Status on branch | +| ----- | ------------------------- | --------------------------------------------------------------------------------- | --------------------------------------- | +| **R** | **Rename + pytest shell** | `inferencex_atom`, `InferenceXAtomJob`, schema_version 1, DTNI conftest | **Done** | +| **0** | **ATOM backend** | ATOM serve + bench + parse; W1 dirs; cluster JSON; legacy `inferencemax/` removed | **Done** (0-1 image/recipe pin partial) | +| **A** | **W1 calibration** | MI300X single-node lab-gated; MI355X seeds pending (Section 1.2) | **MI300X in progress** | +| **B** | **Metric namespace** | IX-native keys; `server.`* lifecycle; Section 6.1 tiers | Partial (`client.*` ATOM) | +| **C** | **P1 workloads** | W2, W3, W13, W17 on MI300X first; MI355X dirs when hardware available | Not started | +| **D** | **Accuracy + CI** | gsm8k M2 on MI300X (Section 5 + Phase D below) | Not started | +| **E** | **Framework parity (M4)** | Single-node W1 triple: `driver=atom` + `vllm_atom` + `sglang`; `compare.*` HTML | Not started (multinode PP shipped via M5 drivers) | +| **F** | **Multi-node + scaling (M5)** | **P1 after M4** when hardware + recipe support `nnodes>1`; `params.nnodes`, sshd, `scaling.*` (Section 1.7) | Not started — infra hooks only on branch | +| **G** | **MTP hardening + P2 (M6)** | W1 MTP3 lab optional; W4–W12, W14–W16, W18 | MTP configs only | +| **H** | **Disagg + DI stack (M7)** | Appendix B when infra ready | Blocked | + + +```mermaid +flowchart TB + PR["R: pytest shell DONE"] + P0["0: ATOM backend DONE"] + PA["A: W1 MI300X lab-gated"] + PAP["A: MI355X pending"] + PB["B: metric namespace"] + PC["C: P1 workloads MI300X"] + PD["D: gsm8k M2"] + PE["E: parity M4"] + PF["F: multi-node M5 P1"] + PFP["F: multi-node lab pending"] + PG["G: MTP + P2 M6"] + PH["H: disagg M7"] + PR --> P0 --> PA + PA -.-> PAP + P0 --> PB + PA --> PD + PB --> PC + PC --> PE --> PF --> PG --> PH + PD --> PE + PF -.-> PFP +``` + + + +--- + +## 8. Action items (detailed) + +### Phase 0 — ATOM backend (complete on branch) + + +| ID | Action | Details | Status | +| --- | ------------------------- | ---------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------- | +| 0-1 | **IX image + inline CLI pin** | W1 configs inline `roles.server.atom_args` (and MTP3 `params.bench_extra_args`); image pin in variant `run_card` / container | **Done** — removed `ix_recipes.json` / `ix_recipe_id` indirection | +| 0-2 | **ATOM serve path** | `InferenceXAtomJob.build_server_cmd` → `python -m atom.entrypoints.openai_server` | **Done** | +| 0-3 | **ATOM bench client** | `atom.benchmarks.benchmark_serving` → `results.json`; `to_client_metrics` | **Done** | +| 0-4 | **DTNI pytest shell** | `conftest.py` + sweep parametrization + tiered `test_cell_metrics`; shared `inference_suite_lifecycle.py` | **Done** | +| 0-5 | **Variant configs W1** | Flat single + mtp3 + baseline_sweep stems for MI300X and MI355X (Section 3.1) | **Done** | +| 0-6 | **Cluster configs** | `inferencex_atom_cluster.json` template; container names in variant config | **Done** | +| 0-7 | **Remove legacy configs** | Delete `inferencemax/`, nested `deepseek_r1_fp8_*` subdirs, old monolithic JSON layouts | **Done** | + + +### Phase A — W1 calibration (MI300X lab-gated; MI355X pending) + + +| ID | Action | Details | Blocker? | +| --- | ----------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------- | +| A-0 | **MI300X path check** | Run `mi300x_inferencex-atom_deepseek-r1_fp8_single` with `-k` one cell before full sweep | **Recommended** before A-1 | +| A-1 | **MI300X single thresholds** | `mi300x_inferencex-atom_deepseek-r1_fp8_single` thresholds from Section 4.1 (10% margin). W1 run: ~17 pytest rows (tiered gates, server reuse on C=256). | **Yes** — M1 close on MI300X | +| A-2 | **MI355X threshold seeds** | `*_mi355x_*` dirs from Section 4.3 (ATOM run 27912164002) | **No** — in tree; lab confirm when hardware available | +| A-3 | **Run card / PR evidence** | HTML report, log file, bundle zip; log image, `gpu_arch`, TP8, KV mode, inline `atom_args` | Per arch | +| A-4 | **Flip `enforce_thresholds`** | MI300X perf: after confirming CVS run. MI355X: when lab available. Smoke/MTP3: stay record-only until explicitly calibrated | MI300X perf only for M1 | +| A-5 | **W1 MTP3 (optional)** | `mi300x_inferencex-atom_deepseek-r1_fp8_mtp3` lab + Section 4.2 thresholds | **No** — post-M1; does not block M2 | + + +### Phase B — Metrics pipeline + + +| ID | Action | Details | +| --- | ---------------------------- | --------------------------------------------------------------------------------------------- | +| B-1 | **IX → threshold key map** | Documented in Section 1.4 + Section 4 + Section 6.1; optional IX-native keys for M4 baselines | +| B-2 | `**client.`* for ATOM perf** | Keep for ATOM W1; deprecate only when baselines move to separate namespace | +| B-3 | **Results table** | `test_print_results_table` columns match tracker P1 dashboard | +| B-4 | **Percentile policy** | Either expand `metric_percentiles` to `90,95,99` or keep record-only p90/p95 (Section 6.1) | +| B-5 | `**server.`* lifecycle** | Promote `lifecycle.record` timings to gated/record metrics (CVS-2, Section 6.1 Tier 3) | +| B-6 | **Sweep summary** | Post-run max-tput / knee detection per ISL/OSL (CVS-10, Section 6.1) | +| B-7 | **`compare.*` namespace** | Ratio / delta metrics for CI and M4 parity (Section 12.6) | +| B-8 | **Upstream CI diff script** | Emit `compare.atom_ci.*` from Section 4 reference (CVS-8) | +| B-9 | **Supplemental perf metrics** | Tier Section 12.4 — stddev, `output_tput_per_gpu`, `gpu.*` when INF-7 lands | + + +### Phase C — P1 workload variants (MI300X leads lab) + + +| ID | Action | Details | +| --- | ------------- | ---------------------------------------------------------------------------------------------- | +| C-0 | **W1** | **Done** on branch (single/mtp3); MI300X single-node lab closes M1 | +| C-2 | **W2** | MI300X first: GPT-OSS MXFP4 TP4, ISL 8K / OSL 1K; replace interim `mi300x_inferencex-atom_gpt-oss-120b_bf16` | +| C-3 | **W3** | MI300X: GLM 5.1 BF16; MI355X dir when hardware available | +| C-4 | **W13** | Kimi K2.7 Code — MI300X first | +| C-5 | **W17** | DeepSeek R1 MXFP4 — MI300X first | +| C-6 | **Parity drivers (M4)** | Single-node W1 stems with `driver=vllm_atom` and `driver=sglang` alongside `driver=atom` (Section 12.3) | + + +### Phase D — Accuracy + CI (M2) + + +| ID | Action | Details | +| --- | ----------------------- | --------------------------------------------------------------------------------------------------------------- | +| D-1 | **Variant stem** | Add `mi300x_inferencex-atom_deepseek-r1_fp8_accuracy` — separate from perf sweep; `enforce_thresholds: true` (Section 5.2) | +| D-2 | **Harness** | `run_gsm8k_eval()` — lm-eval or ATOM eval in container; ACC-1 + ACC-2 filters (Section 5.1) | +| D-3 | **Metric namespace** | `accuracy.gsm8k_exact_match` with `min` ≥ 0.94 FP8; add `ACCURACY_METRICS` display list (Section 5.3) | +| D-4 | **Pytest integration** | `test_gsm8k_accuracy` — not parametrized per conc cell; optional chain after perf job | +| D-5 | **CI split** | Perf job (long) vs accuracy job (medium); smoke stays pre-gate (Section 5.2 table) | +| D-6 | **Threshold ownership** | Document bump process when ATOM image / model revision changes (variant README + Section 4 re-pull) | +| D-7 | **Negative test** | Unit test: `evaluate_all` fails below gsm8k floor | +| D-8 | **W17 MXFP4 gate** | Mirror accuracy variant with floor **0.93** when W17 lands (Section 5.1) | +| D-9 | **MTP quality (P2)** | ACC-4/ACC-5 + Section 12.5 metrics when MTP3 orch complete | +| D-10 | **Workload ACC rows** | ACC-8..ACC-13 per Section 12.2 when W2/W3/W13 land | + + +### Phase E — Framework parity (M4) + + +| ID | Action | Details | +| ---- | --------------------- | ------------------------------------------------------------------------------------- | +| M4-1 | **Parity drivers** | Document and ship single-node W1 variants per `params.driver` (`atom`, `vllm_atom`, `sglang`) | +| M4-2 | **W1 parity triple** | ATOM + vLLM-ATOM + SGLang on MI300X single-node (multinode PP already on M5 drivers) | +| M4-3 | **Compare report** | `compare.vllm.*` / `compare.sglang.*` in HTML (Section 12.6) | + + +### Phase F — Multi-node + scaling (M5 — P1) + + +| ID | Action | Details | Status | +| ---- | --------------------------- | ----------------------------------------------------------------------------------------------------------------- | ------ | +| F-1 | **Multi-node cluster JSON** | `inferencex_atom_cluster.json` with 2+ `node_dict` entries; document head IP → `params.master_addr` | **Shipped** — edit per lab | +| F-2 | **`params.nnodes` in Job** | `InferenceXAtomJob`: vLLM PP flags for `vllm_atom`; SGLang PP for `sglang`; SPMD DP for `atom`; `test_setup_sshd` | **Shipped** | +| F-3 | **W1 multi-node variants** | `*_distributed`, `*_baseline_sweep_distributed` (`vllm_atom`, PP=2); `*_sglang_distributed` | **Shipped** — lab recalibrate thresholds | +| F-4 | **Scaling metrics** | `scaling.efficiency_pct` in multinode thresholds + parser | **Shipped** | +| F-5 | **Thresholds** | Cell keys `PP=2,NNODES=2`; recalibrate after true PP lab runs — do not trust pre-PP-orch numbers | **Pending lab** | +| F-6 | **Single-node parity** | M4: same sweep on `driver=atom` vs uplift `vllm` / future dedicated parity stems | Not started | +| F-7 | **Run card / fabric** | `ib_netdev`, NIC model, container image pin on multinode run card | Partial — `ib_netdev` in config schema | + + +### Phases G–H (M6–M7) + + +| ID | Action | Details | +| --- | ---------------- | ------------------------------------------------------------------------------------- | +| G-1 | **MTP orch** | Wire recipe-specific serve args for `*_mtp3` (Section 4.2); Section 12.5 metrics | +| G-2 | **P2 dirs** | W4–W12, W14–W16, W18 + perf modes PERF-2..8 (Section 12.1) | +| G-3 | **MTP compare** | `_mtp_compare` variant + `mtp.speedup_vs_fp8` (PERF-8) | +| H-1 | **Disagg spike** | Before W5/W6 disagg promises | + + +### Documentation + + +| ID | Action | Details | +| ----- | --------------------------------- | ----------------------------------------------------------------------------------------------------------------- | +| DOC-1 | **Link tracker → plan** | Point readers to W1–W18 table (Section 3) from `docs/reference/configuration-files/inferencex_atom.rst` | +| DOC-2 | **Clarify interim vLLM variants** | Mark `mi300x_inferencex-atom_gpt-oss-120b_bf16` as uplift placeholder until W2 ATOM lands | +| DOC-3 | **MI300X in user docs** | State explicitly that `inferencex_atom` supports **MI300X and MI355X**; MI355X lab pending per Section 1.2 | +| DOC-4 | **Variant README** | Keep `inferencex_atom/README.md` in sync with drivers, multinode PP runbook, and Section 1.5 | +| DOC-5 | **PR checklist** | M1 PR: MI300X HTML + logs; note MI355X pending; `pip install -e` called out | +| DOC-6 | **Multi-node milestone** | Document M5 PP=2 via `vllm_atom`/`sglang`; lab recalibration before enforcing multinode gates | + + +--- + +## 9. Appendix A — W1 inline server CLI reference + +W1 DeepSeek R1 FP8 configs set **`roles.server.atom_args`** inline (vLLM-style, same role as `roles.server.serve_args` on `vllm_single`). Example base FP8 block: + +```json +"atom_args": ["-tp", "8", "--kv_cache_dtype", "fp8", "--trust-remote-code"] +``` + +MTP3 variants append `--method mtp --num-speculative-tokens 3` to `atom_args` and set `params.bench_extra_args` to `--use-chat-template`. + +Pin **docker image** and upstream run URL in variant `run_card`, not in `threshold.json`. + +Maintain **W id → CVS variant stem** in variant README tables (Section 3.1). IX / ATOM catalog names (e.g. `dsr1-fp8-mi300x-atom`) remain documentation labels only — not a CVS config field. + +--- + +## 10. Appendix B — Deferred DI platform matrix (tracking only) + +Unchanged from prior plan: Thor2/AINIC, Optimus, KVMGR, NIXL, MOR-EP, RCCL, MI3XXX/MI4XXX GPU matrix, Gateway, MaaS — implement only after P1 ATOM perf + accuracy gates are green. + +--- + +## 12. Extended coverage — variants, frameworks, and metrics + +Supplements Sections 5–6 with perf variant modes, workload-specific quality tests, **new parity framework suites** (without modifying legacy `vllm_single` or SGLang disagg), and metric keys not yet fully specified. + +### 12.1 Functional and perf variant modes + +Beyond `_single`, `_mtp3`, and `_accuracy`, CVS should support **variant suffixes** (separate dirs or `params.mode`) so the same workload recipe can run different bench shapes without forking the suite id. + +| Mode suffix | Test id | Config knobs | Gate? | Purpose | +| ----------- | ------- | ------------ | ----- | ------- | +| `_single` | **PERF-0** | W1 single-node reference (2 conc cells) | M1 | Primary single-node ATOM stem | +| `_perf` | **PERF-1** | `dataset_name: random`, `request_rate: inf` | **M1** | Primary throughput/latency sweep | +| `_goodput` | **PERF-2** | Finite `request_rate` + optional `goodput_slo` per sweep combo | P2 | Tracker #31; enables non-null `client.goodput` | +| `_trace` | **PERF-3** | `dataset_name: sharegpt` (or IX trace id) | P2 | Realistic arrival / length mix (W2 8K ISL) | +| `_prefix_cache` | **PERF-4** | Enable prefix caching in serve args; shared-prefix bench | P2 | `cache.prefix_hit_rate` vs W1 default (no prefix cache) | +| `_rate_sweep` | **PERF-5** | Multiple `request_rate` values per conc (sub-sweep or extra `runs[]`) | P2 | Latency vs offered load (tracker #30) | +| `_longctx` | **PERF-6** | ISL at tracker max (e.g. W5 5000, W2 8192) | P2 | OOM / TTFT tail stress | +| `_mtp3` | **PERF-7** | Inline MTP3 `atom_args` + `bench_extra_args` in variant config | Post-M1 | Speculative decode perf | +| `_mtp_compare` | **PERF-8** | Paired run: same cell as `_perf` FP8 sibling | P2 | Emits `mtp.speedup_vs_fp8` (Section 12.5) | +| `_api_smoke` | **FUNC-1** | Single chat + completion curl after `wait_ready` | P2 | API contract / chat template sanity | +| `_health` | **FUNC-2** | `/health`, model list, max_tokens=1 | Record | Liveness distinct from bench throughput | + +**Infrastructure tests (already in `inferencex_atom.py`)** + +| Test id | Pytest | Metrics / outcome | +| ------- | ------ | ----------------- | +| **INF-1** | `test_launch_container` | `server.container_launch_s` | +| **INF-2** | `test_setup_sshd` | sshd on :2224 when multi-node | +| **INF-3** | `test_model_fetch` | `server.model_cache_bytes`, `server.model_fetch_s` | +| **INF-4** | `test_teardown` | No stale container | +| **INF-5** | `test_print_results_table` | Sweep summary HTML | + +**Planned infrastructure (Phase B/E)** + +| Test id | Description | Phase | +| ------- | ----------- | ----- | +| **INF-6** | dmesg / GPU hang scan post-run | B | +| **INF-7** | rocm-smi peak memory snapshot during bench | B | +| **INF-8** | Stale container cleanup pre-flight | B | +| **INF-9** | Threshold regression injection (negative test beyond D-7) | D | + +### 12.2 Accuracy and quality by workload type + +Section 5 covers **gsm8k** for general reasoning/quant paths. P1/P2 workloads need **additional ACC rows** keyed to model role. + +| Workload | Model role | ACC tests (beyond gsm8k) | Metric keys | Gate phase | +| -------- | ---------- | ------------------------ | ----------- | ---------- | +| **W1** | General reasoning FP8 | ACC-1 gsm8k | `accuracy.gsm8k_exact_match` | **M2** | +| **W2** | Long-context MXFP4 | ACC-1 + **ACC-8** long-doc subset | `accuracy.gsm8k_exact_match`, `accuracy.longctx_exact_match` | M3 | +| **W3** | GLM BF16 | ACC-1 + **ACC-6** MMLU subset | `accuracy.mmlu_acc` | M3 | +| **W13** | Code | **ACC-9** HumanEval, **ACC-10** MBPP | `accuracy.humaneval_pass_at_1`, `accuracy.mbpp_pass_at_1` | M3 | +| **W17** | MXFP4 reasoning | ACC-1 floor **0.93** | `accuracy.gsm8k_exact_match` | M3 | +| **W7** | Thinking / reasoning | **ACC-11** MATH-500 subset | `accuracy.math500_acc` | P2 | +| **W5 / W6** | Long-context MoE | **ACC-12** needle / RULER at tracker ISL | `accuracy.needle_recall` | P2 | +| **W10 / W12** | Large BF16 | ACC-6 MMLU + gsm8k spot check | `accuracy.mmlu_acc` | P2 | +| **MTP variants** | Spec decode | ACC-4, ACC-5 + **ACC-13** chat-template golden hash | `mtp.*`, `accuracy.chat_template_ok` | P2 | + +**ACC-8 … ACC-13 summary** + +| Test id | Benchmark | Harness | Initial policy | +| ------- | --------- | ------- | -------------- | +| **ACC-8** | Long-context gsm8k slice | lm-eval length filter | Record → gate W2 | +| **ACC-9** | HumanEval | lm-eval `humaneval` | Gate W13 | +| **ACC-10** | MBPP | lm-eval `mbpp` | Gate W13 | +| **ACC-11** | MATH-500 subset | lm-eval | Record-only P2 | +| **ACC-12** | Needle / RULER | Custom or lm-eval | Record-only P2 | +| **ACC-13** | Chat template smoke | Fixed prompt → hash | P2 for MTP | + +Variant naming: `_mi300x_atom_accuracy` for gsm8k; add `_code_accuracy`, `_longctx_accuracy` when a workload needs multiple ACC stages. + +### 12.3 Framework parity — drivers within `inferencex_atom` + +**Policy:** Do **not** extend legacy `vllm_single` or SGLang disagg wrappers for IX parity. Use **`params.driver`** on the same `inferencex_atom` framework and variant layout. + +| Driver | Engine role | Server args | Multinode PP | +| ------ | ----------- | ----------- | ------------ | +| **`atom`** | Standalone ATOM | `roles.server.atom_args` | SPMD DP only (not PP) | +| **`vllm_atom`** | vLLM coordinator + ATOM kernels | `roles.server.serve_args` + `ib_netdev` | **PP=2 shipped** (M5) | +| **`sglang`** | SGLang coordinator | `roles.server.sglang_args` + `ib_netdev` | **PP=2 shipped** (M5) | +| **`vllm`** | Interim ROCm vLLM uplift | `roles.server.serve_args` | Same PP flags as `vllm_atom` when `nnodes>1` | + +**Variant pairing (W1 DeepSeek R1 FP8, MI300X)** + +| Use case | Config stem | +| -------- | ----------- | +| Single-node ATOM reference | `mi300x_inferencex-atom_deepseek-r1_fp8_single` | +| Multinode PP=2 vLLM-ATOM | `mi300x_inferencex-atom_deepseek-r1_fp8_distributed` | +| Multinode PP=2 SGLang | `mi300x_inferencex-atom_deepseek-r1_fp8_sglang_distributed` | +| DTNI baseline sweep multinode PP=2 | `mi300x_inferencex-atom_deepseek-r1_fp8_baseline_sweep_distributed` | + +Rules: + +- **Same** sweep cells and `model.id` within a comparison family; **separate** `threshold.json` per driver/arch. +- Threshold cell keys: single-node `ISL=…,TP=8,CONC=…`; multinode PP `…,PP=2,NNODES=2,CONC=…`. +- Interim `mi300x_inferencex-atom_gpt-oss-120b_bf16` remains `driver=vllm` until W2 ATOM lands. + +**M4 deliverables (still open):** single-node parity triple on MI300X; `compare.vllm.*` / `compare.sglang.*` HTML (Section 12.6). + +**M5 (landed in repo):** multinode PP stems for `vllm_atom` and `sglang`; lab recalibration pending. + +```mermaid +flowchart LR + REF["Same sweep ISL/OSL/conc"] + A["driver=atom"] + V["driver=vllm_atom"] + S["driver=sglang"] + REF --> A + REF --> V + REF --> S + A --> CMP["compare.* metrics"] + V --> CMP + S --> CMP +``` + +### 12.4 Supplemental performance metrics + +| Metric | Namespace key | Producer | Tier | Notes | +| ------ | ------------- | -------- | ---- | ----- | +| Output tput per GPU | `client.output_tput_per_gpu` | `output_throughput / TP` | 2 | Not the same as `per_gpu_throughput` (total/TP) | +| Latency stddev | `client.std_ttft_ms`, `client.std_tpot_ms`, `client.std_itl_ms`, `client.std_e2el_ms` | ATOM JSON | 2 record | Variance / stability | +| Max decode burst | `client.max_output_tokens_per_s` | ATOM JSON | 2 record | Peak vs sustained tput | +| Scheduler saturation | `client.max_concurrency`, `client.max_concurrent_requests` | ATOM JSON | 2 record | Headroom | +| RTFX | `client.rtfx` | ATOM if emitted | 2 record | Document when non-zero | +| Error mix | `client.errors_timeout`, `client.errors_4xx`, `client.errors_5xx` | Log scrape | 2 P2 | Extends tracker #35 | +| Peak GPU memory | `gpu.peak_mem_mb` | rocm-smi (INF-7) | 2 P2 | Tracker #33 | +| KV footprint | `gpu.kv_cache_used_mb` | ATOM / rocm-smi | 2 P2 | Tracker #34 | +| Prefix cache hit | `cache.prefix_hit_rate` | Server when PERF-4 | 2 P2 | Prefix-cache variant | +| Power | `gpu.avg_power_w` | rocm-smi | 3 record | Optional efficiency | + +W1 `_perf` enforces **output throughput**, **mean TTFT**, **mean TPOT** (+ p99 when emitted); other `GATED_METRICS` use loose placeholders until calibrated (Section 1.4). + +### 12.5 MTP metrics (complete) + +| Metric | Namespace key | Source | Gate? | +| ------ | ------------- | ------ | ----- | +| Acceptance rate | `mtp.acceptance_rate` | ATOM MTP stats | P2 min | +| Speculative tokens avg | `mtp.speculative_tokens_avg` | MTP telemetry | record | +| Rejected draft tokens | `mtp.rejected_draft_tokens` | MTP telemetry | record | +| Effective decode steps | `mtp.effective_decode_steps` | MTP telemetry | record | +| Speedup vs FP8 | `mtp.speedup_vs_fp8` | Paired `_mtp_compare` / `_perf` | P2 min | +| TPOT reduction | `mtp.tpot_reduction_pct` | Paired mean TPOT | record | +| TTFT regression | `mtp.ttft_delta_ms` | MTP TTFT − FP8 TTFT | P2 max | +| Degenerate decode ratio | `mtp.empty_or_repeat_ratio` | Log / eval | P2 max | +| Chat template OK | `accuracy.chat_template_ok` | ACC-13 golden hash | P2 | + +Collect in `_mtp3` and `_mtp_compare` variants only — not in FP8 `_perf` thresholds. + +### 12.6 Comparative and CI metrics + +Regression and **M4 parity** metrics. Use `min_ratio` / `max_ratio` / `within` per `dtni-dev-guide.md`. + +| Metric | Namespace key | Reference | Kind | When | +| ------ | ------------- | --------- | ---- | ---- | +| vs upstream ATOM CI | `compare.atom_ci.output_throughput_delta_pct` | Section 4.3 / pinned URL | `within` ±10% | PR / nightly | +| vs prior CVS run | `compare.prev_run.output_throughput_ratio` | Last green artifact | `min_ratio` 0.95 | Nightly | +| vLLM parity vs ATOM | `compare.vllm.output_throughput_ratio` | Same cell ATOM | `min_ratio` TBD | M4 | +| SGLang parity vs ATOM | `compare.sglang.output_throughput_ratio` | Same cell ATOM | `min_ratio` TBD | M4 | +| Latency parity | `compare.vllm.mean_ttft_ms_ratio` | ATOM mean TTFT | `max_ratio` 1.1 | M4 | +| gsm8k regression | `compare.prev_run.gsm8k_delta` | Prior accuracy run | max drop 0.01 | M2+ nightly | +| MTP uplift | `compare.mtp.speedup_ratio` | Same as `mtp.speedup_vs_fp8` | `min_ratio` 1.05 | Post-M1 | + +**CI workflow:** (1) perf job + `compare.atom_ci.*`; (2) accuracy job + `compare.prev_run.*`; (3) M4 parity job runs ATOM + atom-vllm + atom-sglang sequentially; (4) store last-green `results.json` per variant for ratio specs. + +--- + +## 13. Risks and mitigations + + +| Risk | Mitigation | +| ----------------------------------------------- | ------------------------------------------------------------------------------ | +| **Stale installed package on lab runner** | Section 1.5 — `pip install -e .` before every validation run | +| **PP mislabeled on `driver=atom`** | Standalone ATOM has no PP; multinode PP validation must use `vllm_atom` or `sglang` (Section 1.7, 2.1) | +| **Uncoupled multinode replicas** | Never run PP=2 configs with `driver=atom`; orchestrator must inject vLLM/SGLang coordinator flags | +| **vLLM driver mistaken for W1 ATOM done** | W1 single-node gates use `params.driver=atom`; GPT-OSS uplift uses `vllm` only | +| **Wrong workload on branch (GPT-OSS TP8 BF16)** | W2 spec is MXFP4 TP4; track as interim in DOC-2 | +| **Upstream ATOM CI drift** | Pin docker tag + run URL in variant README; re-pull Section 4.3 on image bumps | +| **MI300X vs MI355X threshold bleed** | Separate variant dirs + `threshold.json` per `gpu_arch` | +| **Multinode thresholds from broken runs** | Recalibrate `PP=2` stems after true PP lab; pre-orch numbers measured uncoupled replicas | +| **p90/p95 threshold false failures** | Section 1.4 — record-only when artifact omits percentiles | +| **MTP flakes** | Separate variant dir; post-M1; chat-template checklist | +| **Metric key drift** | B-1 / Section 1.4; forbid thresholds in config | +| **192 matrix scope creep** | MI300X spine first; MI355X parallel track pending Section 1.2 | +| **Missing `ib_netdev` / container image** | Config loader rejects `nnodes>1` without `ib_netdev`; set vLLM+ATOM or SGLang image before lab | +| **Secrets in verbose logs** | Rotate HF token; avoid logging env exports in CI capture when possible | + + +--- + +## Diagrams — milestones + +**Milestone 1:** Phase 0 + Phase A — ATOM backend, W1 **MI300X** lab-gated (Sections 4.1–4.2). W1 **MI355X** dirs + Section 4.3 seeds ship in repo; lab confirm **pending** (Section 1.2). + +```mermaid +flowchart TB + M0["M0: Rename + pytest shell DONE"] + M1["M1: ATOM backend + W1 MI300X lab-gated"] + M1P["M1b: W1 MI355X lab pending"] + M2["M2: gsm8k accuracy MI300X"] + M2P["M2b: gsm8k MI355X pending"] + M3["M3: P1 W2 W3 W13 W17 MI300X"] + M4["M4: atom-vllm + atom-sglang parity"] + M5["M5: multi-node scaling P1"] + M5P["M5b: multi-node lab pending"] + M6["M6: MTP + P2 expansion"] + M7["M7: disagg + DI stack"] + M0 --> M1 --> M2 --> M3 --> M4 --> M5 --> M6 --> M7 + M1 -.-> M1P + M2 -.-> M2P + M5 -.-> M5P +``` + + + +```mermaid +flowchart TB + subgraph spine["P1 spine — MI300X active"] + S1["M1: ATOM W1 MI300X"] + S2["gsm8k accuracy MI300X"] + S3["P1 W2 W3 W13 W17 MI300X"] + S4["atom-vllm + atom-sglang parity M4"] + S5["multi-node scaling M5"] + end + subgraph pending["MI355X / multi-node — pending lab"] + P1["W1 perf confirm"] + P2["gsm8k"] + P3["P1 workloads"] + P4["multi-node confirm"] + end + subgraph widen["P2 widen"] + W1["Remaining W4–W18"] + W2["Full metric matrix"] + W3["MTP + disagg M6–M7"] + end + S1 --> S2 --> S3 --> S4 --> S5 + S5 --> W1 --> W2 --> W3 + S1 -.-> P1 + S2 -.-> P2 + S3 -.-> P3 + S5 -.-> P4 +``` + + +