Hnimrama/ix atom multinode

.gitignore

@@ -17,3 +17,6 @@ docs/sphinx/_toc.yml
 
 # Build distributions
 dist/
+
+# Local sample output
+sample_reports/

cvs/cli_plugins/list_plugin.py

@@ -44,7 +44,14 @@ def discover_tests():
             test_map[pkg_name] = {}
             for root, dirs, files in os.walk(tests_dir):
                 for file in files:
-                    if file.endswith(".py") and file != "__init__.py":
+                    # Skip pytest infra (conftest.py) and private helpers
+                    # (e.g. _shared.py): they are not selectable suites.
+                    if (
+                        file.endswith(".py")
+                        and file != "__init__.py"
+                        and file != "conftest.py"
+                        and not file.startswith("_")
+                    ):
                         rel_path = os.path.relpath(os.path.join(root, file), tests_dir)
                         module_parts = os.path.splitext(rel_path)[0].split(os.sep)
                         # Module path: <tests_path>.<test_name>

cvs/cli_plugins/run_plugin.py

@@ -23,8 +23,12 @@ def get_parser(self, subparsers):
         )
         parser.add_argument(
             "--log-file",
-            default="/tmp/cvs/test.log",
-            help="Pytest: Path to file for logging output (default: /tmp/cvs/test.log)",
+            default=None,
+            metavar="PATH",
+            help=(
+                "Pytest: write logging output to this file (optional). "
+                "Parent directories are created automatically when set."
+            ),
         )
         parser.add_argument(
             "--log-level",

cvs/cli_plugins/unittests/test_run_plugin.py

@@ -79,6 +79,35 @@ def test_run_test_multiple_functions(self, mock_exit, mock_pytest_main):
         mock_pytest_main.assert_called_once_with(expected_args)
         mock_exit.assert_called_once_with(0)
 
+    @patch("cvs.cli_plugins.run_plugin.pytest.main")
+    @patch("cvs.cli_plugins.run_plugin.sys.exit")
+    def test_run_test_omits_log_file_when_not_set(self, mock_exit, mock_pytest_main):
+        """No --log-file is passed to pytest when the user does not request file logging."""
+        args = MagicMock()
+        args.test = "agfhc_cvs"
+        args.function = []
+        args.cluster_file = "/path/to/cluster.json"
+        args.config_file = "/path/to/config.json"
+        args.html = None
+        args.self_contained_html = False
+        args.log_file = None
+        args.log_level = None
+        args.capture = None
+        args.extra_pytest_args = []
+
+        mock_pytest_main.return_value = 0
+
+        with patch.object(self.plugin, "get_test_file", return_value="/mock/path/test.py"):
+            self.plugin.run(args)
+
+        expected_args = [
+            "/mock/path/test.py",
+            "--cluster_file=/path/to/cluster.json",
+            "--config_file=/path/to/config.json",
+        ]
+        mock_pytest_main.assert_called_once_with(expected_args)
+        mock_exit.assert_called_once_with(0)
+
 
 if __name__ == "__main__":
     unittest.main()

cvs/conftest.py

@@ -6,16 +6,19 @@
 """
 
 import importlib.metadata
+import logging
 import sys
 from pathlib import Path
 
 import pytest
 
 from cvs.lib.report_plugins import HtmlReportManager
 
+log = logging.getLogger(__name__)
 
-@pytest.hookimpl(tryfirst=True)
-def pytest_configure(config):
+
+def _sync_suite_name_from_args(config):
+    """Derive suite stem from the first ``*.py`` target in ``config.args``."""
     suite_name = "test"
     for arg in config.args:
         bare = arg.split("::")[0]
@@ -24,7 +27,96 @@ def pytest_configure(config):
             break
     config._suite_name = suite_name
     config._test_html_dir = f"{suite_name}_html"
+
+
+def _ensure_html_report_manager(config):
+    """Create ``HtmlReportManager`` once; safe if ``pytest_configure`` did not run."""
+    _sync_suite_name_from_args(config)
+    mgr = getattr(config, "_html_report_manager", None)
+    if mgr is not None:
+        return mgr
+
     config._html_report_manager = HtmlReportManager(config)
+    return config._html_report_manager
+
+
+def _auto_register_inference_suite_report(config):
+    from cvs.lib.report.auto_register import try_auto_register_inference_suite_report
+
+    _sync_suite_name_from_args(config)
+    return try_auto_register_inference_suite_report(config)
+
+
+@pytest.hookimpl(tryfirst=True)
+def pytest_configure(config):
+    _ensure_html_report_manager(config)
+    _auto_register_inference_suite_report(config)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def _cvs_inference_suite_report_session(request):
+    """Initialize the session report store when a suite preset is registered."""
+    from cvs.lib.report.registry import clear_session_results, get_suite_report_config
+    from cvs.lib.report.types import InferenceReportConfig
+
+    if not isinstance(get_suite_report_config(request.config), InferenceReportConfig):
+        yield
+        return
+
+    clear_session_results()
+    yield
+
+
+@pytest.fixture(scope="module", autouse=True)
+def _cvs_inference_suite_report_bind_module(request, _cvs_inference_suite_report_session):
+    """Bind module-scoped suite fixtures into the session store at module teardown."""
+    from cvs.lib.report.registry import bind_session_results, get_suite_report_config
+    from cvs.lib.report.types import InferenceReportConfig
+
+    if not isinstance(get_suite_report_config(request.config), InferenceReportConfig):
+        yield
+        return
+
+    inf_res_dict = None
+    variant_config = None
+    lifecycle = None
+    try:
+        inf_res_dict = request.getfixturevalue("inf_res_dict")
+    except pytest.FixtureLookupError:
+        log.warning(
+            "Inference suite report preset registered but inf_res_dict fixture is missing; "
+            "session-end report will be skipped"
+        )
+        yield
+        return
+    try:
+        variant_config = request.getfixturevalue("variant_config")
+    except pytest.FixtureLookupError:
+        log.warning(
+            "Inference suite report preset registered but variant_config fixture is missing; "
+            "session-end report will be skipped"
+        )
+        yield
+        return
+    try:
+        lifecycle = request.getfixturevalue("lifecycle")
+    except pytest.FixtureLookupError:
+        log.warning(
+            "Inference suite report preset registered but lifecycle fixture is missing; "
+            "session-end report will be skipped"
+        )
+        yield
+        return
+
+    def _bind_at_module_end():
+        bind_session_results(
+            inf_res_dict=inf_res_dict,
+            variant_config=variant_config,
+            lifecycle=lifecycle,
+        )
+
+    request.addfinalizer(_bind_at_module_end)
+    yield
 
 
 # Add all additional cmd line arguments for the script
@@ -93,15 +185,28 @@ def pytest_metadata(metadata):
 
 # Prepare a clean per-run log directory before tests start.
 def pytest_sessionstart(session):
-    session.config._html_report_manager.setup_log_dir()
+    _auto_register_inference_suite_report(session.config)
+    _ensure_html_report_manager(session.config).setup_log_dir()
 
 
 # Capture each test report and attach a per-test external log link.
 @pytest.hookimpl(hookwrapper=True)
 def pytest_runtest_makereport(item, call):  # noqa: ARG001
     outcome = yield
     report = outcome.get_result()
-    report.extras = item.config._html_report_manager.write_test_log(report, item.originalname)
+    report.extras = _ensure_html_report_manager(item.config).write_test_log(report, item.originalname)
+
+    from cvs.lib.report.registry import get_suite_report_config
+    from cvs.lib.report.types import InferenceReportConfig
+
+    if isinstance(get_suite_report_config(item.config), InferenceReportConfig):
+        from cvs.lib.report.inference_wiring import (
+            attach_inference_suite_lifecycle_table,
+            attach_inference_suite_report_row_extra,
+        )
+
+        attach_inference_suite_lifecycle_table(item, report)
+        attach_inference_suite_report_row_extra(item, report)
 
 
 # Replace inline pytest-html log content with a short externalized-log message.
@@ -118,4 +223,6 @@ def pytest_html_results_summary(prefix, summary, postfix):
 @pytest.hookimpl(hookwrapper=True)
 def pytest_sessionfinish(session, exitstatus):  # noqa: ARG001
     yield  # wait for pytest-html and all other plugins to finish writing the report
-    session.config._html_report_manager.create_zip_bundle(session)
+    mgr = _ensure_html_report_manager(session.config)
+    mgr.generate_suite_reports(session)
+    mgr.create_zip_bundle(session)

cvs/core/orchestrators/container.py

@@ -477,11 +477,16 @@ def setup_sshd(self):
             "chown -R root:root /root/.ssh",
             "bash -c 'chmod 700 /root/.ssh && chmod 600 /root/.ssh/*'",
             "mkdir -p /run/sshd",  # Create privilege separation directory for sshd
+            "bash -c 'which sshd > /dev/null 2>&1 || (apt-get update -qq && apt-get install -y -q openssh-server iproute2)'",
             "/usr/sbin/sshd -p2224",
         ]
 
+        timeouts = {
+            "bash -c 'which sshd > /dev/null 2>&1 || (apt-get update -qq && apt-get install -y -q openssh-server iproute2)'": 120,
+        }
+
         for cmd in ssh_setup_commands:
-            result = self.exec(cmd, timeout=10, detailed=True)
+            result = self.exec(cmd, timeout=timeouts.get(cmd, 10), detailed=True)
             # Check if command succeeded on all hosts
             for hostname, output in result.items():
                 if output['exit_code'] != 0:

cvs/core/runtimes/docker.py

@@ -121,6 +121,14 @@ def setup_containers(
             else:
                 self.log.info(f"Image {container_config['image']} already exists, skipping tar load")
 
+        if not self.check_image_exists(image):
+            self.log.info(f"Image {image} not present on all hosts; pulling before start")
+            pull_result = self.pull_image(image, timeout=600)
+            failed_pull = [host for host, res in pull_result.items() if res.get('exit_code') != 0]
+            if failed_pull:
+                self.log.error(f"Failed to pull image on hosts: {failed_pull}")
+                return False
+
         cmd = f"sudo docker run -d --name {container_name} {all_args_str} {image} sleep infinity"
 
         self.log.info(f"Starting long-running containers on {len(self.orchestrator.hosts)} nodes: {container_name}")
@@ -130,7 +138,7 @@ def setup_containers(
         remove_cmd = f"sudo docker rm -f {container_name} || true"
         self.orchestrator.all.exec(remove_cmd, timeout=30, print_console=False)
 
-        result = self.orchestrator.all.exec(cmd, timeout=60, detailed=True)
+        result = self.orchestrator.all.exec(cmd, timeout=120, detailed=True)
 
         # Check if all hosts started successfully
         success = all(output['exit_code'] == 0 for output in result.values())
@@ -278,6 +286,13 @@ def _build_runtime_args(runtime_args_config):
 
         return args
 
+    def pull_image(self, image_name, timeout=None):
+        """Pull container image on all hosts."""
+        timeout = timeout or 600
+        cmd = f"sudo docker pull {shlex.quote(image_name)}"
+        self.log.info(f"Pulling image on all hosts: {image_name}")
+        return self.orchestrator.all.exec(cmd, timeout=timeout, detailed=True)
+
     def load_image(self, tar_path, timeout=None):
         """Load container image from tar file on all hosts."""
         cmd = f"sudo docker load < {tar_path}"

cvs/core/runtimes/unittests/test_docker.py

@@ -130,6 +130,29 @@ def test_cmd_never_contains_gpus_all(self):
                     f"[{label}] '--gpus all' must never appear in docker cmd:\n{captured[0]}",
                 )
 
+    def test_pulls_image_when_missing_before_run(self):
+        calls = []
+
+        def _fake_exec(cmd, timeout=None, detailed=False, print_console=True):
+            calls.append(cmd)
+            if "docker images" in cmd and "grep" in cmd:
+                return {"host1": {"output": "", "exit_code": 1}}
+            return {"host1": {"output": "", "exit_code": 0}}
+
+        orchestrator = MagicMock()
+        orchestrator.hosts = ["host1"]
+        orchestrator.all.exec.side_effect = _fake_exec
+        rt = DockerRuntime(MagicMock(), orchestrator)
+
+        result = rt.setup_containers(
+            container_config=_container_config(),
+            container_name="cvs_iter_test",
+            volumes=["/home/u:/workspace"],
+        )
+        self.assertTrue(result)
+        self.assertTrue(any("docker pull" in c for c in calls))
+        self.assertTrue(any(c.startswith("sudo docker run") for c in calls))
+
 
 if __name__ == "__main__":
     unittest.main()

cvs/input/cluster_file/inferencex_atom_cluster.json

@@ -0,0 +1,34 @@
+{
+  "_comment": "InferenceX ATOM cluster template. Edit node_dict to match variant params.nnodes (1 host for single-node sweeps; 2+ for multinode). Set head_node_dict.mgmt_ip to the rank-0 VPC IP (params.master_addr for nnodes>1). Model weights under /home/models on each GPU node. Variant config overrides container image/name.",
+  "orchestrator": "container",
+  "username": "{user-id}",
+  "priv_key_file": "/home/{user-id}/.ssh/id_rsa",
+  "head_node_dict": {
+    "mgmt_ip": "{head-node-ip}"
+  },
+  "env_vars": {},
+  "node_dict": {
+    "{head-node-ip}": {
+      "bmc_ip": "NA",
+      "vpc_ip": "{head-node-ip}"
+    },
+    "{worker-node-ip}": {
+      "bmc_ip": "NA",
+      "vpc_ip": "{worker-node-ip}"
+    }
+  },
+  "container": {
+    "lifetime": "per_run",
+    "image": "rocm/atom-dev:latest",
+    "name": "inferencex_atom",
+    "runtime": {
+      "name": "docker",
+      "args": {
+        "network": "host",
+        "ipc": "host",
+        "privileged": true,
+        "shm_size": "128G"
+      }
+    }
+  }
+}