From 48b11def8fabc3b165851c1577cadc835f1c593e Mon Sep 17 00:00:00 2001 From: Tom Allsop Date: Tue, 11 Aug 2026 15:39:21 +0100 Subject: [PATCH] Arm backend: Add --dump_artifacts pytest option * Add --dump_artifacts option to dump all artifacts of all selected tests * --dump_artifacts option accepts path to write artifacts to Signed-off-by: Tom Allsop Change-Id: Ia08bce8b93a4dc178fc87c58234788bf41906205 --- backends/arm/test/common.py | 35 ++++++++++------ backends/arm/test/conftest.py | 8 ++++ .../arm/test/misc/test_artifact_dumping.py | 40 +++++++++++++++++++ 3 files changed, 70 insertions(+), 13 deletions(-) create mode 100644 backends/arm/test/misc/test_artifact_dumping.py diff --git a/backends/arm/test/common.py b/backends/arm/test/common.py index abc12077e3d..6ea3d2fd168 100644 --- a/backends/arm/test/common.py +++ b/backends/arm/test/common.py @@ -49,10 +49,21 @@ def get_time_formatted_path(path: str, log_prefix: str) -> str: ) +def maybe_get_tosa_artifact_path() -> str | None: + """Return the configured artifact directory for the current test.""" + artifact_base_path = getattr(pytest, "_test_options", {}).get("dump_artifacts") + if artifact_base_path: + current_test = os.environ.get("PYTEST_CURRENT_TEST") + if current_test is None: + raise RuntimeError("Could not determine the current pytest test name") + test_name = current_test.split(" (")[0].rsplit("::", 1)[-1] + return os.path.join(artifact_base_path, test_name) + + return maybe_get_tosa_collate_path() + + def maybe_get_tosa_collate_path() -> str | None: - """Checks the environment variable TOSA_TESTCASES_BASE_PATH and returns the - path to the where to store the current tests if it is set. - """ + """Return the current test's TOSA collation directory, when configured.""" tosa_test_base = os.environ.get("TOSA_TESTCASES_BASE_PATH") if tosa_test_base: current_test = os.environ.get("PYTEST_CURRENT_TEST") @@ -76,7 +87,7 @@ def get_tosa_compile_spec( ) -> TosaCompileSpec: """Get the compile spec for default TOSA tests.""" if not custom_path: - custom_path = maybe_get_tosa_collate_path() + custom_path = maybe_get_tosa_artifact_path() if custom_path is not None: os.makedirs(custom_path, exist_ok=True) @@ -99,7 +110,7 @@ def get_u55_compile_spec( ) -> EthosUCompileSpec: """Default compile spec for Ethos-U55 tests.""" if not custom_path: - custom_path = maybe_get_tosa_collate_path() + custom_path = maybe_get_tosa_artifact_path() if custom_path is not None: os.makedirs(custom_path, exist_ok=True) @@ -134,9 +145,8 @@ def get_u85_compile_spec( tosa_debug_mode: EthosUCompileSpec.DebugMode | None = None, ) -> EthosUCompileSpec: """Default compile spec for Ethos-U85 tests.""" - if not custom_path: - custom_path = maybe_get_tosa_collate_path() + custom_path = maybe_get_tosa_artifact_path() if custom_path is not None: os.makedirs(custom_path, exist_ok=True) @@ -172,7 +182,7 @@ def get_u65_compile_spec( ) -> EthosUCompileSpec: """Default compile spec for Ethos-U65 tests.""" if not custom_path: - custom_path = maybe_get_tosa_collate_path() + custom_path = maybe_get_tosa_artifact_path() if custom_path is not None: os.makedirs(custom_path, exist_ok=True) @@ -207,9 +217,11 @@ def get_vgf_compile_spec( """Get the ArmCompileSpec for the default VGF tests, to modify the compile spec before calling .build() to finalize it. """ - if not custom_path: - custom_path = maybe_get_tosa_collate_path() + custom_path = maybe_get_tosa_artifact_path() + if custom_path is not None: + os.makedirs(custom_path, exist_ok=True) + profiles = [] if "FP" in repr(tosa_spec): profiles.append("fp") @@ -217,9 +229,6 @@ def get_vgf_compile_spec( profiles.append("int") if len(profiles) == 0: raise ValueError(f"Unsupported vgf compile_spec: {repr(tosa_spec)}") - - if custom_path is not None: - os.makedirs(custom_path, exist_ok=True) if compiler_flags is not None: compiler_flags_list = compiler_flags.split(" ") else: diff --git a/backends/arm/test/conftest.py b/backends/arm/test/conftest.py index 52288821af8..7e6b395fc64 100644 --- a/backends/arm/test/conftest.py +++ b/backends/arm/test/conftest.py @@ -25,6 +25,8 @@ def pytest_configure(config): if getattr(config.option, "llama_inputs", False) and config.option.llama_inputs: pytest._test_options["llama_inputs"] = config.option.llama_inputs # type: ignore[attr-defined] + if config.option.dump_artifacts: + pytest._test_options["dump_artifacts"] = config.option.dump_artifacts # type: ignore[attr-defined] logging.basicConfig(stream=sys.stdout) seed, seed_label = _setup_random_seed() @@ -105,6 +107,12 @@ def try_addoption(*args, **kwargs): nargs="+", help="List of two files. Firstly .pt file. Secondly .json", ) + try_addoption( + "--dump_artifacts", + dest="dump_artifacts", + metavar="DIR", + help="Dump Arm test artifacts into DIR/.", + ) def pytest_sessionstart(session): diff --git a/backends/arm/test/misc/test_artifact_dumping.py b/backends/arm/test/misc/test_artifact_dumping.py new file mode 100644 index 00000000000..676493e4912 --- /dev/null +++ b/backends/arm/test/misc/test_artifact_dumping.py @@ -0,0 +1,40 @@ +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import pytest + +from executorch.backends.arm.test import common + + +def test_dump_artifacts_uses_test_name(monkeypatch, tmp_path) -> None: + monkeypatch.setattr( + pytest, + "_test_options", + {"dump_artifacts": str(tmp_path)}, + raising=False, + ) + monkeypatch.setenv( + "PYTEST_CURRENT_TEST", + "backends/arm/test/ops/test_add.py::test_add_tosa_INT[shape] (call)", + ) + + assert common.maybe_get_tosa_artifact_path() == str( + tmp_path / "test_add_tosa_INT[shape]" + ) + + +def test_custom_path_overrides_dump_artifacts(monkeypatch, tmp_path) -> None: + monkeypatch.setattr( + pytest, + "_test_options", + {"dump_artifacts": str(tmp_path / "artifacts")}, + raising=False, + ) + + compile_spec = common.get_tosa_compile_spec( + "TOSA-1.0+INT", custom_path=str(tmp_path / "custom") + ) + + assert compile_spec._get_intermediate_path() == str(tmp_path / "custom")