diff --git a/.ci/scripts/test_backend.sh b/.ci/scripts/test_backend.sh index 6de519d3bec..b917588b8ae 100755 --- a/.ci/scripts/test_backend.sh +++ b/.ci/scripts/test_backend.sh @@ -96,6 +96,15 @@ if [[ "$FLOW" == *arm* ]]; then fi fi +if [[ "$FLOW" == *cortex_m* ]]; then + # Cortex-M runs on the Corstone-300 FVP, using the same Arm toolchain as the + # Ethos-U flows but its own semihosting runner. + .ci/scripts/setup-arm-baremetal-tools.sh + source examples/arm/arm-scratch/setup_path.sh + + backends/cortex_m/test/build_test_runner.sh +fi + if [[ "$FLOW" == *openvino* ]]; then # Setup OpenVINO environment source .ci/scripts/setup-openvino.sh --nightly diff --git a/.github/workflows/test-backend-cortex-m.yml b/.github/workflows/test-backend-cortex-m.yml new file mode 100644 index 00000000000..a3a4e39c9e6 --- /dev/null +++ b/.github/workflows/test-backend-cortex-m.yml @@ -0,0 +1,64 @@ +name: Test Cortex-M Backend + +on: + schedule: + - cron: 0 2 * * * + push: + branches: + - main + - release/* + tags: + - ciflow/nightly/* + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }} + cancel-in-progress: true + +jobs: + # Emits PR diff file list; non-PR events emit '*' so the per-job + # `if:` short-circuits via `event_name != 'pull_request'`. + changed-files: + name: Get changed files + uses: ./.github/workflows/_get-changed-files.yml + + test-cortex-m: + needs: changed-files + # Every case in this flow serializes a .pte and runs it on the Corstone-300 + # FVP, on top of an Arm toolchain install and a runner build. The Arm + # workflow keeps that off unrelated PRs by running only its TOSA and VGF + # flows there; Cortex-M has no non-FVP flow to fall back to, so it is + # path-gated instead and still runs in full on the nightly schedule. + # + # backends/arm is gated whole rather than by subdirectory. The flow borrows + # Arm's test harness, and importing it pulls in around 350 arm modules -- + # the passes, the TOSA operators, the quantizer -- so a break almost + # anywhere in that tree reaches this job. It costs little to be broad: + # over the last 300 commits, 66 touched backends/arm and 61 of those + # touched backends/arm/test or backends/arm/scripts anyway. + if: | + github.event_name != 'pull_request' || + contains(needs.changed-files.outputs.changed-files, 'backends/cortex_m') || + contains(needs.changed-files.outputs.changed-files, 'backends/arm') || + contains(needs.changed-files.outputs.changed-files, 'examples/arm') || + contains(needs.changed-files.outputs.changed-files, 'backends/test/suite') || + contains(needs.changed-files.outputs.changed-files, 'backends/test/harness') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/test_backend.sh') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/setup-arm-baremetal-tools.sh') || + contains(needs.changed-files.outputs.changed-files, '.github/workflows/test-backend-cortex-m.yml') || + contains(needs.changed-files.outputs.changed-files, '.github/workflows/_test_backend.yml') + uses: ./.github/workflows/_test_backend.yml + with: + backend: cortex_m + flows: '["cortex_m"]' + # Most of the operator suite still fails on a kernel the runner does not + # carry. Growing ops_list in backends/cortex_m/test/build_test_runner.sh + # is the wrong fix -- portable kernels bloat the binary, and most of these + # operators should be lowered, folded or decomposed instead of reaching + # one. Excluded until enough of them lower to be worth the runtime. + exclude: '[{"flow": "cortex_m", "suite": "operators"}]' + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout: 120 + run-linux: true + docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk diff --git a/backends/cortex_m/test/tester.py b/backends/cortex_m/test/tester.py index beed6e3aed0..a1b5245b80b 100644 --- a/backends/cortex_m/test/tester.py +++ b/backends/cortex_m/test/tester.py @@ -25,6 +25,7 @@ RunPasses, StageType, ToEdge, + ToEdgeTransformAndLower, ToExecutorch, ) @@ -51,8 +52,29 @@ def __init__(self, target_config: Optional[CortexMTargetConfig] = None): ) -class CortexMSerialize(Serialize): +class CortexMToEdgeTransformAndLower(ToEdgeTransformAndLower): + """to_edge with no partitioner, then CortexMPassManager. + + Cortex-M rewrites edge operators in place rather than delegating a subgraph, + so this is its equivalent of to_edge_transform_and_lower, which is the only + lowering entry point the shared backend test suite drives. + """ + def __init__(self, target_config: Optional[CortexMTargetConfig] = None): + super().__init__(edge_compile_config=cortex_m_edge_compile_config()) + self._run_passes = CortexMRunPasses(target_config) + + def run(self, artifact, inputs=None, generate_etrecord: bool = False) -> None: + super().run(artifact, inputs, generate_etrecord=generate_etrecord) + self._run_passes.run(self.edge_dialect_program, inputs) # type: ignore[arg-type] + + +class CortexMSerialize(Serialize): + def __init__( + self, + target_config: Optional[CortexMTargetConfig] = None, + timeout: int = 120, + ): target_config = target_config or CortexMTargetConfig(cpu=CortexM.M55) compile_spec = get_u55_compile_spec() # Select the runner built for this target (build_test_runner.sh writes @@ -60,6 +82,7 @@ def __init__(self, target_config: Optional[CortexMTargetConfig] = None): super().__init__( compile_spec, None, + timeout=timeout, build_dir_suffix=f"_{target_config.target_string}", ) @@ -69,6 +92,7 @@ def __init__(self, target_config: Optional[CortexMTargetConfig] = None): StageType.QUANTIZE: CortexMQuantize, StageType.RUN_PASSES: CortexMRunPasses, StageType.TO_EDGE: CortexMToEdge, + StageType.TO_EDGE_TRANSFORM_AND_LOWER: CortexMToEdgeTransformAndLower, StageType.TO_EXECUTORCH: ToExecutorch, StageType.SERIALIZE: CortexMSerialize, } @@ -80,6 +104,7 @@ def __init__( module, example_inputs, target_config: Optional[CortexMTargetConfig] = None, + timeout: int = 120, ): if callable(example_inputs): resolved_example_inputs = example_inputs() @@ -92,8 +117,11 @@ def __init__( stage_classes[StageType.RUN_PASSES] = lambda: CortexMRunPasses( target_config=target_config ) + stage_classes[StageType.TO_EDGE_TRANSFORM_AND_LOWER] = ( + lambda: CortexMToEdgeTransformAndLower(target_config=target_config) + ) stage_classes[StageType.SERIALIZE] = lambda: CortexMSerialize( - target_config=target_config + target_config=target_config, timeout=timeout ) super().__init__(module, resolved_example_inputs, stage_classes) diff --git a/backends/test/suite/conftest.py b/backends/test/suite/conftest.py index ecde002dd94..cc339d5ffc1 100644 --- a/backends/test/suite/conftest.py +++ b/backends/test/suite/conftest.py @@ -30,6 +30,13 @@ def pytest_collection_modifyitems(config, items): should_skip, reason = flow.should_skip_test(test_name, callspec.params) if should_skip: item.add_marker(pytest.mark.skip(reason)) + elif flow.should_xfail_test(test_name): + item.add_marker( + pytest.mark.xfail( + reason=f"Expected to fail by {flow.name} xfail_patterns", + strict=True, + ) + ) item_path = str(getattr(item, "path", "")) for suite_prefix, timeout_s in FLOW_TEST_CASE_TIMEOUTS.items(): diff --git a/backends/test/suite/flow.py b/backends/test/suite/flow.py index 0e5fe2a4ba1..81bfa65949e 100644 --- a/backends/test/suite/flow.py +++ b/backends/test/suite/flow.py @@ -5,6 +5,7 @@ import logging import os +import shutil from dataclasses import dataclass, field from typing import Any, Callable @@ -44,6 +45,11 @@ class TestFlow: skip_patterns: list[str] = field(default_factory=lambda: []) """ Tests with names containing any substrings in this list are skipped. """ + xfail_patterns: list[str] = field(default_factory=lambda: []) + """ Tests with names containing any substrings in this list are expected to fail. + They still run, so the report keeps recording how they fail; the marker is strict, + so one that starts passing is reported rather than silently ignored. """ + param_skip_reasons: dict[str, dict[Any, str]] = field(default_factory=dict) """ Skip tests with a given reason when a pytest parameter matches a given value.""" @@ -69,6 +75,9 @@ def should_skip_test( return False, "" + def should_xfail_test(self, test_name: str) -> bool: + return any(pattern in test_name for pattern in self.xfail_patterns) + def __str__(self): return self.name @@ -176,6 +185,17 @@ def _load_arm() -> list[TestFlow]: ] +def _load_cortex_m() -> list[TestFlow]: + # Every case runs on the FVP, so without it the whole flow fails the same way. + if not shutil.which("FVP_Corstone_SSE-300_Ethos-U55"): + logger.info("Skipping Cortex-M flow registration: Corstone-300 FVP not on PATH") + return [] + + from executorch.backends.test.suite.flows.cortex_m import CORTEX_M_TEST_FLOW + + return [CORTEX_M_TEST_FLOW] + + def all_flows() -> dict[str, TestFlow]: from executorch.backends.test.suite.flows.portable import PORTABLE_TEST_FLOW @@ -188,6 +208,7 @@ def all_flows() -> dict[str, TestFlow]: + _register_flow(_load_openvino, "OpenVINO") + _register_flow(_load_qnn, "QNN") + _register_flow(_load_arm, "ARM") + + _register_flow(_load_cortex_m, "Cortex-M") ) try: diff --git a/backends/test/suite/flows/cortex_m.py b/backends/test/suite/flows/cortex_m.py new file mode 100644 index 00000000000..8c10d162d90 --- /dev/null +++ b/backends/test/suite/flows/cortex_m.py @@ -0,0 +1,109 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Cortex-M flow for the backend test suite. + +Every case is serialized and executed on the Corstone-300 FVP with the +semihosting runner from backends/cortex_m/test/build_test_runner.sh. That runner +registers the cortex_m kernels along with the portable fallbacks listed in its +ops_list, so an operator the backend does not lower still runs when the list +happens to carry one. + +The kernels are CMSIS-NN compiled for the device and cannot run on a host CPU, +so the simulator is the only place to execute them. The Arm Ethos-U flows reach +the Corstone FVP the same way. flow.py leaves the flow unregistered when the FVP +is not on PATH. + +Comparison uses the suite's fixed atol of 1e-1, which is not tuned for int8, so +a mismatch is not by itself evidence of a lowering bug. +""" + +import torch + +from executorch.backends.cortex_m.test.tester import CortexMQuantize, CortexMTester +from executorch.backends.test.suite.flow import TestFlow + + +def _create_cortex_m_tester(model, inputs, **kwargs) -> CortexMTester: + # The CMSIS-NN kernels are NHWC, and CortexMConv2DCheck rejects any pattern + # whose tensors are not channels_last. The suite hands over contiguous + # tensors, which silently costs every convolution in the model. + inputs = tuple( + ( + t.to(memory_format=torch.channels_last) + if isinstance(t, torch.Tensor) and t.dim() == 4 + else t + ) + for t in inputs + ) + # The Serialize stage is also the stage that invokes the ELF, so its timeout + # is the FVP's --timelimit rather than a serialization budget, and 120s is + # not enough for an ImageNet-sized model on an M55 with no NPU. Kept under + # the suite conftest's 1200s pytest timeout so the FVP reports the overrun + # instead of pytest killing the process. + return CortexMTester(model, inputs, timeout=900, **kwargs) + + +# Models that cannot run on the FVP runner at all, so there is nothing for the +# report to record. A skipped test produces no row, which is why the sizes are +# written down here. +CORTEX_M_SKIPS = [ + # Over the runner's 60 MiB pool even fully quantized: 86.6M and 68.9M + # parameters. Raising ET_ARM_BAREMETAL_SEMIHOSTING_FILE_ALLOCATOR_POOL_SIZE + # is the only thing that would change that. + "test_vit_b_16", + "test_wide_resnet50_2", + # The program is under a megabyte; it is the activation arena that does not + # fit, and it crosses the pool at a sequence length of about 353. The test + # draws lengths from randint(1, 400), so it fits roughly one run in three. + "test_conformer", +] + +# Models that run to completion and fail. They stay in the suite so the report +# keeps recording how they fail; the marker only keeps a known state from +# turning the job red, and it is strict, so anything fixed reports XPASS and +# this list has to shrink. +CORTEX_M_XFAILS = [ + # Whatever a model leaves unlowered runs on the portable kernels compiled + # into the runner, and ops_list in build_test_runner.sh is a fixed list. + "test_convnext_small", + "test_densenet161", + "test_maxvit_t", + "test_mnasnet1_0", + "test_shufflenet_v2_x1_0", + # 4.00 bytes per parameter: none of the convolutions lower, so the weights + # stay fp32 and the 100 MiB program overruns the pool. Lowering conv1d + # brings it to 26 MiB, which fits. + "test_wav2letter", + # Runs on target and does not match. + "test_inception_v3", + "test_resnet50", + "test_squeezenet1_1", + # AtenToCortexMPass rejects the graph. + "test_swin_v2_t", +] + + +def _create_cortex_m_flow() -> TestFlow: + return TestFlow( + "cortex_m", + backend="cortex_m", + tester_factory=_create_cortex_m_tester, + quantize=True, + quantize_stage_factory=CortexMQuantize, + is_delegated=False, + param_skip_reasons={ + "use_dynamic_shapes": { + True: "Cortex-M lowers for a fixed shape; the CMSIS-NN kernels " + "take their dimensions from the graph." + } + }, + skip_patterns=CORTEX_M_SKIPS, + xfail_patterns=CORTEX_M_XFAILS, + ) + + +CORTEX_M_TEST_FLOW = _create_cortex_m_flow()