From 618ffb1263e8f79221b091b948f978596497fab7 Mon Sep 17 00:00:00 2001 From: RJ Ascani Date: Fri, 14 Aug 2026 10:00:59 -0700 Subject: [PATCH] Register a Cortex-M flow in the backend test suite The shared suite at backends/test/suite runs 23 torchvision and torchaudio models against every other major backend; Cortex-M had none, so its model coverage was hand-written one file at a time. This adds one, running on the Corstone-300 FVP. Five models pass and 15 are listed in CORTEX_M_SKIPS with a comment recording why each fails; read that list as a to-do. A pass means the model lowered and ran rather than that it was accurate, for the reason the module docstring gives. The suite drives to_edge_transform_and_lower, so CortexMTester grows a stage of that name: to_edge with no partitioner followed by CortexMPassManager, since Cortex-M rewrites operators in place rather than delegating a subgraph. The flow converts 4D inputs to channels_last, without which CortexMConv2DCheck rejects every convolution in every model and raises the FVP timeout, which an ImageNet-sized model exceeds at the 120 second default. CI goes through test_backend.sh like every other backend, path-gated because every case installs the Arm toolchain, builds a runner and drives the FVP. The operator suite is excluded and the exclusion says why. Authored with Claude Code. --- .ci/scripts/test_backend.sh | 9 ++ .github/workflows/test-backend-cortex-m.yml | 68 +++++++++++++++ backends/cortex_m/test/tester.py | 32 +++++++- backends/test/suite/flow.py | 14 ++++ backends/test/suite/flows/cortex_m.py | 91 +++++++++++++++++++++ 5 files changed, 212 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/test-backend-cortex-m.yml create mode 100644 backends/test/suite/flows/cortex_m.py diff --git a/.ci/scripts/test_backend.sh b/.ci/scripts/test_backend.sh index 6de519d3bec..b917588b8ae 100755 --- a/.ci/scripts/test_backend.sh +++ b/.ci/scripts/test_backend.sh @@ -96,6 +96,15 @@ if [[ "$FLOW" == *arm* ]]; then fi fi +if [[ "$FLOW" == *cortex_m* ]]; then + # Cortex-M runs on the Corstone-300 FVP, using the same Arm toolchain as the + # Ethos-U flows but its own semihosting runner. + .ci/scripts/setup-arm-baremetal-tools.sh + source examples/arm/arm-scratch/setup_path.sh + + backends/cortex_m/test/build_test_runner.sh +fi + if [[ "$FLOW" == *openvino* ]]; then # Setup OpenVINO environment source .ci/scripts/setup-openvino.sh --nightly diff --git a/.github/workflows/test-backend-cortex-m.yml b/.github/workflows/test-backend-cortex-m.yml new file mode 100644 index 00000000000..61444286314 --- /dev/null +++ b/.github/workflows/test-backend-cortex-m.yml @@ -0,0 +1,68 @@ +name: Test Cortex-M Backend + +on: + schedule: + - cron: 0 2 * * * + push: + branches: + - main + - release/* + tags: + - ciflow/nightly/* + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }} + cancel-in-progress: true + +jobs: + # Emits PR diff file list; non-PR events emit '*' so the per-job + # `if:` short-circuits via `event_name != 'pull_request'`. + changed-files: + name: Get changed files + uses: ./.github/workflows/_get-changed-files.yml + + test-cortex-m: + needs: changed-files + # Every case in this flow serializes a .pte and runs it on the Corstone-300 + # FVP, on top of an Arm toolchain install and a runner build. The Arm + # workflow keeps that off unrelated PRs by running only its TOSA and VGF + # flows there; Cortex-M has no non-FVP flow to fall back to, so it is + # path-gated instead and still runs in full on the nightly schedule. + # + # backends/arm is gated whole rather than by subdirectory. The flow borrows + # Arm's test harness, and importing it pulls in around 350 arm modules -- + # the passes, the TOSA operators, the quantizer -- so a break almost + # anywhere in that tree reaches this job. It costs little to be broad: + # over the last 300 commits, 66 touched backends/arm and 61 of those + # touched backends/arm/test or backends/arm/scripts anyway. + if: | + github.event_name != 'pull_request' || + contains(needs.changed-files.outputs.changed-files, 'backends/cortex_m') || + contains(needs.changed-files.outputs.changed-files, 'backends/arm') || + contains(needs.changed-files.outputs.changed-files, 'examples/arm') || + contains(needs.changed-files.outputs.changed-files, 'backends/test/suite') || + contains(needs.changed-files.outputs.changed-files, 'backends/test/harness') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/test_backend.sh') || + contains(needs.changed-files.outputs.changed-files, '.ci/scripts/setup-arm-baremetal-tools.sh') || + contains(needs.changed-files.outputs.changed-files, '.github/workflows/test-backend-cortex-m.yml') || + contains(needs.changed-files.outputs.changed-files, '.github/workflows/_test_backend.yml') + uses: ./.github/workflows/_test_backend.yml + with: + backend: cortex_m + flows: '["cortex_m"]' + # 276 of the operator suite's 467 cases fail, and 200 of those abort on a + # kernel the runner does not carry. Growing ops_list in + # backends/cortex_m/test/build_test_runner.sh is the wrong fix -- portable + # kernels bloat the binary, and most of these should never reach one. Of + # the 44 operators involved, 18 are unary maps that + # cortex_m::quantized_activation already implements for sigmoid and + # friends, 5 are variants of an op we have, 6 should be folded or + # decomposed ahead of time, 3 are 3D pooling, and 12 want a new cortex_m + # operator. The suite runs in ~8 minutes once they lower. + exclude: '[{"flow": "cortex_m", "suite": "operators"}]' + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout: 120 + run-linux: true + docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk diff --git a/backends/cortex_m/test/tester.py b/backends/cortex_m/test/tester.py index beed6e3aed0..a1b5245b80b 100644 --- a/backends/cortex_m/test/tester.py +++ b/backends/cortex_m/test/tester.py @@ -25,6 +25,7 @@ RunPasses, StageType, ToEdge, + ToEdgeTransformAndLower, ToExecutorch, ) @@ -51,8 +52,29 @@ def __init__(self, target_config: Optional[CortexMTargetConfig] = None): ) -class CortexMSerialize(Serialize): +class CortexMToEdgeTransformAndLower(ToEdgeTransformAndLower): + """to_edge with no partitioner, then CortexMPassManager. + + Cortex-M rewrites edge operators in place rather than delegating a subgraph, + so this is its equivalent of to_edge_transform_and_lower, which is the only + lowering entry point the shared backend test suite drives. + """ + def __init__(self, target_config: Optional[CortexMTargetConfig] = None): + super().__init__(edge_compile_config=cortex_m_edge_compile_config()) + self._run_passes = CortexMRunPasses(target_config) + + def run(self, artifact, inputs=None, generate_etrecord: bool = False) -> None: + super().run(artifact, inputs, generate_etrecord=generate_etrecord) + self._run_passes.run(self.edge_dialect_program, inputs) # type: ignore[arg-type] + + +class CortexMSerialize(Serialize): + def __init__( + self, + target_config: Optional[CortexMTargetConfig] = None, + timeout: int = 120, + ): target_config = target_config or CortexMTargetConfig(cpu=CortexM.M55) compile_spec = get_u55_compile_spec() # Select the runner built for this target (build_test_runner.sh writes @@ -60,6 +82,7 @@ def __init__(self, target_config: Optional[CortexMTargetConfig] = None): super().__init__( compile_spec, None, + timeout=timeout, build_dir_suffix=f"_{target_config.target_string}", ) @@ -69,6 +92,7 @@ def __init__(self, target_config: Optional[CortexMTargetConfig] = None): StageType.QUANTIZE: CortexMQuantize, StageType.RUN_PASSES: CortexMRunPasses, StageType.TO_EDGE: CortexMToEdge, + StageType.TO_EDGE_TRANSFORM_AND_LOWER: CortexMToEdgeTransformAndLower, StageType.TO_EXECUTORCH: ToExecutorch, StageType.SERIALIZE: CortexMSerialize, } @@ -80,6 +104,7 @@ def __init__( module, example_inputs, target_config: Optional[CortexMTargetConfig] = None, + timeout: int = 120, ): if callable(example_inputs): resolved_example_inputs = example_inputs() @@ -92,8 +117,11 @@ def __init__( stage_classes[StageType.RUN_PASSES] = lambda: CortexMRunPasses( target_config=target_config ) + stage_classes[StageType.TO_EDGE_TRANSFORM_AND_LOWER] = ( + lambda: CortexMToEdgeTransformAndLower(target_config=target_config) + ) stage_classes[StageType.SERIALIZE] = lambda: CortexMSerialize( - target_config=target_config + target_config=target_config, timeout=timeout ) super().__init__(module, resolved_example_inputs, stage_classes) diff --git a/backends/test/suite/flow.py b/backends/test/suite/flow.py index 0e5fe2a4ba1..f41ff763958 100644 --- a/backends/test/suite/flow.py +++ b/backends/test/suite/flow.py @@ -5,6 +5,7 @@ import logging import os +import shutil from dataclasses import dataclass, field from typing import Any, Callable @@ -176,6 +177,18 @@ def _load_arm() -> list[TestFlow]: ] +def _load_cortex_m() -> list[TestFlow]: + # Every case runs on the FVP. Without it each one fails the same way, which the + # flow's xfail list would report as expected. + if not shutil.which("FVP_Corstone_SSE-300_Ethos-U55"): + logger.info("Skipping Cortex-M flow registration: Corstone-300 FVP not on PATH") + return [] + + from executorch.backends.test.suite.flows.cortex_m import CORTEX_M_TEST_FLOW + + return [CORTEX_M_TEST_FLOW] + + def all_flows() -> dict[str, TestFlow]: from executorch.backends.test.suite.flows.portable import PORTABLE_TEST_FLOW @@ -188,6 +201,7 @@ def all_flows() -> dict[str, TestFlow]: + _register_flow(_load_openvino, "OpenVINO") + _register_flow(_load_qnn, "QNN") + _register_flow(_load_arm, "ARM") + + _register_flow(_load_cortex_m, "Cortex-M") ) try: diff --git a/backends/test/suite/flows/cortex_m.py b/backends/test/suite/flows/cortex_m.py new file mode 100644 index 00000000000..9ef42b3809b --- /dev/null +++ b/backends/test/suite/flows/cortex_m.py @@ -0,0 +1,91 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Cortex-M flow for the backend test suite. + +Runs on the Corstone-300 FVP, using the runner from +backends/cortex_m/test/build_test_runner.sh. There is no host path: +EXECUTORCH_BUILD_CORTEX_M is off in the default preset, so the host runtime +carries no cortex_m kernels. + +Read a pass here as "the model lowered and ran", not as an accuracy result. +""" + +import torch + +from executorch.backends.cortex_m.test.tester import CortexMQuantize, CortexMTester +from executorch.backends.test.suite.flow import TestFlow + + +def _create_cortex_m_tester(model, inputs, **kwargs) -> CortexMTester: + # The CMSIS-NN kernels are NHWC, and CortexMConv2DCheck rejects any pattern + # whose tensors are not channels_last. The suite hands over contiguous + # tensors, which silently costs every convolution in the model. + inputs = tuple( + ( + t.to(memory_format=torch.channels_last) + if isinstance(t, torch.Tensor) and t.dim() == 4 + else t + ) + for t in inputs + ) + # An FVP run of an ImageNet-sized model exceeds Serialize's 120s default. + return CortexMTester(model, inputs, timeout=1200, **kwargs) + + +# Models that do not run today. The reason each one fails is recorded here +# because the suite only reports that a test was skipped. +CORTEX_M_SKIPS = [ + # Whatever a model leaves unlowered runs on the portable kernels compiled + # into the runner, and ops_list in build_test_runner.sh is a fixed list. + "test_conformer", # aten::native_layer_norm.out + "test_convnext_small", # aten::native_layer_norm.out + "test_efficientnet_b4", # aten::silu.out + "test_efficientnet_v2_s", # aten::silu.out + "test_mnasnet1_0", # dim_order_ops::_to_dim_order_copy.out + "test_shufflenet_v2_x1_0", # aten::split_with_sizes_copy.out + # These two want a kernel the runner should not have to carry: the batch + # norm should have been folded, and cortex_m::transpose already exists. + "test_densenet161", # aten::_native_batch_norm_legit_no_training.out + "test_maxvit_t", # aten::permute_copy.out + # The .pte does not fit. The runner reads the whole program into a 60 MiB + # pool before memory planning is consulted. wav2letter is 100 MiB because + # its conv1d layers never lower, so the weights stay fp32. + "test_vit_b_16", # 84 MiB + "test_wav2letter", # 100 MiB + "test_wide_resnet50_2", # 109 MiB + # Run on target and return corrupt values. Both are concatenation-heavy and + # the portable NHWC cat kernel is known to corrupt results. + "test_inception_v3", # max error 7e11 + "test_squeezenet1_1", + # Ordinary int8 error on a 1000-way logit vector, at 35 dB SNR, rather than + # a wrong result -- but the suite's atol is fixed at 1e-1. + "test_resnet50", # max error 1.4 + # AtenToCortexMPass rejects the attention graph: "unsupported param type, + # call_function" on a bias that is a computed node rather than a parameter. + "test_swin_v2_t", +] + + +def _create_cortex_m_flow() -> TestFlow: + return TestFlow( + "cortex_m", + backend="cortex_m", + tester_factory=_create_cortex_m_tester, + quantize=True, + quantize_stage_factory=CortexMQuantize, + is_delegated=False, + param_skip_reasons={ + "use_dynamic_shapes": { + True: "Cortex-M lowers for a fixed shape; the CMSIS-NN kernels " + "take their dimensions from the graph." + } + }, + skip_patterns=CORTEX_M_SKIPS, + ) + + +CORTEX_M_TEST_FLOW = _create_cortex_m_flow()