Skip to content

Commit a2e09f5

Browse files
committed
feat(fastsac): support MPS training on macOS
- resolve_default_device(): cuda -> mps -> cpu fallback shared by the sync/async trainers and the learner worker - collector inference accepts explicit 'mps'; generic 'cuda' degrades to the default device when CUDA is unavailable (MPS on Apple Silicon) - collector staged-host inference path for MPS (no pinned memory, no CUDA-graph sync); compile/AMP stay CUDA-only - EmpiricalNormalization local accumulators stay float64 on CPU (MPS has no float64); the multi-learner merge path remains CUDA-only - topology/numa guards for platforms without sched_getaffinity - per-stage learner timings logged to TensorBoard (perf/learner_*) - learner updates compiled with backend="aot_eager" on MPS: measured 86ms vs 106ms per update(4) (~19%); inductor measured 2.6x SLOWER than eager MPSGraph and stays CUDA-only; MPS autocast gains ~0 and is not enabled Verified end-to-end on a Mac Studio (M5 Ultra, torch 2.11): g1-wbt-dance motrix.fastsac trains with learner=mps collector_inference=mps at ~23k env steps/s (+71% vs eager) and ~44 updates/s (~95% of the standalone learner benchmark). CUDA paths are unchanged.
1 parent 64f7f28 commit a2e09f5

10 files changed

Lines changed: 100 additions & 21 deletions

File tree

‎motrix_rl_builtin/src/motrix_rl_builtin/fastsac/agent.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,16 @@ def __init__(
181181
# keeping the all-reduce outside the compiled region lets DDP ranks
182182
# (world_size > 1) use the same reduce-overhead graphs as the
183183
# single-learner path instead of falling back to eager.
184-
if bool(cfg.compile) and device.type == "cuda":
184+
if bool(cfg.compile) and device.type == "mps":
185+
# MPS: inductor kernels measured ~2.6x SLOWER than eager
186+
# MPSGraph on the g1-wbt-dance update (Mac Studio, torch 2.11),
187+
# and MPS autocast bf16/fp16 gains nothing (~1%). aot_eager
188+
# keeps the same MPSGraph kernels but removes the Python /
189+
# dispatch overhead: measured 86ms vs 106ms per update(4),
190+
# ~19% faster.
191+
self._critic_backward_runtime = torch.compile(self._critic_backward, backend="aot_eager", dynamic=False)
192+
self._actor_backward_runtime = torch.compile(self._actor_backward, backend="aot_eager", dynamic=False)
193+
elif bool(cfg.compile) and device.type == "cuda":
185194
import torch._inductor.config as inductor_config
186195

187196
inductor_config.compile_threads = 1

‎motrix_rl_builtin/src/motrix_rl_builtin/fastsac/async_impl/collector.py‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,12 @@ def resolve_collector_inference_device(device_spec: str) -> torch.device:
3838
device = torch.device(device_spec)
3939
if device.type == "cpu":
4040
return device
41+
if device.type == "mps":
42+
if not torch.backends.mps.is_available():
43+
raise RuntimeError(f"collector_inference_device='{device_spec}' requested MPS, but MPS is unavailable")
44+
return device
4145
if device.type != "cuda":
42-
raise ValueError(f"collector_inference_device must be cpu or cuda, got '{device_spec}'")
46+
raise ValueError(f"collector_inference_device must be cpu, cuda or mps, got '{device_spec}'")
4347
if not torch.cuda.is_available():
4448
raise RuntimeError(f"collector_inference_device='{device_spec}' requested CUDA, but CUDA is unavailable")
4549
index = torch.cuda.current_device() if device.index is None else device.index
@@ -136,15 +140,19 @@ def __init__(
136140
except KeyError as exc:
137141
raise ValueError("collector_amp_dtype must be 'fp16' or 'bf16'") from exc
138142

139-
self._flat_params = bind_flat_params(self.actor) if self.device.type == "cuda" else None
143+
# MPS has no pinned host memory, CUDA graphs or stream-ordered copies;
144+
# it shares the staged-host path with plain CPU staging buffers.
145+
self._is_cuda = self.device.type == "cuda"
146+
self._flat_params = bind_flat_params(self.actor) if self._is_cuda else None
140147
self._obs_host = None
141148
self._obs_device = None
142149
self._actions_host = None
143-
if self.device.type == "cuda":
144-
self._obs_host = torch.empty((self.num_envs, obs_dim), dtype=torch.float32, pin_memory=True)
150+
if self.device.type in ("cuda", "mps"):
151+
pin = self._is_cuda
152+
self._obs_host = torch.empty((self.num_envs, obs_dim), dtype=torch.float32, pin_memory=pin)
145153
self._obs_device = torch.empty_like(self._obs_host, device=self.device)
146-
self._actions_host = torch.empty((self.num_envs, act_dim), dtype=torch.float32, pin_memory=True)
147-
if bool(self.async_options.collector_compile):
154+
self._actions_host = torch.empty((self.num_envs, act_dim), dtype=torch.float32, pin_memory=pin)
155+
if self._is_cuda and bool(self.async_options.collector_compile):
148156
import torch._inductor.config as inductor_config
149157

150158
inductor_config.compile_threads = 1
@@ -201,7 +209,8 @@ def _infer(self, obs: torch.Tensor) -> torch.Tensor:
201209
self._actions_host.copy_(device_actions, non_blocking=True)
202210
# reduce-overhead may reuse CUDA Graph output storage. Complete D2H
203211
# before returning so the next replay cannot overwrite env actions.
204-
torch.cuda.current_stream(self.device).synchronize()
212+
if self._is_cuda:
213+
torch.cuda.current_stream(self.device).synchronize()
205214
return self._actions_host
206215

207216
def warmup_inference(self) -> None:

‎motrix_rl_builtin/src/motrix_rl_builtin/fastsac/async_impl/numa.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,9 @@ def select_collector_cpus(
103103

104104
def apply_cpu_affinity(cpus: list[int], role: str) -> None:
105105
"""Pin the current process to ``cpus``, intersected with allowed CPUs."""
106+
if not hasattr(os, "sched_getaffinity"):
107+
logger.warning("%s: CPU affinity is unsupported on this platform; skipping pinning to %s", role, sorted(cpus))
108+
return
106109
allowed = os.sched_getaffinity(0)
107110
selected = sorted(set(cpus) & allowed)
108111
if not selected:

‎motrix_rl_builtin/src/motrix_rl_builtin/fastsac/async_impl/topology.py‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@
3939
from motrix_rl_builtin.fastsac.async_impl import numa
4040
from motrix_rl_builtin.fastsac.config import FastSacAsyncOptionsCfg
4141

42+
logger = logging.getLogger(__name__)
43+
4244

4345
@dataclass(frozen=True)
4446
class LearnerInfo:
@@ -148,8 +150,18 @@ def resolve_collector_inference_devices(
148150
:func:`motrix_rl_builtin.fastsac.async_impl.collector.resolve_collector_inference_device`.
149151
"""
150152
device = torch.device(collector_inference_device)
151-
if device.type not in ("cpu", "cuda"):
152-
raise ValueError(f"collector_inference_device must be cpu or cuda, got '{collector_inference_device}'")
153+
if device.type not in ("cpu", "cuda", "mps"):
154+
raise ValueError(f"collector_inference_device must be cpu, cuda or mps, got '{collector_inference_device}'")
155+
if device.type == "mps" and not torch.backends.mps.is_available():
156+
raise RuntimeError(
157+
f"collector_inference_device='{collector_inference_device}' requested MPS, but MPS is unavailable"
158+
) # noqa: E501
159+
if device.type == "cuda" and device.index is None and not torch.cuda.is_available():
160+
# Generic "cuda" means "the trainer's accelerator": on a CUDA-less
161+
# host it degrades to the default device (MPS on Apple Silicon, CPU
162+
# elsewhere), mirroring the learner's device resolution.
163+
logger.warning("collector_inference_device='cuda' but CUDA is unavailable; using %s", default_device)
164+
return [default_device] * num_collectors
153165
if (
154166
device.type == "cuda"
155167
and device.index is not None
@@ -305,6 +317,9 @@ def resolve_trainer_topology(
305317
# best-effort contract.
306318
def _base_cpus(node: int | None) -> list[int]:
307319
if node is None:
320+
# macOS has no sched_getaffinity; degrade to no binding like worker.py
321+
if not hasattr(os, "sched_getaffinity"):
322+
return []
308323
return sorted(os.sched_getaffinity(0))
309324
try:
310325
return numa.numa_node_cpus(node)

‎motrix_rl_builtin/src/motrix_rl_builtin/fastsac/async_impl/train.py‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
run_learner_process,
5656
)
5757
from motrix_rl_builtin.fastsac.config import FastSacCfg
58+
from motrix_rl_builtin.fastsac.device import resolve_default_device
5859
from motrix_rl_builtin.fastsac.wrap import FastSacEnvWrap
5960

6061
torch.set_float32_matmul_precision("high")
@@ -75,7 +76,7 @@ def __init__(self, *, context: TrainerContext[FastSacCfg]) -> None:
7576
def _device(self) -> torch.device:
7677
if self._rlcfg.device:
7778
return torch.device(self._rlcfg.device)
78-
return torch.device("cuda" if torch.cuda.is_available() else "cpu")
79+
return resolve_default_device()
7980

8081
def _make_env(
8182
self,
@@ -690,6 +691,20 @@ def _render_panel(force: bool = False) -> None:
690691
tb_writer.add_scalar(f"perf/collector_{k}_ms", v, step)
691692
tb_writer.add_scalar("perf/learn_ms_total", learn_ms, step)
692693
tb_writer.add_scalar("perf/learn_pct", learn_pct, step)
694+
for rank_key, group in timing_groups.items():
695+
if rank_key == "collector":
696+
continue
697+
rank_tag = "" if rank_key == "learner" else f"{rank_key}/"
698+
699+
def _emit_timing(prefix: str, node: dict) -> None:
700+
for key, value in node.items():
701+
name = f"{prefix}{key.replace(' ', '_')}"
702+
if isinstance(value, dict):
703+
_emit_timing(f"{name}_", value)
704+
else:
705+
tb_writer.add_scalar(f"perf/learner_{rank_tag}{name}_ms", value, step)
706+
707+
_emit_timing("", group)
693708
for k, v in last_stats["env_metrics"].items():
694709
tb_writer.add_scalar(f"metrics/{k}", v, step)
695710
for k, v in last_stats["reward_terms"].items():

‎motrix_rl_builtin/src/motrix_rl_builtin/fastsac/async_impl/worker.py‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@
5757
weight_receiver_for,
5858
)
5959
from motrix_rl_builtin.fastsac.config import FastSacCfg
60+
from motrix_rl_builtin.fastsac.device import resolve_default_device
6061
from motrix_rl_builtin.fastsac.wrap import FastSacEnvWrap
6162
from motrix_rl_builtin.fastsac.wrap_np import FastSacNpEnvWrap
6263
from motrix_rl_builtin.fastsac.wrap_torch import FastSacTorchEnvWrap
@@ -490,11 +491,12 @@ def run_learner_process(
490491
opts = cfg.trainer.async_options
491492
_pin_worker_cpus(_resolve_cpu_set(opts.learner_cpu_cores, "learner_cpu_cores"))
492493
num_collectors = len(all_rings)
493-
device = (
494-
torch.device(learner_device)
495-
if learner_device is not None
496-
else torch.device(cfg.device or ("cuda" if torch.cuda.is_available() else "cpu"))
497-
)
494+
if learner_device is not None:
495+
device = torch.device(learner_device)
496+
elif cfg.device:
497+
device = torch.device(cfg.device)
498+
else:
499+
device = resolve_default_device()
498500
if num_learners > 1:
499501
if device.type != "cuda" or device.index is None:
500502
raise ValueError(f"multi-learner requires explicit indexed CUDA learner devices, got {device}")

‎motrix_rl_builtin/src/motrix_rl_builtin/fastsac/buffer.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -239,8 +239,11 @@ def __init__(self, shape, device, eps=1e-2, until=None):
239239
self.eps = eps
240240
self.until = until
241241
self.local_enabled = False
242-
self._local_mean = torch.zeros(shape, dtype=torch.float64, device=device)
243-
self._local_var = torch.ones(shape, dtype=torch.float64, device=device)
242+
# MPS has no float64; the local accumulators exist only for the
243+
# multi-learner merge (a CUDA-only path), so keep them on the CPU.
244+
acc_device = device if torch.device(device).type != "mps" else torch.device("cpu")
245+
self._local_mean = torch.zeros(shape, dtype=torch.float64, device=acc_device)
246+
self._local_var = torch.ones(shape, dtype=torch.float64, device=acc_device)
244247
self._local_count = 0
245248
self.register_buffer("_mean", torch.zeros(shape).unsqueeze(0).to(device))
246249
self.register_buffer("_var", torch.ones(shape).unsqueeze(0).to(device))
Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Copyright Motphys Technology Co., Ltd. 2025, 2026
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
"""Default Torch device resolution shared by the FastSAC trainers.
5+
6+
CUDA first (identical to the historical behavior), then MPS on Apple
7+
Silicon, then CPU. Explicitly configured devices (``rl_cfg.device``,
8+
``collector_inference_device``) never pass through here.
9+
"""
10+
11+
from __future__ import annotations
12+
13+
import torch
14+
15+
16+
def resolve_default_device() -> torch.device:
17+
"""The trainer device when the configuration leaves it unset."""
18+
if torch.cuda.is_available():
19+
return torch.device("cuda")
20+
if torch.backends.mps.is_available():
21+
return torch.device("mps")
22+
return torch.device("cpu")

‎motrix_rl_builtin/src/motrix_rl_builtin/fastsac/sync/train.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
)
2525
from motrix_rl_builtin.fastsac.agent import FastSacAgent
2626
from motrix_rl_builtin.fastsac.config import FastSacCfg
27+
from motrix_rl_builtin.fastsac.device import resolve_default_device
2728
from motrix_rl_builtin.fastsac.wrap import FastSacEnvWrap
2829
from motrix_rl_builtin.fastsac.wrap_np import FastSacNpEnvWrap
2930
from motrix_rl_builtin.fastsac.wrap_torch import FastSacTorchEnvWrap
@@ -53,7 +54,7 @@ def __init__(
5354
def _device(self) -> torch.device:
5455
if self._rlcfg.device:
5556
return torch.device(self._rlcfg.device)
56-
return torch.device("cuda" if torch.cuda.is_available() else "cpu")
57+
return resolve_default_device()
5758

5859
def _make_env(
5960
self,

‎motrix_rl_builtin/tests/test_fastsac_collector.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -262,8 +262,8 @@ def test_collector_cuda_request_fails_without_cuda(monkeypatch) -> None:
262262
resolve_collector_inference_device("cuda")
263263

264264

265-
def test_collector_rejects_non_cpu_cuda_device() -> None:
266-
with pytest.raises(ValueError, match="must be cpu or cuda"):
265+
def test_collector_rejects_unsupported_device() -> None:
266+
with pytest.raises(ValueError, match="must be cpu, cuda or mps"):
267267
resolve_collector_inference_device("meta")
268268

269269

0 commit comments

Comments
 (0)