test: add veryslow marker and comprehensive training integration tests

- Add @pytest.mark.veryslow for full training iteration tests, excluded by
  default even from `make test-slow`
- Expand env tests from 4 instantiation-only to 7 envs with full
  reset + step verification (dict obs keys, shapes, reward, done)
- Add RSL-RL PPO 1-iteration test for Go2 and G1 environments
- Add MLX PPO 1-iteration test on real Go2 env (macOS only)
- Mark SAC/TD3 learn tests as veryslow
- Fix APPO learn test: use real env instead of DummyFlatTest which is not
  registered in the collector subprocess (mp.spawn), causing 60s timeout
  per iteration instead of actual training
- Add early crash detection in APPORunner.learn() — fail fast when
  collector subprocess dies instead of waiting through 60s timeouts
- Fix RSL-RL wrapper _obs_to_tensordict to include "privileged" key in
  TensorDict so rsl_rl v5 resolve_obs_groups can find it

Author: TATP-233

CLAUDE.md

@@ -35,6 +35,7 @@ make check      # make format && make type
 make test       # Run all non-slow tests (default)
 make test-cov   # Run non-slow tests with coverage report
 make test-slow  # Run slow integration tests (requires MuJoCo)
+make test-veryslow  # Run full training iteration tests (minutes per test)
 make test-all   # make check && make test-cov
 ```
 
@@ -56,7 +57,10 @@ uv run pytest -m "not slow"
 uv run pytest -m "not slow" --cov=unilab --cov-report=term-missing
 
 # Slow integration tests (need MuJoCo installed)
-uv run pytest -m slow -v
+uv run pytest -m "slow and not veryslow" -v
+
+# Very slow: full training iteration tests (minutes per test)
+uv run pytest -m veryslow -v
 ```
 
 ## Test Structure
@@ -91,6 +95,9 @@ tests/
 Tests marked `@pytest.mark.slow` require a real MuJoCo environment and are excluded from CI
 by default. Run them locally when working on runner/learner code.
 
+Tests marked `@pytest.mark.veryslow` run full training iterations (minutes per test). They are
+excluded by default even when running `make test-slow`. Run `make test-veryslow` explicitly.
+
 ## Testing
 
 **New features must ship with tests.** When developing a new feature or refactoring, design and write comprehensive unit tests alongside the feature code — not as an afterthought. Tests should cover:

Makefile

@@ -29,7 +29,11 @@ test-fast:
 
 .PHONY: test-slow
 test-slow:
-	uv run pytest -m "slow"
+	uv run pytest -m "slow and not veryslow" -v
+
+.PHONY: test-veryslow
+test-veryslow:
+	uv run pytest -m "veryslow" -v
 
 .PHONY: test-all
 test-all: check test-cov

pyproject.toml

@@ -77,8 +77,11 @@ exclude = [
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]
-markers = ["slow: requires real env or long-running processes"]
-addopts = "--tb=short"
+markers = [
+    "slow: requires real env or long-running processes",
+    "veryslow: full training iteration tests (minutes per test)",
+]
+addopts = "--tb=short -m 'not veryslow'"
 
 [tool.coverage.run]
 source = ["unilab"]

scripts/train_rsl_rl.py

@@ -80,15 +80,13 @@ def __init__(self, env, device="cuda"):
 
     def _obs_to_tensordict(self, obs: dict[str, np.ndarray]) -> TensorDict:
         actor = to_torch(obs["actor"], self.device)
+        td = {"actor": actor}
         if "privileged" in obs:
-            policy = to_torch(flatten_obs_dict(obs), self.device)
+            td["privileged"] = to_torch(obs["privileged"], self.device)
+            td["policy"] = to_torch(flatten_obs_dict(obs), self.device)
         else:
-            policy = actor
-        return TensorDict(
-            {"policy": policy, "actor": actor},
-            batch_size=self.num_envs,
-            device=self.device,
-        )
+            td["policy"] = actor
+        return TensorDict(td, batch_size=self.num_envs, device=self.device)
 
     def step(self, actions):
         # Convert actions to numpy (CPU)

src/unilab/algos/torch/appo/runner.py

@@ -240,6 +240,16 @@ def learn(
 
             data_ready = shared_storage.wait_for_data(timeout=60.0)
             if not data_ready:
+                # Check if the collector subprocess died — fail fast instead of
+                # burning through remaining iterations with 60s timeouts each.
+                if not self._check_collector_alive():
+                    self._drain_metrics(
+                        metrics_queue, reward_history, latest_reward_components, logger
+                    )
+                    raise RuntimeError(
+                        "APPO collector process died before producing data. "
+                        "Check stderr for [APPO WORKER CRASH] messages."
+                    )
                 logger.log_status(
                     f"[yellow]Warning: Timeout waiting for data at iteration {iteration}[/]"
                 )

tests/algos/test_appo_runner.py

@@ -1,12 +1,12 @@
 """Slow integration tests for APPORunner.
 
 Requires MuJoCo to be installed. Run with:
-    uv run pytest -m slow -v
+    uv run pytest -m slow -v           # init + close tests
+    uv run pytest -m veryslow -v       # full training iteration
 """
 
 from __future__ import annotations
 
-import os
 import tempfile
 
 import pytest
@@ -33,24 +33,33 @@ def test_appo_runner_init_no_crash(mock_env_name):
     runner.close()
 
 
-# @pytest.mark.slow
-# def test_appo_runner_learn_two_iterations(mock_env_name):
-#     cfg = appo_config(mock_env_name).to_dict()
-#     cfg["num_envs"] = 4
-#     cfg["steps_per_env"] = 4
+@pytest.mark.slow
+@pytest.mark.veryslow
+@pytest.mark.parametrize("env_name", ["Go2JoystickFlatTerrain"])
+def test_appo_runner_learn_two_iterations(env_name):
+    """APPO learn test must use a real env — DummyFlatTest is not registered in
+    the collector subprocess (mp.spawn) so registry.make() would fail there."""
+    cfg = appo_config(env_name).to_dict()
+    cfg["num_envs"] = 128
+    cfg["steps_per_env"] = 8
+    # Small network for smoke test speed
+    cfg["actor"]["hidden_dims"] = [64, 64]
+    cfg["critic"]["hidden_dims"] = [64, 64]
+    cfg["algorithm"]["num_learning_epochs"] = 1
+    cfg["algorithm"]["num_mini_batches"] = 2
 
-#     runner = APPORunner(
-#         env_name=mock_env_name,
-#         env_cfg_overrides={},
-#         rl_cfg=cfg,
-#         num_envs=4,
-#         steps_per_env=4,
-#     )
+    runner = APPORunner(
+        env_name=env_name,
+        env_cfg_overrides={},
+        rl_cfg=cfg,
+        num_envs=128,
+        steps_per_env=8,
+    )
 
-#     with tempfile.TemporaryDirectory() as tmpdir:
-#         runner.learn(max_iterations=2, save_interval=0, log_dir=tmpdir)
+    with tempfile.TemporaryDirectory() as tmpdir:
+        runner.learn(max_iterations=2, save_interval=0, log_dir=tmpdir)
 
-#     runner.close()
+    runner.close()
 
 
 @pytest.mark.slow

tests/algos/test_mlx_ppo.py

@@ -300,3 +300,100 @@ def test_ppo_trainer_update_decreases_loss():
     metrics = trainer.update(buf, iteration=0)
     assert np.isfinite(metrics["surrogate"])
     assert np.isfinite(metrics["value"])
+
+
+# ---------------------------------------------------------------------------
+# Full training iteration on real env (veryslow)
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.slow
+@pytest.mark.veryslow
+def test_mlx_ppo_one_iteration_real_env():
+    """Run 1 full MLX PPO iteration (collect rollout + update) on a real env."""
+    _mujoco = pytest.importorskip("mujoco")
+
+    from unilab.base import registry
+    from unilab.config import locomotion_params
+    from unilab.utils.algo_utils import ensure_registries
+    from unilab.utils.obs_utils import flatten_obs_dict
+
+    ensure_registries()
+
+    env_name = "Go2JoystickFlatTerrain"
+    num_envs = 4
+    num_steps = 8
+
+    cfg = locomotion_params.ppo_config(env_name)
+    algo_cfg = cfg.algorithm
+
+    env = registry.make(env_name, num_envs=num_envs, sim_backend="mujoco")
+    obs_dim = sum(env.obs_groups_spec.values())
+    action_dim = env.action_space.shape[0]
+
+    model = MLPActorCritic(
+        obs_dim=obs_dim,
+        action_dim=action_dim,
+        actor_hidden_dims=[64, 64],
+        critic_hidden_dims=[64, 64],
+    )
+    ppo_cfg = PPOConfig(
+        num_learning_epochs=1,
+        num_mini_batches=2,
+        clip_param=float(algo_cfg.clip_param),
+        gamma=float(algo_cfg.gamma),
+        lam=float(algo_cfg.lam),
+    )
+    trainer = PPOTrainer(model, ppo_cfg)
+
+    # Init env and get first obs
+    if env.state is None:
+        env.init_state()
+    reset_indices = np.arange(num_envs, dtype=np.int32)
+    _, obs_dict, _ = env.reset(reset_indices)
+    obs = mx.array(flatten_obs_dict(obs_dict))
+
+    # Collect rollout
+    buffer = RolloutBuffer(
+        num_steps=num_steps,
+        num_envs=num_envs,
+        obs_dim=obs_dim,
+        action_dim=action_dim,
+        gamma=ppo_cfg.gamma,
+        lam=ppo_cfg.lam,
+    )
+
+    for _ in range(num_steps):
+        actions, log_probs, values, mean, std = model.act(obs)
+        mx.eval(actions, log_probs, values, mean, std)
+
+        env_actions = np.asarray(actions)
+        state = env.step(env_actions)
+        raw_obs = flatten_obs_dict(state.obs)
+        rewards = mx.array(state.reward)
+        dones = mx.array(state.done.astype(np.float32))
+
+        buffer.add(
+            obs=obs,
+            actions=actions,
+            log_probs=log_probs,
+            action_mean=mean,
+            action_std=std,
+            rewards=rewards,
+            dones=dones,
+            values=values,
+        )
+        obs = mx.array(raw_obs)
+
+    last_values = model.value(obs)
+    buffer.compute_returns_and_advantages(last_values)
+
+    # PPO update
+    metrics = trainer.update(buffer, iteration=0)
+    assert isinstance(metrics, dict)
+    assert np.isfinite(metrics["surrogate"]), (
+        f"surrogate loss is not finite: {metrics['surrogate']}"
+    )
+    assert np.isfinite(metrics["value"]), f"value loss is not finite: {metrics['value']}"
+
+    env.close()

tests/algos/test_offpolicy_runner.py

@@ -1,7 +1,8 @@
 """Slow integration tests for OffPolicyRunner (FastSAC/FastTD3).
 
 Requires MuJoCo to be installed. Run with:
-    uv run pytest -m slow -v
+    uv run pytest -m slow -v           # init + close tests
+    uv run pytest -m veryslow -v       # full training iteration
 """
 
 from __future__ import annotations
@@ -53,6 +54,7 @@ def test_offpolicy_runner_sac_init_no_crash(mock_env_name):
 
 
 @pytest.mark.slow
+@pytest.mark.veryslow
 def test_offpolicy_runner_sac_learn_two_iterations(mock_env_name):
     runner = _make_sac_runner(mock_env_name)
     with tempfile.TemporaryDirectory() as tmpdir:
@@ -109,6 +111,7 @@ def test_offpolicy_runner_td3_init_no_crash(mock_env_name):
 
 
 @pytest.mark.slow
+@pytest.mark.veryslow
 def test_offpolicy_runner_td3_learn_two_iterations(mock_env_name):
     runner = _make_td3_runner(mock_env_name)
     with tempfile.TemporaryDirectory() as tmpdir: