feat: sync go1 motrix ppo/sac and g1 motrix ppo tuning, fix related bugs

.github/workflows/ci.yml

@@ -42,7 +42,7 @@ jobs:
       - name: Install dependencies
         run: uv sync --extra dev
       - name: mypy
-        run: uv run mypy unilab
+        run: uv run mypy src/unilab
       - name: pyright
         run: uv run pyright
 

.gitignore

@@ -16,6 +16,7 @@ dist/
 
 # Project specific
 logs/
+.worktrees/
 *.log
 .DS_Store
 

conf/offpolicy/reward/go1_sac_motrix.yaml

@@ -1,13 +1,12 @@
 # @package _global_
 reward:
   scales:
-    tracking_lin_vel: 1.2
-    tracking_ang_vel: 0.3
-    lin_vel_z: -6.0
-    ang_vel_xy: -0.15
-    base_height: -120.0
-    action_rate: -0.006
-    similar_to_default: -0.12
-    contact: 0.28
+    tracking_lin_vel: 1.0
+    tracking_ang_vel: 0.2
+    lin_vel_z: -5.0
+    ang_vel_xy: -0.1
+    base_height: -100.0
+    action_rate: -0.005
+    similar_to_default: -0.1
   tracking_sigma: 0.25
   base_height_target: 0.3

conf/offpolicy/task/g1_sac.yaml

@@ -13,3 +13,7 @@ algo:
   algo_params:
     alpha_init: 0.01
     target_entropy_ratio: 0.0
+motrix_legacy:
+  enabled: true
+  algo_overrides:
+    use_symmetry: false

conf/offpolicy/task/go1_joystick.yaml

@@ -7,3 +7,27 @@ training:
 algo:
   num_envs: 2048
   max_iterations: 3000
+motrix_legacy:
+  enabled: true
+  applies_to:
+    algo: sac
+  algo_overrides:
+    num_envs: 4096
+    max_iterations: 2000
+  env_cfg_override:
+    legacy_motrix_profile:
+      enabled: true
+      command_vel_limit:
+        - [0.5, 0.0, 0.0]
+        - [0.5, 0.0, 0.0]
+    reward_config:
+      scales:
+        tracking_lin_vel: 1.0
+        tracking_ang_vel: 0.2
+        lin_vel_z: -5.0
+        ang_vel_xy: -0.1
+        base_height: -100.0
+        action_rate: -0.005
+        similar_to_default: -0.1
+      tracking_sigma: 0.25
+      base_height_target: 0.3

conf/ppo/reward/g1_ppo_motrix.yaml

@@ -1,19 +1,22 @@
 # @package _global_
 reward:
   scales:
-    tracking_lin_vel: 2.4
-    tracking_ang_vel: 0.3
-    feet_phase: 1.2
-    lin_vel_z: -1.2
-    ang_vel_xy: -0.3
-    base_height: -600.0
-    orientation: -6.0
-    action_rate: -0.012
-    pose: -0.12
+    tracking_lin_vel: 3.0
+    tracking_ang_vel: 0.25
+    forward_progress: 1.5
+    under_speed: -1.0
+    upper_body_pose: -0.05
+    feet_phase: 0.5
+    lin_vel_z: -1.0
+    ang_vel_xy: -0.2
+    base_height: -120.0
+    orientation: -2.5
+    action_rate: -0.005
+    pose: -0.05
   tracking_sigma: 0.25
   gait_frequency: 1.5
   feet_phase_swing_height: 0.09
   feet_phase_tracking_sigma: 0.008
   base_height_target: 0.754
-  min_base_height: 0.55
-  max_tilt_deg: 25.0
+  min_base_height: 0.5
+  max_tilt_deg: 35.0

conf/ppo/reward/go1_ppo_motrix.yaml

@@ -1,13 +1,12 @@
 # @package _global_
 reward:
   scales:
-    tracking_lin_vel: 1.2
-    tracking_ang_vel: 0.3
-    lin_vel_z: -6.0
-    ang_vel_xy: -0.15
-    base_height: -120.0
-    action_rate: -0.006
-    similar_to_default: -0.12
-    contact: 0.28
+    tracking_lin_vel: 1.0
+    tracking_ang_vel: 0.2
+    lin_vel_z: -5.0
+    ang_vel_xy: -0.1
+    base_height: -100.0
+    action_rate: -0.005
+    similar_to_default: -0.1
   tracking_sigma: 0.25
   base_height_target: 0.3

conf/ppo/task/g1_joystick.yaml

@@ -12,3 +12,50 @@ algo:
       - policy
     actor:
       - actor
+motrix_legacy:
+  enabled: true
+  algo_overrides:
+    max_iterations: 151
+    empirical_normalization: true
+    obs_groups:
+      actor:
+        - policy
+    policy:
+      init_noise_std: 0.5
+    algorithm:
+      learning_rate: 3.0e-4
+      entropy_coef: 5.0e-3
+  env_cfg_override:
+    reward_config:
+      scales:
+        tracking_lin_vel: 3.0
+        tracking_ang_vel: 0.25
+        forward_progress: 1.5
+        under_speed: -1.0
+        upper_body_pose: -0.05
+        feet_phase: 0.5
+        lin_vel_z: -1.0
+        ang_vel_xy: -0.2
+        base_height: -120.0
+        orientation: -2.5
+        action_rate: -0.005
+        pose: -0.05
+      tracking_sigma: 0.25
+      gait_frequency: 1.5
+      feet_phase_swing_height: 0.09
+      feet_phase_tracking_sigma: 0.008
+      base_height_target: 0.754
+      min_base_height: 0.5
+      max_tilt_deg: 35.0
+    backend_overrides:
+      enabled: true
+      control_action_scale: 0.5
+      command_vel_limit:
+        - [0.45, 0.0, 0.0]
+        - [0.55, 0.0, 0.0]
+      gait_phase_init_mode: independent
+      reset_base_qvel_limit: 0.05
+      reward_scale_overrides:
+        under_speed: -1.0
+        upper_body_pose: -0.05
+        tracking_ang_vel: 0.25

conf/ppo/task/go1_joystick.yaml

@@ -11,3 +11,30 @@ algo:
       - policy
     actor:
       - actor
+motrix_legacy:
+  enabled: true
+  algo_overrides:
+    max_iterations: 151
+    empirical_normalization: true
+    policy:
+      init_noise_std: 0.5
+    algorithm:
+      learning_rate: 3.0e-4
+      entropy_coef: 1.0e-3
+  env_cfg_override:
+    legacy_motrix_profile:
+      enabled: true
+      command_vel_limit:
+        - [0.5, 0.0, 0.0]
+        - [0.5, 0.0, 0.0]
+    reward_config:
+      scales:
+        tracking_lin_vel: 1.0
+        tracking_ang_vel: 0.2
+        lin_vel_z: -5.0
+        ang_vel_xy: -0.1
+        base_height: -100.0
+        action_rate: -0.005
+        similar_to_default: -0.1
+      tracking_sigma: 0.25
+      base_height_target: 0.3

pyproject.toml

@@ -50,6 +50,7 @@ mujoco-uni = { index = "test-pypi" }
 line-length = 100
 target-version = "py310"
 src = ["src"]
+exclude = [".history"]
 
 [tool.ruff.lint]
 select = ["E", "F", "I", "N", "W"]

scripts/train_appo.py

@@ -54,8 +54,8 @@ def play_appo(cfg: DictConfig, rl_cfg: dict):
 
     from unilab.base import registry
     from unilab.utils import render_many
-    from unilab.utils.rsl_rl_compat import convert_config_v3_to_v4, is_rsl_rl_v4, is_rsl_rl_v5
     from unilab.utils.reward_utils import extract_reward_config
+    from unilab.utils.rsl_rl_compat import convert_config_v3_to_v4, is_rsl_rl_v4, is_rsl_rl_v5
 
     env_cfg_override = extract_reward_config(cfg)
 
@@ -75,6 +75,7 @@ def play_appo(cfg: DictConfig, rl_cfg: dict):
         env_cfg_override=env_cfg_override,
     )
     from unilab.utils.obs_utils import get_obs_dims
+
     obs_dim, privileged_dim = get_obs_dims(env.obs_groups_spec)
     action_dim = env.action_space.shape[0]