Add NuRec Real2Sim Isaac Lab 3 support

compass/residual_rl/residual_ppo_trainer.py

@@ -18,12 +18,14 @@
 from contextlib import contextmanager
 from datetime import datetime
 
+# pylint: disable=wrong-import-position
 import numpy as np
 import h5py
 import gin
 # Set matplotlib backend before importing pyplot to avoid GUI backend issues
 import matplotlib
-matplotlib.use('Agg')  # Use non-interactive backend for headless environments
+
+matplotlib.use('Agg')    # Use non-interactive backend for headless environments
 import matplotlib.pyplot as plt
 import torch
 import torch.distributed as dist
@@ -51,7 +53,8 @@ def __init__(self,
                  ckpt_save_interval=50,
                  debug_viz=False,
                  max_debug_images=2,
-                 debug_image_interval=10):
+                 debug_image_interval=10,
+                 debug_image_step=0):
         # Prepare log directory. exist_ok=True avoids a TOCTOU race when several
         # torchrun ranks check + create concurrently.
         os.makedirs(output_dir, exist_ok=True)
@@ -112,7 +115,8 @@ def __init__(self,
 
         # Init debug images directory and counter. Independent of `debug_viz`
         # (which controls action-arrow visualization); debug image saving is
-        # gated by `max_debug_images` (None/0 = disabled) and `debug_image_interval`.
+        # gated by `max_debug_images` (None/0 = disabled), `debug_image_interval`,
+        # and `debug_image_step`.
         self.debug_images_dir = os.path.join(output_dir, 'debug_images')
         if not os.path.exists(self.debug_images_dir):
             os.makedirs(self.debug_images_dir, exist_ok=True)
@@ -121,8 +125,9 @@ def __init__(self,
         # each grid is comprised of 8 images, and so for 64 envs, there would be
         # 64 / 8 = 8 grids. But we can save less than 8 grids if we want to.
         self.max_debug_images = max_debug_images
-        # Save images every `debug_image_interval` iterations
+        # Save one rollout step every `debug_image_interval` iterations.
         self.debug_image_interval = debug_image_interval
+        self.debug_image_step = debug_image_step
 
         self.env.reset()
 
@@ -318,7 +323,7 @@ def learn(self, num_learning_iterations):
                             times[f"rollout/env_step/{_k}"] = times.get(
                                 f"rollout/env_step/{_k}", 0.0) + _v
 
-                        # Save debug camera grids (self-gated by max_debug_images / interval / step).
+                        # Save debug camera grids, gated by interval and step.
                         self._save_debug_images(obs_dict, it, _)
 
                         # Move time out information to the extras dict
@@ -589,20 +594,20 @@ def _upload_video(self, iteration):
                                   step=target_iteration)
 
     def _save_debug_images(self, obs_dict, iteration, step):
-        """Save debug images from all cameras as multiple grids for 1 step during training."""
+        """Save debug images from all cameras as multiple grids during training."""
         try:
             if self.max_debug_images is None or self.max_debug_images == 0:
                 return
 
             if "policy" not in obs_dict or "camera_rgb_img" not in obs_dict["policy"]:
                 return
 
-            # Only save images for the first step to avoid too many files
-            if step != 0:
+            # Only save one rollout step to avoid too many files.
+            if step != self.debug_image_step:
                 return
 
-            # Only save images at specified iteration intervals
-            if iteration % self.debug_image_interval != 0:
+            # Only save images at specified iteration intervals.
+            if self.debug_image_interval <= 0 or iteration % self.debug_image_interval != 0:
                 return
 
             camera_rgb = obs_dict["policy"]["camera_rgb_img"]
@@ -635,7 +640,8 @@ def _save_debug_images(self, obs_dict, iteration, step):
 
                 for env_idx in range(start_env, end_env):
                     # Process RGB image
-                    rgb_img = rgb_images_np[env_idx].copy()  # Make a copy to avoid modifying original
+                    rgb_img = rgb_images_np[env_idx].copy(
+                    )    # Make a copy to avoid modifying original
 
                     # Handle different image shapes - camera images are flattened in observations
                     if len(rgb_img.shape) == 1:
@@ -646,7 +652,8 @@ def _save_debug_images(self, obs_dict, iteration, step):
                         if rgb_img.size == expected_size:
                             rgb_img = rgb_img.reshape(height, width, channels)
                         else:
-                            print(f"[WARNING] Image size mismatch: expected {expected_size}, got {rgb_img.size}")
+                            print("[WARNING] Image size mismatch: "
+                                  f"expected {expected_size}, got {rgb_img.size}")
                             continue
                     elif len(rgb_img.shape) == 3:
                         # Already in (H, W, C) or (C, H, W) format
@@ -719,9 +726,47 @@ def _save_debug_images(self, obs_dict, iteration, step):
                                               image_path=grid_image_path,
                                               step=iteration)
 
+            self._save_debug_viewport_image(iteration, step)
+
         except (KeyError, IOError, OSError, ValueError, RuntimeError, AttributeError) as e:
             print(f"Warning: Failed to save debug images: {e}")
 
+    def _save_debug_viewport_image(self, iteration, step):
+        """Best-effort capture of the active Kit viewport for debugging GUI camera issues."""
+        if not self.is_rank_zero:
+            return
+
+        try:
+            import asyncio    # pylint: disable=import-outside-toplevel
+            import omni.kit.viewport.utility as viewport_utils    # pylint: disable=import-outside-toplevel
+
+            viewport = viewport_utils.get_active_viewport()
+            if viewport is None:
+                return
+
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f")[:-3]
+            filename = f"kit_viewport_iter_{iteration:04d}_step_{step:04d}_{timestamp}.png"
+            filepath = os.path.join(self.debug_images_dir, filename)
+            capture_helper = viewport_utils.capture_viewport_to_file(viewport, file_path=filepath)
+
+            if hasattr(capture_helper, "wait_for_result"):
+                wait_task = capture_helper.wait_for_result(completion_frames=5)
+                loop = asyncio.get_event_loop()
+                if loop.is_running():
+                    asyncio.ensure_future(wait_task)
+                else:
+                    loop.run_until_complete(wait_task)
+
+            metadata_path = f"{filepath}.txt"
+            with open(metadata_path, "w", encoding="utf-8") as metadata_file:
+                metadata_file.write(f"camera_path: {viewport.camera_path}\n")
+                metadata_file.write(
+                    f"render_product_path: {getattr(viewport, 'render_product_path', '')}\n")
+                metadata_file.write(f"resolution: {getattr(viewport, 'resolution', '')}\n")
+
+        except Exception as exc:    # pylint: disable=broad-except
+            print(f"Warning: Failed to save Kit viewport debug image: {type(exc).__name__}: {exc}")
+
     def _create_image_grid(self, images, subtitles, iteration, step, grid_idx=0):
         """Create and save a grid of images. Returns the filepath of the saved image."""
 
@@ -786,8 +831,13 @@ def _create_image_grid(self, images, subtitles, iteration, step, grid_idx=0):
             plt.tight_layout()
             # Use PNG format for better compatibility and lossless quality
             # Set format explicitly and ensure proper saving
-            plt.savefig(filepath, dpi=150, bbox_inches='tight', format='png',
-                       facecolor='white', edgecolor='none', pad_inches=0.1)
+            plt.savefig(filepath,
+                        dpi=150,
+                        bbox_inches='tight',
+                        format='png',
+                        facecolor='white',
+                        edgecolor='none',
+                        pad_inches=0.1)
             plt.close()
 
             # Verify file was created and is readable

compass/rl_env/exts/mobility_es/mobility_es/config/carter_env_cfg.py

@@ -16,7 +16,7 @@
 from __future__ import annotations
 
 from isaaclab.envs import mdp
-from isaaclab.utils import configclass
+from isaaclab.utils.configclass import configclass
 from isaaclab.managers import TerminationTermCfg as DoneTerm
 from isaaclab.managers import ObservationTermCfg as ObsTerm
 from isaaclab.managers import EventTermCfg as EventTerm

compass/rl_env/exts/mobility_es/mobility_es/config/digit_env_cfg.py

@@ -23,7 +23,7 @@
 from isaaclab.managers import TerminationTermCfg as DoneTerm
 from isaaclab.managers import ObservationTermCfg as ObsTerm
 from isaaclab.managers import SceneEntityCfg
-from isaaclab.utils import configclass
+from isaaclab.utils.configclass import configclass
 from isaaclab.utils.noise import UniformNoiseCfg as Unoise
 
 from mobility_es.config import scene_assets

compass/rl_env/exts/mobility_es/mobility_es/config/env_cfg.py

@@ -26,7 +26,7 @@
 from isaaclab.managers import SceneEntityCfg
 from isaaclab.sensors import ContactSensorCfg
 from isaaclab.scene import InteractiveSceneCfg
-from isaaclab.utils import configclass
+from isaaclab.utils.configclass import configclass
 from isaaclab.utils.noise import UniformNoiseCfg as Unoise
 from isaaclab_physx.physics import PhysxCfg
 
@@ -208,7 +208,7 @@ class EventCfg:
                 "pitch": (0.0, 0.0),
                 "yaw": (0.0, 0.0),
             },
-            # Default collision distance for start pose sampling
+    # Default collision distance for start pose sampling
             "collision_distance": 0.75,
         },
     )
@@ -322,10 +322,8 @@ def __post_init__(self):
         self.sim.physics_material.static_friction = 1.0
         self.sim.physics_material.dynamic_friction = 1.0
         self.sim.physics_material.restitution = 0.0
+        # Keep synthetic scenes on the policy cadence unless run.render_interval overrides it.
         self.sim.render_interval = self.decimation
-        # render settings
-        self.sim.render.enable_dl_denoiser = True
-        self.sim.render.antialiasing_mode = 'DLAA'
         # Update sensor update.
         if self.scene.contact_forces is not None:
             self.scene.contact_forces.update_period = self.sim.dt

compass/rl_env/exts/mobility_es/mobility_es/config/environments.py

@@ -17,7 +17,7 @@
 
 import isaaclab.sim as sim_utils
 from isaaclab.assets import AssetBaseCfg
-from isaaclab.utils import configclass
+from isaaclab.utils.configclass import configclass
 from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR
 
 USD_PATHS = {
@@ -42,10 +42,10 @@
                      "../usd/simple_warehouse_no_roof/simple_warehouse_no_roof.usd"),
     'Hospital':
         f'{ISAAC_NUCLEUS_DIR}/Environments/Hospital/hospital.usd',
-    # NuRec scene entries are injected here from nurec_scenes.py (imported at end of file).
-    # This is example manual entry scene kept for reference.
+    # NuRec scene entries are registered at runtime.
+    # Example manual entry scene kept for reference.
     # 'NovaCarterGalileo_NuRec':
-    #     os.path.join(os.path.dirname(__file__), "../usd/nova_carter-galileo/3dgrt/real2sim_galileo.usd"),
+    #     os.path.join(os.path.dirname(__file__), "../usd/nova_carter-galileo/..."),
 }
 
 # Values may be a string path for legacy COMPASS top-left-origin maps, or a dict
@@ -68,11 +68,11 @@
         os.path.join(os.path.dirname(__file__), "../usd/office/omap/occupancy_map.yaml"),
     'Hospital':
         os.path.join(os.path.dirname(__file__), "../usd/hospital/omap/occupancy_map.yaml"),
-    # NuRec scene entries are injected here from nurec_scenes.py (imported at end of file).
+    # NuRec scene entries are registered at runtime.
     # Example to show how we can define OMAP_PATHS with origin_convention
     # 'NovaCarterGalileo_NuRec':
     #     {
-    #         "path": os.path.join(os.path.dirname(__file__), "../usd/nova_carter-galileo/occupancy_map.yaml"),
+    #         "path": os.path.join(os.path.dirname(__file__), "../usd/.../occupancy_map.yaml"),
     #         "origin_convention": "bottom-left"
     #     },
 }
@@ -93,10 +93,6 @@ class EnvSceneAssetCfg(AssetBaseCfg):
     replicate_physics = True
 
 
-# NuRec Real2Sim scenes are defined in ``nurec_scenes.py`` and registered into
-# USD_PATHS / OMAP_PATHS / ``nurec_envs`` via a bottom-of-file import (see end of module).
-
-
 # Adding a USD scene with combined office, galileo lab and warehouse single rack.
 combined_single_rack = EnvSceneAssetCfg(
     prim_path="{ENV_REGEX_NS}/CombinedSingleRack",
@@ -266,29 +262,4 @@ class EnvSceneAssetCfg(AssetBaseCfg):
     replicate_physics=False,
 )
 
-# NuRec scenes (incl. ``nova_carter-galileo``) are defined in nurec_scenes.py and exposed
-# via the ``nurec_envs`` dict (re-exported at the end of this file).
-
-# nova_carter_galileo_nurec_1 = EnvSceneAssetCfg(
-#     prim_path="{ENV_REGEX_NS}/NovaCarterGalileo_NuRec_1",
-#     init_state=AssetBaseCfg.InitialStateCfg(
-#         pos=(0, 0, 0.01),
-#         rot=(0.0, 0.0, 0.0, 1.0),
-#     ),
-#     spawn=sim_utils.UsdFileCfg(
-#         usd_path=USD_PATHS['NovaCarterGalileo_NuRec_1'],
-#         scale=(1.0, 1.0, 1.0),
-#         rigid_props=sim_utils.RigidBodyPropertiesCfg(
-#             disable_gravity=None,
-#             solver_position_iteration_count=4,
-#             solver_velocity_iteration_count=1,
-#         ),
-#     ),
-#     env_spacing=500,
-# )
-
-
-# Register NuRec Real2Sim scenes (defined in nurec_scenes.py). Imported at the bottom so
-# EnvSceneAssetCfg / USD_PATHS / OMAP_PATHS already exist; importing nurec_scenes injects the
-# NuRec entries into USD_PATHS/OMAP_PATHS and re-exports ``nurec_envs`` (used by run.py).
-from mobility_es.config.nurec_scenes import nurec_envs  # noqa: E402,F401  pylint: disable=wrong-import-position,unused-import
+# NuRec scene cfgs are built at runtime.

compass/rl_env/exts/mobility_es/mobility_es/config/g1_env_cfg.py

@@ -17,7 +17,7 @@
 
 import os
 
-from isaaclab.utils import configclass
+from isaaclab.utils.configclass import configclass
 from isaaclab.sensors import CameraCfg
 
 from mobility_es.config import scene_assets

compass/rl_env/exts/mobility_es/mobility_es/config/h1_env_cfg.py

@@ -17,7 +17,7 @@
 
 import os
 
-from isaaclab.utils import configclass
+from isaaclab.utils.configclass import configclass
 from isaaclab.sensors import CameraCfg
 
 from mobility_es.config import scene_assets