From db2b633b8decef47c6e5ea25015f3a003ef3c77f Mon Sep 17 00:00:00 2001 From: yuecideng Date: Thu, 4 Jun 2026 13:07:39 +0800 Subject: [PATCH 1/2] docs(sim): align window controls with DexSim defaults Remove redundant EmbodiChain default window control registration now that DexSim provides camera, selection, and focus shortcuts natively. Update window interaction docs to document DexSim controls and keep EmbodiChain-specific viewer recording hotkey guidance. Co-authored-by: Cursor --- docs/source/features/interaction/window.md | 48 ++++++++++++++++++---- embodichain/lab/sim/sim_manager.py | 24 ----------- 2 files changed, 39 insertions(+), 33 deletions(-) diff --git a/docs/source/features/interaction/window.md b/docs/source/features/interaction/window.md index e19b0da0..a83e5ccf 100644 --- a/docs/source/features/interaction/window.md +++ b/docs/source/features/interaction/window.md @@ -2,20 +2,48 @@ This section describes the default window interaction controls available in the simulation. These controls allow users to interact with the simulation environment using keyboard, mouse, and customizable input events. -## Default Window Events +The main visualization window is provided by **DexSim**. When `SimConfig.headless=False` or `SimulationManager.open_window()` is called, DexSim creates the viewer with **ORBIT** camera control by default. -The simulation window comes with a set of default controls that enable users to perform various actions, such as selecting objects, manipulating the camera view, and triggering specific events. These controls are implemented using the `ObjectManipulator` class (provided by `dexsim`). +## Default Window Controls -| Events | Description | -|---------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------| -| **Raycast Information Display** | Press the right mouse button to select a point and the 'C' key to print the raycast distance and hit position of a surface (world coordinates) to the console. Useful for debugging and checking the position of objects in the simulation. | -| **Viewer recording (toggle)** | Press **`r`** to **start** recording what the interactive viewer shows, and press **`r`** again to **stop** and save as MP4 videos. Recording uses a hidden camera that follows the live viewer camera pose, so the exported videos match the on-screen view. Useful for debugging and recording the demos.| +### Mouse Controls -> **Note:** We will add more interaction features in future releases. Stay tuned for updates! +| Input | Operation | +|-------|-----------| +| Left drag / Middle drag | Rotate around the current target point. | +| Right drag | Pan the camera and target together. | +| Mouse wheel | Dolly the camera closer to or farther from the target. | + +### Keyboard Controls + +| Input | Operation | +|-------|-----------| +| Space | Reset the window camera to its home view. | +| Left Ctrl + W / S | Temporarily translate the view forward / backward. | +| Left Ctrl + A / D | Temporarily translate the view left / right. | +| Left Ctrl + Q / E | Temporarily translate the view down / up. | + +In ORBIT mode, plain `W/A/S/D/Q/E` does not move the view. Hold **Left Ctrl** while pressing those keys to translate both the camera eye and target. + +### Selection and Focus + +| Input | Operation | +|-------|-----------| +| Left click | Select the object under the cursor in the main visualization window. | +| F | Focus the selected object and frame it in the view. | +| L | Toggle selection log output in the terminal. Selection logs are disabled by default. When enabled, left-clicking an object prints its id, name, world position, and rotation. | + +### EmbodiChain Extensions + +| Input | Operation | +|-------|-----------| +| **Viewer recording (toggle)** | Press **`r`** to **start** recording what the interactive viewer shows, and press **`r`** again to **stop** and save as MP4 videos. Recording uses a hidden camera that follows the live viewer camera pose, so the exported videos match the on-screen view. Useful for debugging and recording demos. | + +Recording hotkey registration is controlled by `SimConfig.window_record.enable_hotkey` (enabled by default). You can also call `SimulationManager.start_window_record()`, `stop_window_record()`, or `toggle_window_record()` programmatically. ## Customizing Window Events -Users can create their own custom window interaction controls by subclassing the `ObjectManipulator` class. This allows for the implementation of specific behaviors and responses to user inputs. +Users can create their own custom window interaction controls by subclassing the `ObjectManipulator` class (provided by `dexsim`). This allows for the implementation of specific behaviors and responses to user inputs. Here's an example of how to create a custom window event that responds to key presses: @@ -34,10 +62,11 @@ class CustomWindowEvent(ObjectManipulator): # sim_manager = SimulationManager(...) # Register the custom window event handler with the simulation: -sim_manager.add_custom_window_control(CustomWindowEvent()) +sim_manager.add_custom_window_control([CustomWindowEvent()]) ``` The functions table below summarizes the key methods available in the `ObjectManipulator` class for customizing window events: + | Method | Description | |----------------------|---------------------------------------------------------------------------------------------------| | `on_key_down(key)` | Triggered when a key is pressed down. The `key` parameter indicates which key was pressed. | @@ -46,3 +75,4 @@ The functions table below summarizes the key methods available in the `ObjectMan | `on_mouse_down(button, x, y)` | Triggered when a mouse button is pressed. The `button` parameter indicates which button was pressed, and `x`, `y` indicate the mouse position. | | `on_mouse_up(button, x, y)` | Triggered when a mouse button is released. The `button` parameter indicates which button was released, and `x`, `y` indicate the mouse position. | | `on_mouse_wheel(delta)` | Triggered when the mouse wheel is scrolled. The `delta` parameter indicates the amount of scroll. | +| `enable_selection_cache(enable)` | When enabled, caches the last raycast selection so `selected_object`, `selected_position`, and `selected_distance` are available in callbacks. | diff --git a/embodichain/lab/sim/sim_manager.py b/embodichain/lab/sim/sim_manager.py index 757c83c3..111958eb 100644 --- a/embodichain/lab/sim/sim_manager.py +++ b/embodichain/lab/sim/sim_manager.py @@ -238,7 +238,6 @@ def __init__( self._world: dexsim.World = dexsim.World(world_config) self._window: Windows | None = None - self._is_registered_window_control = False self._window_record_state: _WindowRecordState | None = None self._window_record_camera: object | None = None wr = sim_config.window_record @@ -307,7 +306,6 @@ def __init__( if sim_config.headless is False: self._window = self._world.get_windows() - self._register_default_window_control() @classmethod def get_instance(cls, instance_id: int = 0) -> SimulationManager: @@ -549,8 +547,6 @@ def open_window(self) -> None: self._world.open_window() self._window = self._world.get_windows() - # TODO: will open these features after fix the related blocking issues. - self._register_default_window_control() if ( self._window_record_hotkey_cfg is not None and self._window_record_input_control is None @@ -1645,26 +1641,6 @@ def remove_marker(self, name: str) -> bool: logger.log_warning(f"Failed to remove marker {name}: {str(e)}") return False - def _register_default_window_control(self) -> None: - """Register default window controls for better simulation interaction.""" - from dexsim.types import InputKey - - if self._is_registered_window_control: - return - - class WindowDefaultEvent(ObjectManipulator): - - def on_key_down(self, key): - if key == InputKey.SCANCODE_C.value: - print(f"Raycast distance: {self.selected_distance}") - print(f"Hit position: {self.selected_position}") - - manipulator = WindowDefaultEvent() - manipulator.enable_selection_cache(True) - self._window.add_input_control(manipulator) - - self._is_registered_window_control = True - def add_custom_window_control(self, controls: list[ObjectManipulator]) -> None: """Add one or more custom window input controls. From 7b947b44c3c0f9085aa7e7f9ca24fc80023b6f56 Mon Sep 17 00:00:00 2001 From: yuecideng Date: Thu, 4 Jun 2026 13:07:53 +0800 Subject: [PATCH 2/2] wip --- configs/gym/pour_water/gym_config.json | 41 -------------------------- 1 file changed, 41 deletions(-) diff --git a/configs/gym/pour_water/gym_config.json b/configs/gym/pour_water/gym_config.json index 1c3e2876..79727df3 100644 --- a/configs/gym/pour_water/gym_config.json +++ b/configs/gym/pour_water/gym_config.json @@ -219,43 +219,6 @@ "params": { "entity_cfg": {"uid": "cup"} } - }, - "cam_high_semantic_mask_l": { - "func": "compute_semantic_mask", - "mode": "add", - "name": "sensor/cam_high/semantic_mask_l", - "params": { - "entity_cfg": {"uid": "cam_high"}, - "foreground_uids": ["bottle", "cup"] - } - }, - "cam_high_semantic_mask_r": { - "func": "compute_semantic_mask", - "mode": "add", - "name": "sensor/cam_high/semantic_mask_r", - "params": { - "entity_cfg": {"uid": "cam_high"}, - "foreground_uids": ["bottle", "cup"], - "is_right": true - } - }, - "cam_left_semantic_mask": { - "func": "compute_semantic_mask", - "mode": "add", - "name": "sensor/cam_left_wrist/semantic_mask_l", - "params": { - "entity_cfg": {"uid": "cam_left_wrist"}, - "foreground_uids": ["bottle", "cup"] - } - }, - "cam_right_semantic_mask": { - "func": "compute_semantic_mask", - "mode": "add", - "name": "sensor/cam_right_wrist/semantic_mask_l", - "params": { - "entity_cfg": {"uid": "cam_right_wrist"}, - "foreground_uids": ["bottle", "cup"] - } } }, "dataset": { @@ -293,8 +256,6 @@ "uid": "cam_high", "width": 960, "height": 540, - "enable_mask": true, - "enable_depth": true, "left_to_right_pos": [0.059684025824163614, 0, 0], "intrinsics": [453.851402686215, 453.8347628855552, 469.827725021235, 258.6656181845155], "intrinsics_right": [453.4536601653505, 453.3306024582175, 499.13697412367776, 297.7176248477935], @@ -309,7 +270,6 @@ "uid": "cam_right_wrist", "width": 640, "height": 480, - "enable_mask": true, "intrinsics": [488.1665344238281, 488.1665344238281, 322.7323303222656, 213.17434692382812], "extrinsics": { "parent": "right_link6", @@ -322,7 +282,6 @@ "uid": "cam_left_wrist", "width": 640, "height": 480, - "enable_mask": true, "intrinsics": [488.1665344238281, 488.1665344238281, 322.7323303222656, 213.17434692382812], "extrinsics": { "parent": "left_link6",