feat: add SO-101 robot arm teleoperation plugin

CMakeLists.txt

@@ -143,6 +143,8 @@ if(BUILD_PLUGINS)
     add_subdirectory(src/plugins/controller_synthetic_hands)
     add_subdirectory(src/plugins/generic_3axis_pedal)
     add_subdirectory(src/plugins/manus)
+    add_subdirectory(src/plugins/haply)
+    add_subdirectory(src/plugins/so101)
     if(BUILD_PLUGIN_OAK_CAMERA)
         add_subdirectory(src/plugins/oak)
     endif()

src/core/schema/fbs/so101_arm.fbs

@@ -0,0 +1,35 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+include "timestamp.fbs";
+
+namespace core;
+
+// Joint positions from an SO-101 6-DOF robot arm (leader arm for teleoperation).
+// Positions are normalized to [-1.0, 1.0] range from the servo's 0-4095 raw range,
+// centered around the calibrated midpoint.
+table SO101ArmOutput {
+  shoulder_pan: float (id: 0);
+  shoulder_lift: float (id: 1);
+  elbow_flex: float (id: 2);
+  wrist_flex: float (id: 3);
+  wrist_roll: float (id: 4);
+  gripper: float (id: 5);     // 0.0 = open, 1.0 = closed
+}
+
+// Tracked wrapper for the in-memory tracker API (data is null when no arm data available).
+table SO101ArmOutputTracked {
+    data: SO101ArmOutput (id: 0);
+}
+
+// MCAP recording wrapper for SO101ArmOutput.
+//
+// Record types are the root types written to MCAP channels by the McapRecorder.
+// Trackers serialize into Record types via their serialize() method, but the
+// public query API returns the inner data type directly.
+table SO101ArmOutputRecord {
+    data: SO101ArmOutput (id: 0);
+    timestamp: DeviceDataTimestamp (id: 1);
+}
+
+root_type SO101ArmOutputRecord;

src/plugins/haply/.gitignore

@@ -0,0 +1,5 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Build artifacts
+build*/

src/plugins/haply/CMakeLists.txt

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+include(GNUInstallDirs)
+
+find_package(Threads REQUIRED)
+
+set(HAPLY_PLUGIN_INC_DIR "${CMAKE_CURRENT_SOURCE_DIR}/inc")
+
+add_subdirectory(core)
+add_library(isaac::haply_plugin ALIAS haply_plugin_core)
+
+add_subdirectory(app)
+add_subdirectory(tools)

src/plugins/haply/CMakePresets.json

@@ -0,0 +1,35 @@
+{
+  "version": 3,
+  "cmakeMinimumRequired": { "major": 3, "minor": 22, "patch": 0 },
+  "configurePresets": [
+    {
+      "name": "linux-haply-default",
+      "displayName": "Linux Haply Default (RelWithDebInfo)",
+      "generator": "Unix Makefiles",
+      "binaryDir": "${sourceDir}/build-haply-default",
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "RelWithDebInfo"
+      }
+    },
+    {
+      "name": "linux-haply-debug",
+      "displayName": "Linux Haply Debug",
+      "inherits": "linux-haply-default",
+      "binaryDir": "${sourceDir}/build-haply-debug",
+      "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug" }
+    },
+    {
+      "name": "linux-haply-release",
+      "displayName": "Linux Haply Release",
+      "inherits": "linux-haply-default",
+      "binaryDir": "${sourceDir}/build-haply-release",
+      "cacheVariables": { "CMAKE_BUILD_TYPE": "Release" }
+    }
+  ],
+  "buildPresets": [
+    { "name": "haply-default", "configurePreset": "linux-haply-default" },
+    { "name": "haply-debug",   "configurePreset": "linux-haply-debug" },
+    { "name": "haply-release", "configurePreset": "linux-haply-release" }
+  ],
+  "testPresets": []
+}

src/plugins/haply/CMakePresets.json.license

@@ -0,0 +1,2 @@
+SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0

src/plugins/haply/README.md

@@ -0,0 +1,85 @@
+<!--
+SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+-->
+
+# Isaac Teleop Device Plugins — Haply
+
+This folder provides a Linux-only plugin for using the Haply Inverse3 haptic device and VerseGrip controller for hand tracking within the Isaac Teleop framework.
+
+## Components
+
+- **Core Library** (`haply_plugin_core`): Connects to the Haply SDK service via WebSocket and injects hand data into the OpenXR pipeline (`libIsaacTeleopPluginsHaply.so`).
+- **Plugin Executable** (`haply_hand_plugin`): The main plugin executable that integrates with the Teleop system.
+- **CLI Tool** (`haply_hand_tracker_printer`): A standalone debugging tool that connects directly to the Haply SDK WebSocket and prints raw device state. Does not require OpenXR.
+
+## Prerequisites
+
+- **Linux** (x86_64, tested on Ubuntu 22.04)
+- **Haply SDK** runtime service running on localhost (or a reachable host)
+- **Haply Inverse3** haptic device connected
+- **VerseGrip** controller paired (optional — provides orientation and buttons)
+
+## Environment Variables
+
+| Variable | Default | Description |
+|---|---|---|
+| `HAPLY_WEBSOCKET_HOST` | `127.0.0.1` | Hostname or IP of the Haply SDK WebSocket service |
+| `HAPLY_WEBSOCKET_PORT` | `10001` | Port of the Haply SDK WebSocket service |
+
+## Build
+
+This plugin is built as part of the Isaac Teleop project. No external SDK files are needed at build time — the plugin connects to the Haply SDK service at runtime via WebSocket.
+
+Run the project build from the repository root:
+
+```bash
+cmake -S . -B build
+cmake --build build -j
+```
+
+## Running
+
+### 1. Start the Haply SDK Service
+
+Ensure the Haply SDK runtime service is running and the Inverse3 + VerseGrip hardware is connected. The service should be listening on the configured WebSocket host and port.
+
+### 2. Verify with CLI Tool
+
+Use the standalone printer tool to confirm data reception before starting the full plugin:
+
+```bash
+./build/bin/haply_hand_tracker_printer
+```
+
+You should see position, velocity, orientation, and button data printed to the terminal. Press `Ctrl+C` to stop.
+
+### 3. Run the Plugin
+
+```bash
+./install/plugins/haply/haply_hand_plugin
+```
+
+The plugin will:
+1. Connect to the Haply SDK WebSocket service
+2. Initialize an OpenXR session
+3. Map Inverse3 cursor position and VerseGrip orientation to an OpenXR hand
+4. Push hand joint data at 90 Hz
+
+## How It Works
+
+The Haply Inverse3 provides a 3-DOF cursor position in space. The VerseGrip adds orientation and button inputs. The plugin maps this data to an OpenXR hand model:
+
+- **Wrist / Palm**: Positioned at the Inverse3 cursor position with VerseGrip orientation
+- **Finger joints**: Synthesized at the wrist position (valid but not individually tracked)
+- **Root tracking**: If an OpenXR controller is active, its aim pose is used as the root coordinate frame
+
+## Troubleshooting
+
+- **Connection failed**: Ensure the Haply SDK service is running and reachable at the configured host/port. Check `HAPLY_WEBSOCKET_HOST` and `HAPLY_WEBSOCKET_PORT`.
+- **No data appearing**: Verify the Inverse3 is powered on and connected. Use `haply_hand_tracker_printer` to test independently of OpenXR.
+- **Reconnection**: The plugin automatically reconnects with exponential backoff if the WebSocket connection is lost.
+
+## License
+
+Source files are under the Apache-2.0 license. The Haply SDK service is a separate component not distributed by this project.

src/plugins/haply/app/CMakeLists.txt

@@ -0,0 +1,33 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+add_executable(haply_hand_plugin
+  main.cpp
+)
+
+target_include_directories(haply_hand_plugin
+  PRIVATE
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../inc>
+)
+
+target_link_libraries(haply_hand_plugin
+  PRIVATE
+    haply_plugin_core
+)
+
+set_target_properties(haply_hand_plugin PROPERTIES
+  CXX_VISIBILITY_PRESET hidden
+  VISIBILITY_INLINES_HIDDEN YES
+  BUILD_WITH_INSTALL_RPATH NO
+  SKIP_BUILD_RPATH NO
+  BUILD_RPATH "$ORIGIN;$ORIGIN/../lib"
+  INSTALL_RPATH "$ORIGIN/../../${CMAKE_INSTALL_LIBDIR}"
+)
+
+install(TARGETS haply_hand_plugin
+    RUNTIME DESTINATION plugins/haply
+)
+
+install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/../plugin.yaml" "${CMAKE_CURRENT_SOURCE_DIR}/../README.md"
+    DESTINATION plugins/haply
+)

src/plugins/haply/app/main.cpp

@@ -0,0 +1,60 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+#include <core/haply_hand_tracking_plugin.hpp>
+
+#include <atomic>
+#include <chrono>
+#include <csignal>
+#include <iostream>
+#include <thread>
+
+using namespace plugins::haply;
+
+namespace
+{
+std::atomic<bool> g_should_exit{ false };
+
+void signal_handler(int)
+{
+    g_should_exit.store(true);
+}
+} // namespace
+
+int main(int argc, char** argv)
+try
+{
+    std::cout << "Haply Hand Plugin starting..." << std::endl;
+
+    std::signal(SIGINT, signal_handler);
+    std::signal(SIGTERM, signal_handler);
+
+    auto& tracker = HaplyTracker::instance();
+
+    std::cout << "Plugin running. Press Ctrl+C to stop." << std::endl;
+
+    // Target 90Hz frequency (~11.1ms period)
+    const auto target_frame_duration = std::chrono::nanoseconds(1000000000 / 90);
+
+    while (!g_should_exit.load())
+    {
+        auto frame_start = std::chrono::steady_clock::now();
+
+        tracker.update();
+
+        std::this_thread::sleep_until(frame_start + target_frame_duration);
+    }
+
+    std::cout << "Haply Hand Plugin shutting down." << std::endl;
+    return 0;
+}
+catch (const std::exception& e)
+{
+    std::cerr << argv[0] << ": " << e.what() << std::endl;
+    return 1;
+}
+catch (...)
+{
+    std::cerr << argv[0] << ": Unknown error occurred" << std::endl;
+    return 1;
+}

src/plugins/haply/core/CMakeLists.txt

@@ -0,0 +1,42 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+add_library(haply_plugin_core SHARED
+  haply_hand_tracking_plugin.cpp
+  ${HAPLY_PLUGIN_INC_DIR}/core/haply_hand_tracking_plugin.hpp
+)
+
+target_include_directories(haply_plugin_core
+  PUBLIC
+    $<BUILD_INTERFACE:${HAPLY_PLUGIN_INC_DIR}>
+  PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}/../third_party
+)
+
+target_link_libraries(haply_plugin_core
+  PUBLIC
+    Teleop::plugin_utils
+    deviceio::deviceio_core
+    OpenXR::openxr_loader
+    oxr::oxr_core
+  PRIVATE
+    oxr::oxr_utils
+    Teleop::openxr_extensions
+    Threads::Threads
+)
+
+set_target_properties(haply_plugin_core PROPERTIES
+  OUTPUT_NAME IsaacTeleopPluginsHaply
+  LIBRARY_OUTPUT_DIRECTORY "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}"
+  CXX_VISIBILITY_PRESET hidden
+  VISIBILITY_INLINES_HIDDEN YES
+  BUILD_WITH_INSTALL_RPATH NO
+  SKIP_BUILD_RPATH NO
+  BUILD_RPATH "$ORIGIN;$ORIGIN/../lib"
+  INSTALL_RPATH "$ORIGIN"
+)
+
+install(TARGETS haply_plugin_core
+    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+)