refactor: replace HandInjector with SchemaPusher pipeline for Haply device

CMakeLists.txt

@@ -143,6 +143,7 @@ if(BUILD_PLUGINS)
     add_subdirectory(src/plugins/controller_synthetic_hands)
     add_subdirectory(src/plugins/generic_3axis_pedal)
     add_subdirectory(src/plugins/manus)
+    add_subdirectory(src/plugins/haply)
     if(BUILD_PLUGIN_OAK_CAMERA)
         add_subdirectory(src/plugins/oak)
     endif()

src/core/deviceio/cpp/CMakeLists.txt

@@ -23,6 +23,7 @@ add_library(deviceio_core STATIC
     controller_tracker.cpp
     schema_tracker.cpp
     generic_3axis_pedal_tracker.cpp
+    haply_device_tracker.cpp
     frame_metadata_tracker_oak.cpp
     full_body_tracker_pico.cpp
     deviceio_session.cpp
@@ -32,6 +33,7 @@ add_library(deviceio_core STATIC
     inc/deviceio/controller_tracker.hpp
     inc/deviceio/schema_tracker.hpp
     inc/deviceio/generic_3axis_pedal_tracker.hpp
+    inc/deviceio/haply_device_tracker.hpp
     inc/deviceio/frame_metadata_tracker_oak.hpp
     inc/deviceio/deviceio_session.hpp
 )

src/core/deviceio/cpp/haply_device_tracker.cpp

@@ -0,0 +1,176 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+#include "inc/deviceio/haply_device_tracker.hpp"
+
+#include "inc/deviceio/deviceio_session.hpp"
+
+#include <flatbuffers/flatbuffers.h>
+#include <oxr_utils/oxr_time.hpp>
+#include <schema/haply_device_bfbs_generated.h>
+
+#include <vector>
+
+namespace core
+{
+
+// ============================================================================
+// HaplyDeviceTracker::Impl
+// ============================================================================
+
+class HaplyDeviceTracker::Impl : public ITrackerImpl
+{
+public:
+    Impl(const OpenXRSessionHandles& handles, SchemaTrackerConfig config);
+
+    bool update(XrTime time) override;
+    void serialize_all(size_t channel_index, const RecordCallback& callback) const override;
+
+    const HaplyDeviceOutputTrackedT& get_data() const;
+
+private:
+    SchemaTracker m_schema_reader;
+    XrTimeConverter m_time_converter_;
+    XrTime m_last_update_time_ = 0;
+    bool m_collection_present = false;
+    HaplyDeviceOutputTrackedT m_tracked;
+    std::vector<SchemaTracker::SampleResult> m_pending_records;
+};
+
+HaplyDeviceTracker::Impl::Impl(const OpenXRSessionHandles& handles, SchemaTrackerConfig config)
+    : m_schema_reader(handles, std::move(config)), m_time_converter_(handles)
+{
+}
+
+bool HaplyDeviceTracker::Impl::update(XrTime time)
+{
+    m_last_update_time_ = time;
+    m_pending_records.clear();
+    m_collection_present = m_schema_reader.read_all_samples(m_pending_records);
+
+    if (!m_collection_present)
+    {
+        // Device disappeared: clear tracked data so get_data() reflects absence.
+        m_tracked.data.reset();
+        return true;
+    }
+
+    // Deserialize only the last sample to keep m_tracked current for get_data().
+    // Full per-sample deserialization is deferred to serialize_all().
+    if (!m_pending_records.empty())
+    {
+        auto fb = flatbuffers::GetRoot<HaplyDeviceOutput>(m_pending_records.back().buffer.data());
+        if (fb)
+        {
+            if (!m_tracked.data)
+            {
+                m_tracked.data = std::make_shared<HaplyDeviceOutputT>();
+            }
+            fb->UnPackTo(m_tracked.data.get());
+        }
+    }
+    // When no samples arrive but the collection is present, m_tracked retains
+    // the last seen value so get_data() reflects the most recently received state.
+
+    return true;
+}
+
+void HaplyDeviceTracker::Impl::serialize_all(size_t channel_index, const RecordCallback& callback) const
+{
+    if (channel_index != 0)
+    {
+        return;
+    }
+
+    int64_t update_ns = m_time_converter_.convert_xrtime_to_monotonic_ns(m_last_update_time_);
+
+    if (m_pending_records.empty())
+    {
+        if (!m_collection_present)
+        {
+            // Device disappeared: emit one empty record to mark the absence in the MCAP stream.
+            DeviceDataTimestamp update_timestamp(update_ns, 0, 0);
+            flatbuffers::FlatBufferBuilder builder(64);
+            HaplyDeviceOutputRecordBuilder record_builder(builder);
+            record_builder.add_timestamp(&update_timestamp);
+            builder.Finish(record_builder.Finish());
+            callback(update_ns, builder.GetBufferPointer(), builder.GetSize());
+        }
+        return;
+    }
+
+    for (const auto& sample : m_pending_records)
+    {
+        auto fb = flatbuffers::GetRoot<HaplyDeviceOutput>(sample.buffer.data());
+        if (!fb)
+        {
+            continue;
+        }
+
+        HaplyDeviceOutputT parsed;
+        fb->UnPackTo(&parsed);
+
+        flatbuffers::FlatBufferBuilder builder(256);
+        auto data_offset = HaplyDeviceOutput::Pack(builder, &parsed);
+        HaplyDeviceOutputRecordBuilder record_builder(builder);
+        record_builder.add_data(data_offset);
+        record_builder.add_timestamp(&sample.timestamp);
+        builder.Finish(record_builder.Finish());
+        callback(update_ns, builder.GetBufferPointer(), builder.GetSize());
+    }
+}
+
+const HaplyDeviceOutputTrackedT& HaplyDeviceTracker::Impl::get_data() const
+{
+    return m_tracked;
+}
+
+// ============================================================================
+// HaplyDeviceTracker
+// ============================================================================
+
+HaplyDeviceTracker::HaplyDeviceTracker(const std::string& collection_id, size_t max_flatbuffer_size)
+    : m_config{.collection_id = collection_id,
+               .max_flatbuffer_size = max_flatbuffer_size,
+               .tensor_identifier = "haply_device",
+               .localized_name = "HaplyDeviceTracker"}
+{
+}
+
+std::vector<std::string> HaplyDeviceTracker::get_required_extensions() const
+{
+    return SchemaTracker::get_required_extensions();
+}
+
+std::string_view HaplyDeviceTracker::get_name() const
+{
+    return "HaplyDeviceTracker";
+}
+
+std::string_view HaplyDeviceTracker::get_schema_name() const
+{
+    return "core.HaplyDeviceOutputRecord";
+}
+
+std::string_view HaplyDeviceTracker::get_schema_text() const
+{
+    return std::string_view(reinterpret_cast<const char*>(HaplyDeviceOutputRecordBinarySchema::data()),
+                            HaplyDeviceOutputRecordBinarySchema::size());
+}
+
+const SchemaTrackerConfig& HaplyDeviceTracker::get_config() const
+{
+    return m_config;
+}
+
+const HaplyDeviceOutputTrackedT& HaplyDeviceTracker::get_data(const DeviceIOSession& session) const
+{
+    return static_cast<const Impl&>(session.get_tracker_impl(*this)).get_data();
+}
+
+std::shared_ptr<ITrackerImpl> HaplyDeviceTracker::create_tracker(const OpenXRSessionHandles& handles) const
+{
+    return std::make_shared<Impl>(handles, get_config());
+}
+
+} // namespace core

src/core/deviceio/cpp/inc/deviceio/haply_device_tracker.hpp

@@ -0,0 +1,77 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include "schema_tracker.hpp"
+#include "tracker.hpp"
+
+#include <schema/haply_device_generated.h>
+
+#include <memory>
+#include <string>
+
+namespace core
+{
+
+/*!
+ * @brief Tracker for reading HaplyDeviceOutput FlatBuffer messages via OpenXR tensor extensions.
+ *
+ * This tracker reads Haply Inverse3 + VerseGrip device state (cursor position,
+ * velocity, orientation, buttons) pushed by the Haply plugin using the
+ * SchemaTracker utility. Both pusher and reader must agree on the collection_id
+ * and use the HaplyDeviceOutput schema.
+ *
+ * Usage:
+ * @code
+ * auto tracker = std::make_shared<HaplyDeviceTracker>("haply_device");
+ * // ... create DeviceIOSession with tracker ...
+ * session->update();
+ * const auto& data = tracker->get_data(*session);
+ * @endcode
+ */
+class HaplyDeviceTracker : public ITracker
+{
+public:
+    //! Default maximum FlatBuffer size for HaplyDeviceOutput messages.
+    static constexpr size_t DEFAULT_MAX_FLATBUFFER_SIZE = 256;
+
+    /*!
+     * @brief Constructs a HaplyDeviceTracker.
+     * @param collection_id Tensor collection identifier for discovery.
+     * @param max_flatbuffer_size Maximum serialized FlatBuffer size (default: 256 bytes).
+     */
+    explicit HaplyDeviceTracker(const std::string& collection_id = "haply_device",
+                                size_t max_flatbuffer_size = DEFAULT_MAX_FLATBUFFER_SIZE);
+
+    // ITracker interface
+    std::vector<std::string> get_required_extensions() const override;
+    std::string_view get_name() const override;
+    std::string_view get_schema_name() const override;
+    std::string_view get_schema_text() const override;
+
+    std::vector<std::string> get_record_channels() const override
+    {
+        return {"haply_device"};
+    }
+
+    /*!
+     * @brief Get the current Haply device data (tracked.data is null when no data available).
+     */
+    const HaplyDeviceOutputTrackedT& get_data(const DeviceIOSession& session) const;
+
+protected:
+    /*!
+     * @brief Access the configuration for subclass use.
+     */
+    const SchemaTrackerConfig& get_config() const;
+
+private:
+    std::shared_ptr<ITrackerImpl> create_tracker(const OpenXRSessionHandles& handles) const override;
+
+    SchemaTrackerConfig m_config;
+
+    class Impl;
+};
+
+} // namespace core

src/core/deviceio/python/deviceio_bindings.cpp

@@ -6,6 +6,7 @@
 #include <deviceio/full_body_tracker_pico.hpp>
 #include <deviceio/generic_3axis_pedal_tracker.hpp>
 #include <deviceio/hand_tracker.hpp>
+#include <deviceio/haply_device_tracker.hpp>
 #include <deviceio/head_tracker.hpp>
 #include <deviceio_py_utils/session.hpp>
 #include <openxr/openxr.h>
@@ -101,6 +102,20 @@ PYBIND11_MODULE(_deviceio, m)
             py::arg("session"), py::return_value_policy::reference_internal,
             "Get full body pose tracked state (data is None if inactive)");
 
+    // HaplyDeviceTracker class
+    py::class_<core::HaplyDeviceTracker, core::ITracker, std::shared_ptr<core::HaplyDeviceTracker>>(
+        m, "HaplyDeviceTracker")
+        .def(py::init<const std::string&, size_t>(), py::arg("collection_id") = "haply_device",
+             py::arg("max_flatbuffer_size") = core::HaplyDeviceTracker::DEFAULT_MAX_FLATBUFFER_SIZE,
+             "Construct a HaplyDeviceTracker for the given tensor collection ID")
+        .def(
+            "get_data",
+            [](core::HaplyDeviceTracker& self,
+               PyDeviceIOSession& session) -> const core::HaplyDeviceOutputTrackedT&
+            { return self.get_data(session.native()); },
+            py::arg("session"), py::return_value_policy::reference_internal,
+            "Get the current Haply device tracked state (data is None when no data available)");
+
     // DeviceIOSession class (bound via wrapper for context management)
     // Other C++ modules (like mcap) should include <py_deviceio/session.hpp> and accept
     // PyDeviceIOSession& directly, calling .native() internally in C++ code.

src/core/deviceio/python/deviceio_init.py

@@ -20,6 +20,7 @@
     ControllerTracker,
     FrameMetadataTrackerOak,
     Generic3AxisPedalTracker,
+    HaplyDeviceTracker,
     FullBodyTrackerPico,
     DeviceIOSession,
     NUM_JOINTS,
@@ -57,6 +58,7 @@
     "ControllerTracker",
     "FrameMetadataTrackerOak",
     "Generic3AxisPedalTracker",
+    "HaplyDeviceTracker",
     "FullBodyTrackerPico",
     "OpenXRSessionHandles",
     "DeviceIOSession",

src/core/retargeting_engine/python/deviceio_source_nodes/__init__.py

@@ -8,17 +8,20 @@
 from .hands_source import HandsSource
 from .controllers_source import ControllersSource
 from .pedals_source import Generic3AxisPedalSource
+from .haply_source import HaplyDeviceSource
 from .full_body_source import FullBodySource
 from .deviceio_tensor_types import (
     HeadPoseTrackedType,
     HandPoseTrackedType,
     ControllerSnapshotTrackedType,
     Generic3AxisPedalOutputTrackedType,
+    HaplyDeviceOutputTrackedType,
     FullBodyPosePicoTrackedType,
     DeviceIOHeadPoseTracked,
     DeviceIOHandPoseTracked,
     DeviceIOControllerSnapshotTracked,
     DeviceIOGeneric3AxisPedalOutputTracked,
+    DeviceIOHaplyDeviceOutputTracked,
     DeviceIOFullBodyPosePicoTracked,
 )
 
@@ -28,15 +31,18 @@
     "HandsSource",
     "ControllersSource",
     "Generic3AxisPedalSource",
+    "HaplyDeviceSource",
     "FullBodySource",
     "HeadPoseTrackedType",
     "HandPoseTrackedType",
     "ControllerSnapshotTrackedType",
     "Generic3AxisPedalOutputTrackedType",
+    "HaplyDeviceOutputTrackedType",
     "FullBodyPosePicoTrackedType",
     "DeviceIOHeadPoseTracked",
     "DeviceIOHandPoseTracked",
     "DeviceIOControllerSnapshotTracked",
     "DeviceIOGeneric3AxisPedalOutputTracked",
+    "DeviceIOHaplyDeviceOutputTracked",
     "DeviceIOFullBodyPosePicoTracked",
 ]