Add clock sync server and client, and clock sync client library

examples/oxr/python/test_controller_tracker.py

@@ -57,9 +57,8 @@
 
         # Test 5: Check initial controller state
         print("[Test 5] Checking controller state...")
-        controller_data = controller_tracker.get_controller_data(session)
-        left_snap = controller_data.left_controller
-        right_snap = controller_data.right_controller
+        left_snap = controller_tracker.get_left_controller(session)
+        right_snap = controller_tracker.get_right_controller(session)
         print(
             f"  Left controller: {'ACTIVE' if left_snap and left_snap.is_active else 'INACTIVE'}"
         )
@@ -101,9 +100,8 @@
                 current_time = time.time()
                 if current_time - last_status_print >= 0.5:  # Print every 0.5 seconds
                     elapsed = current_time - start_time
-                    controller_data = controller_tracker.get_controller_data(session)
-                    left_snap = controller_data.left_controller
-                    right_snap = controller_data.right_controller
+                    left_snap = controller_tracker.get_left_controller(session)
+                    right_snap = controller_tracker.get_right_controller(session)
 
                     # Show current state
                     left_trigger = left_snap.inputs.trigger_value if left_snap else 0.0
@@ -212,10 +210,11 @@ def print_controller_summary(hand_name, snapshot):
             else:
                 print("    Status: INACTIVE")
 
-        controller_data = controller_tracker.get_controller_data(session)
-        print_controller_summary("Left", controller_data.left_controller)
+        left_snap = controller_tracker.get_left_controller(session)
+        right_snap = controller_tracker.get_right_controller(session)
+        print_controller_summary("Left", left_snap)
         print()
-        print_controller_summary("Right", controller_data.right_controller)
+        print_controller_summary("Right", right_snap)
         print()
 
         # Cleanup

examples/retargeting/python/sources_example.py

@@ -132,7 +132,10 @@ def main():
                     hand_left_raw = hand_tracker.get_left_hand(session)
                     hand_right_raw = hand_tracker.get_right_hand(session)
                     head_raw = head_tracker.get_head(session)
-                    controller_data_raw = controller_tracker.get_controller_data(
+                    left_controller_raw = controller_tracker.get_left_controller(
+                        session
+                    )
+                    right_controller_raw = controller_tracker.get_right_controller(
                         session
                     )
 
@@ -163,9 +166,9 @@ def main():
                     for input_name, group_type in controllers_input_spec.items():
                         tg = TensorGroup(group_type)
                         if "left" in input_name.lower():
-                            tg[0] = controller_data_raw.left_controller
+                            tg[0] = left_controller_raw
                         elif "right" in input_name.lower():
-                            tg[0] = controller_data_raw.right_controller
+                            tg[0] = right_controller_raw
                         controllers_inputs[input_name] = tg
 
                     # ====================================================

examples/schemaio/frame_metadata_printer.cpp

@@ -33,17 +33,7 @@ static constexpr size_t MAX_FLATBUFFER_SIZE = 128;
 
 void print_frame_metadata(const core::FrameMetadataT& data, size_t sample_count)
 {
-    std::cout << "Sample " << sample_count;
-
-    std::cout << " [seq=" << data.sequence_number;
-    if (data.timestamp)
-    {
-        std::cout << ", device_time=" << data.timestamp->device_time()
-                  << ", common_time=" << data.timestamp->common_time();
-    }
-    std::cout << "]";
-
-    std::cout << std::endl;
+    std::cout << "Sample " << sample_count << " [seq=" << data.sequence_number << "]" << std::endl;
 }
 
 void print_usage(const char* program_name)
@@ -123,9 +113,9 @@ try
             break;
         }
 
-        // Print when we have new data (timestamp indicates real data; sequence_number changed)
+        // Print when we have new data (sequence_number changed)
         const auto& data = tracker->get_data(*session);
-        if (data.timestamp && data.sequence_number != last_printed_sequence)
+        if (data.sequence_number != last_printed_sequence)
         {
             print_frame_metadata(data, ++received_count);
             last_printed_sequence = data.sequence_number;

examples/schemaio/pedal_pusher.cpp

@@ -51,13 +51,13 @@ class Generic3AxisPedalPusher
      * @param data The Generic3AxisPedalOutputT native object to serialize and push.
      * @throws std::runtime_error if the push fails.
      */
-    void push(const core::Generic3AxisPedalOutputT& data)
+    void push(const core::Generic3AxisPedalOutputT& data, int64_t device_time_ns, int64_t common_time_ns)
     {
         flatbuffers::FlatBufferBuilder builder(m_pusher.config().max_flatbuffer_size);
         auto offset = core::Generic3AxisPedalOutput::Pack(builder, &data);
         builder.Finish(offset);
 
-        m_pusher.push_buffer(builder.GetBufferPointer(), builder.GetSize());
+        m_pusher.push_buffer(builder.GetBufferPointer(), builder.GetSize(), device_time_ns, common_time_ns);
     }
 
 private:
@@ -105,9 +105,8 @@ try
 
         auto now = std::chrono::steady_clock::now();
         auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(now.time_since_epoch()).count();
-        pedal_output.timestamp = std::make_shared<core::Timestamp>(ns, ns);
 
-        pusher->push(pedal_output);
+        pusher->push(pedal_output, ns, ns);
         std::cout << "Pushed sample " << i << std::fixed << std::setprecision(3) << " [left=" << left_pedal
                   << ", right=" << right_pedal << ", rudder=" << rudder << "]" << std::endl;
 

src/core/CMakeLists.txt

@@ -36,6 +36,9 @@ add_subdirectory(retargeting_engine_ui)
 # Build TeleopSessionManager (pure Python module)
 add_subdirectory(teleop_session_manager)
 
+# Build Synchronization utilities (clock sync, etc.)
+add_subdirectory(synchronization)
+
 # Python wheel packaging (combines both modules)
 if(BUILD_PYTHON_BINDINGS)
     add_subdirectory(python)

src/core/deviceio/cpp/controller_tracker.cpp

@@ -278,16 +278,13 @@ bool ControllerTracker::Impl::update(XrTime time)
         return false;
     }
 
-    // Helper to update a single controller - creates a new immutable struct snapshot
     auto update_controller = [&](XrPath hand_path, const XrSpacePtr& grip_space, const XrSpacePtr& aim_space,
-                                 std::shared_ptr<ControllerSnapshot>& snapshot_ptr)
+                                 ControllerSnapshot& snapshot_out)
     {
-        // Initialize with default values
         ControllerPose grip_pose{};
         ControllerPose aim_pose{};
         ControllerInputState inputs{};
         bool is_active = false;
-        Timestamp timestamp{};
 
         // Update grip pose
         XrSpaceLocation grip_location{ XR_TYPE_SPACE_LOCATION };
@@ -321,9 +318,6 @@ bool ControllerTracker::Impl::update(XrTime time)
 
         is_active = grip_pose.is_valid() || aim_pose.is_valid();
 
-        // Update timestamp
-        timestamp = Timestamp(time, time);
-
         // Update all input values
         bool primary_click = get_boolean_action_state(session_, core_funcs_, primary_click_action_, hand_path);
         bool secondary_click = get_boolean_action_state(session_, core_funcs_, secondary_click_action_, hand_path);
@@ -338,37 +332,42 @@ bool ControllerTracker::Impl::update(XrTime time)
         inputs = ControllerInputState(
             primary_click, secondary_click, thumbstick_click, thumbstick_x, thumbstick_y, squeeze_value, trigger_value);
 
-        // Create new snapshot struct
-        snapshot_ptr = std::make_shared<ControllerSnapshot>(grip_pose, aim_pose, inputs, is_active, timestamp);
+        snapshot_out = ControllerSnapshot(grip_pose, aim_pose, inputs, is_active);
     };
 
-    // Update both controllers
-    update_controller(left_hand_path_, left_grip_space_, left_aim_space_, controller_data_.left_controller);
-    update_controller(right_hand_path_, right_grip_space_, right_aim_space_, controller_data_.right_controller);
+    update_controller(left_hand_path_, left_grip_space_, left_aim_space_, left_controller_);
+    update_controller(right_hand_path_, right_grip_space_, right_aim_space_, right_controller_);
+
+    last_timestamp_ = DeviceDataTimestamp(time, time, 0);
 
-    return controller_data_.left_controller->is_active() || controller_data_.right_controller->is_active();
+    return left_controller_.is_active() || right_controller_.is_active();
 }
 
-const ControllerDataT& ControllerTracker::Impl::get_controller_data() const
+const ControllerSnapshot& ControllerTracker::Impl::get_left_controller() const
 {
-    return controller_data_;
+    return left_controller_;
 }
 
-Timestamp ControllerTracker::Impl::serialize(flatbuffers::FlatBufferBuilder& builder) const
+const ControllerSnapshot& ControllerTracker::Impl::get_right_controller() const
 {
-    auto offset = ControllerData::Pack(builder, &controller_data_);
-    builder.Finish(offset);
+    return right_controller_;
+}
 
-    // Use left controller timestamp (or right if left is inactive)
-    if (controller_data_.left_controller && controller_data_.left_controller->is_active())
-    {
-        return controller_data_.left_controller->timestamp();
-    }
-    else if (controller_data_.right_controller && controller_data_.right_controller->is_active())
-    {
-        return controller_data_.right_controller->timestamp();
-    }
-    return Timestamp{};
+XrTime ControllerTracker::Impl::get_last_update_time() const
+{
+    return last_timestamp_.sample_time_device_clock();
+}
+
+DeviceDataTimestamp ControllerTracker::Impl::serialize(flatbuffers::FlatBufferBuilder& builder, size_t channel_index) const
+{
+    const ControllerSnapshot& snapshot = (channel_index == 0) ? left_controller_ : right_controller_;
+
+    ControllerSnapshotRecordBuilder record_builder(builder);
+    record_builder.add_data(&snapshot);
+    record_builder.add_timestamp(&last_timestamp_);
+    builder.Finish(record_builder.Finish());
+
+    return last_timestamp_;
 }
 
 // ============================================================================
@@ -381,9 +380,19 @@ std::vector<std::string> ControllerTracker::get_required_extensions() const
     return {};
 }
 
-const ControllerDataT& ControllerTracker::get_controller_data(const DeviceIOSession& session) const
+const ControllerSnapshot& ControllerTracker::get_left_controller(const DeviceIOSession& session) const
+{
+    return static_cast<const Impl&>(session.get_tracker_impl(*this)).get_left_controller();
+}
+
+const ControllerSnapshot& ControllerTracker::get_right_controller(const DeviceIOSession& session) const
+{
+    return static_cast<const Impl&>(session.get_tracker_impl(*this)).get_right_controller();
+}
+
+XrTime ControllerTracker::get_last_update_time(const DeviceIOSession& session) const
 {
-    return static_cast<const Impl&>(session.get_tracker_impl(*this)).get_controller_data();
+    return static_cast<const Impl&>(session.get_tracker_impl(*this)).get_last_update_time();
 }
 
 std::shared_ptr<ITrackerImpl> ControllerTracker::create_tracker(const OpenXRSessionHandles& handles) const

src/core/deviceio/cpp/frame_metadata_tracker_oak.cpp

@@ -26,25 +26,57 @@ class FrameMetadataTrackerOak::Impl : public ITrackerImpl
 
     bool update(XrTime /* time */) override
     {
-        // Try to read new data from tensor stream
-        if (m_schema_reader.read_buffer(m_buffer))
+        m_pending_records.clear();
+
+        std::vector<SchemaTracker::SampleResult> raw_samples;
+        m_schema_reader.read_all_samples(raw_samples);
+
+        for (auto& sample : raw_samples)
         {
-            auto fb = GetFrameMetadata(m_buffer.data());
+            auto fb = flatbuffers::GetRoot<FrameMetadata>(sample.buffer.data());
             if (fb)
             {
-                fb->UnPackTo(&m_data);
-                return true;
+                FrameMetadataT parsed;
+                fb->UnPackTo(&parsed);
+                m_pending_records.push_back({ std::move(parsed), sample.timestamp });
             }
         }
-        // Return true even if no new data - we're still running
+
+        if (!m_pending_records.empty())
+        {
+            m_data = m_pending_records.back().data;
+            m_last_timestamp = m_pending_records.back().timestamp;
+        }
+
         return true;
     }
 
-    Timestamp serialize(flatbuffers::FlatBufferBuilder& builder) const override
+    DeviceDataTimestamp serialize(flatbuffers::FlatBufferBuilder& builder, size_t /* channel_index */) const override
+    {
+        auto data_offset = FrameMetadata::Pack(builder, &m_data);
+
+        FrameMetadataRecordBuilder record_builder(builder);
+        record_builder.add_data(data_offset);
+        record_builder.add_timestamp(&m_last_timestamp);
+        builder.Finish(record_builder.Finish());
+
+        return m_last_timestamp;
+    }
+
+    void serialize_all(size_t /* channel_index */, const RecordCallback& callback) const override
     {
-        auto offset = FrameMetadata::Pack(builder, &m_data);
-        builder.Finish(offset);
-        return m_data.timestamp ? *m_data.timestamp : Timestamp{};
+        for (const auto& record : m_pending_records)
+        {
+            flatbuffers::FlatBufferBuilder builder(256);
+            auto data_offset = FrameMetadata::Pack(builder, &record.data);
+
+            FrameMetadataRecordBuilder record_builder(builder);
+            record_builder.add_data(data_offset);
+            record_builder.add_timestamp(&record.timestamp);
+            builder.Finish(record_builder.Finish());
+
+            callback(record.timestamp, builder.GetBufferPointer(), builder.GetSize());
+        }
     }
 
     const FrameMetadataT& get_data() const
@@ -53,9 +85,16 @@ class FrameMetadataTrackerOak::Impl : public ITrackerImpl
     }
 
 private:
+    struct PendingRecord
+    {
+        FrameMetadataT data;
+        DeviceDataTimestamp timestamp;
+    };
+
     SchemaTracker m_schema_reader;
-    std::vector<uint8_t> m_buffer;
     FrameMetadataT m_data;
+    DeviceDataTimestamp m_last_timestamp{};
+    std::vector<PendingRecord> m_pending_records;
 };
 
 // ============================================================================
@@ -82,13 +121,18 @@ std::string_view FrameMetadataTrackerOak::get_name() const
 
 std::string_view FrameMetadataTrackerOak::get_schema_name() const
 {
-    return "core.FrameMetadata";
+    return "core.FrameMetadataRecord";
 }
 
 std::string_view FrameMetadataTrackerOak::get_schema_text() const
 {
-    return std::string_view(
-        reinterpret_cast<const char*>(FrameMetadataBinarySchema::data()), FrameMetadataBinarySchema::size());
+    return std::string_view(reinterpret_cast<const char*>(FrameMetadataRecordBinarySchema::data()),
+                            FrameMetadataRecordBinarySchema::size());
+}
+
+std::vector<std::string> FrameMetadataTrackerOak::get_record_channels() const
+{
+    return { "frame_metadata" };
 }
 
 const SchemaTrackerConfig& FrameMetadataTrackerOak::get_config() const