enable preview for oak camera

examples/oxr/python/test_oak_camera.py

@@ -185,6 +185,7 @@ def run_test(duration: float = 10.0, mode: str = MODE_NO_METADATA):
     extra_args = [
         f"--add-stream=camera=Color,output={color_path}",
         f"--add-stream=camera=MonoLeft,output={mono_left_path}",
+        "--preview",
     ]
 
     # For plugin-mcap, resolve to absolute path so the plugin can find it

src/plugins/oak/CMakeLists.txt

@@ -18,6 +18,7 @@ message(STATUS "Configuring DepthAI...")
 set(DEPTHAI_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
 set(DEPTHAI_BUILD_TESTS OFF CACHE BOOL "" FORCE)
 set(DEPTHAI_BUILD_DOCS OFF CACHE BOOL "" FORCE)
+set(DEPTHAI_OPENCV_SUPPORT OFF CACHE BOOL "" FORCE)
 
 FetchContent_Declare(
     depthai
@@ -28,6 +29,23 @@ FetchContent_MakeAvailable(depthai)
 
 message(STATUS "Building OAK camera plugin with DepthAI ${depthai_VERSION}")
 
+# ==============================================================================
+# SDL2 (for live preview window, bundled via FetchContent)
+# ==============================================================================
+set(SDL2_VERSION "2.32.10")
+
+set(SDL_TEST OFF CACHE BOOL "" FORCE)
+set(SDL_SHARED OFF CACHE BOOL "" FORCE)
+set(SDL_STATIC ON CACHE BOOL "" FORCE)
+
+FetchContent_Declare(
+    sdl2
+    URL "https://github.com/libsdl-org/SDL/releases/download/release-${SDL2_VERSION}/SDL2-${SDL2_VERSION}.tar.gz"
+    URL_HASH "SHA256=5f5993c530f084535c65a6879e9b26ad441169b3e25d789d83287040a9ca5165"
+)
+FetchContent_MakeAvailable(sdl2)
+message(STATUS "SDL2 ${SDL2_VERSION} — live preview support enabled")
+
 # ==============================================================================
 # Build OAK Plugin
 # ==============================================================================
@@ -36,6 +54,7 @@ add_executable(camera_plugin_oak
     core/oak_camera.cpp
     core/rawdata_writer.cpp
     core/frame_sink.cpp
+    core/preview_stream.cpp
 )
 
 target_link_libraries(camera_plugin_oak
@@ -45,9 +64,10 @@ target_link_libraries(camera_plugin_oak
         mcap::mcap
         oxr::oxr_core
         pusherio::pusherio
+        SDL2::SDL2-static
 )
 
-# Set RPATH to find bundled libraries in same directory
+# Set RPATH so the binary finds bundled shared libraries (e.g. libusb) in its own directory
 set_target_properties(camera_plugin_oak PROPERTIES
     INSTALL_RPATH "$ORIGIN"
 )

src/plugins/oak/README.md

@@ -22,18 +22,30 @@ DepthAI is fetched and built automatically via FetchContent. The first build tak
 cd IsaacTeleop
 
 # Configure and build
-cmake -B build
+cmake -B build -DBUILD_PLUGIN_OAK_CAMERA=ON
 cmake --build build --target camera_plugin_oak --parallel
 ```
 
 ## Usage
 
 ```bash
-# Record to a file (--output is required)
-./build/src/plugins/oak/camera_plugin_oak --output=./recordings/session.h264
+# Record a single color stream
+./build/src/plugins/oak/camera_plugin_oak --add-stream=camera=Color,output=./color.h264
 
-# Custom path and camera settings
-./build/src/plugins/oak/camera_plugin_oak --output=/tmp/session.h264 --width=1920 --height=1080 --fps=30 --bitrate=15000000
+# Record multiple streams
+./build/src/plugins/oak/camera_plugin_oak \
+  --add-stream=camera=Color,output=./color.h264 \
+  --add-stream=camera=MonoLeft,output=./left.h264 \
+  --add-stream=camera=MonoRight,output=./right.h264
+
+# Record with a live preview window
+./build/src/plugins/oak/camera_plugin_oak \
+  --add-stream=camera=Color,output=./color.h264 --preview
+
+# Record metadata to MCAP
+./build/src/plugins/oak/camera_plugin_oak \
+  --add-stream=camera=Color,output=./color.h264 \
+  --mcap-filename=./metadata.mcap
 
 # Show help
 ./build/src/plugins/oak/camera_plugin_oak --help
@@ -45,13 +57,14 @@ Press `Ctrl+C` to stop recording.
 
 | Option | Default | Description |
 |--------|---------|-------------|
-| `--width` | 1920 | Frame width |
-| `--height` | 1080 | Frame height |
-| `--fps` | 30 | Frame rate |
-| `--bitrate` | 8000000 | H.264 bitrate (bps) |
-| `--quality` | 80 | H.264 quality (1-100) |
-| `--output` | (required) | Full path for recording file |
-| `--collection-prefix` | oak_camera | Tensor collection prefix for metadata (per-stream IDs: `prefix/StreamName`) |
+| `--add-stream=camera=<name>,output=<path>` | (at least one required) | Add a capture stream. `camera` is one of `Color`, `MonoLeft`, `MonoRight`. Repeatable. |
+| `--fps=N` | 30 | Frame rate for all streams |
+| `--bitrate=N` | 8000000 | H.264 bitrate (bps) |
+| `--quality=N` | 80 | H.264 quality (1-100) |
+| `--device-id=ID` | first available | OAK device MxId |
+| `--preview` | off | Open a live SDL2 window showing the color camera feed |
+| `--collection-prefix=PREFIX` | | Push per-frame metadata via OpenXR tensor extensions |
+| `--mcap-filename=PATH` | | Record per-frame metadata to an MCAP file |
 
 ## Architecture
 
@@ -70,15 +83,10 @@ Press `Ctrl+C` to stop recording.
 
 ## Dependencies
 
-**System dependencies** (install before building):
-
-```bash
-sudo apt install libusb-1.0-0-dev
-```
-
-**Automatically built via CMake:**
+All dependencies are built automatically via CMake:
 
 - **DepthAI** - OAK camera interface
+- **SDL2** - Live preview window (used by `--preview`)
 
 ## Output Format
 

src/plugins/oak/core/oak_camera.cpp

@@ -4,6 +4,7 @@
 #include "oak_camera.hpp"
 
 #include "frame_sink.hpp"
+#include "preview_stream.hpp"
 
 #include <algorithm>
 #include <iostream>
@@ -42,18 +43,32 @@ OakCamera::OakCamera(const OakConfig& config, const std::vector<StreamConfig>& s
     for (const auto& [socket, name] : sensors)
         std::cout << "  Socket " << static_cast<int>(socket) << ": " << name << std::endl;
 
-    create_pipeline(config, streams, sensors);
+    if (sensors.find(dai::CameraBoardSocket::CAM_A) == sensors.end())
+        throw std::runtime_error("Color sensor not found on CAM_A");
+    auto color_sensor_name = sensors.find(dai::CameraBoardSocket::CAM_A)->second;
+    auto color_resolution = color_sensor_name == "OV9782" ? dai::ColorCameraProperties::SensorResolution::THE_800_P :
+                                                            dai::ColorCameraProperties::SensorResolution::THE_1080_P;
 
-    m_device->startPipeline(*m_pipeline);
+    static constexpr const char* kPreviewStreamName = "ColorPreview";
+
+    auto pipeline = create_pipeline(config, streams, color_resolution);
+
+    if (config.preview)
+        m_preview = PreviewStream::create(kPreviewStreamName, pipeline, color_resolution);
+
+    m_device->startPipeline(pipeline);
 
     for (const auto& s : streams)
-    {
         m_queues[s.camera] = m_device->getOutputQueue(core::EnumNameStreamType(s.camera), 8, false);
-    }
+
+    if (m_preview)
+        m_preview->setOutputQueue(m_device->getOutputQueue(kPreviewStreamName, 4, false));
 
     std::cout << "OAK camera pipeline started" << std::endl;
 }
 
+OakCamera::~OakCamera() = default;
+
 dai::DeviceInfo OakCamera::find_device(const std::string& device_id)
 {
     auto devices = dai::Device::getAllAvailableDevices();
@@ -83,27 +98,27 @@ dai::DeviceInfo OakCamera::find_device(const std::string& device_id)
 // Pipeline construction
 // =============================================================================
 
-void OakCamera::create_pipeline(const OakConfig& config,
-                                const std::vector<StreamConfig>& streams,
-                                const std::unordered_map<dai::CameraBoardSocket, std::string>& sensors)
+dai::Pipeline OakCamera::create_pipeline(const OakConfig& config,
+                                         const std::vector<StreamConfig>& streams,
+                                         dai::ColorCameraProperties::SensorResolution color_resolution)
 {
-    m_pipeline = std::make_shared<dai::Pipeline>();
+    dai::Pipeline pipeline;
 
     bool need_color = has_stream(streams, core::StreamType_Color);
     bool need_mono_left = has_stream(streams, core::StreamType_MonoLeft);
     bool need_mono_right = has_stream(streams, core::StreamType_MonoRight);
 
     auto create_h264_output = [&](dai::Node::Output& source, const char* stream_name)
     {
-        auto enc = m_pipeline->create<dai::node::VideoEncoder>();
+        auto enc = pipeline.create<dai::node::VideoEncoder>();
         enc->setDefaultProfilePreset(config.fps, dai::VideoEncoderProperties::Profile::H264_BASELINE);
         enc->setBitrate(config.bitrate);
         enc->setQuality(config.quality);
         enc->setKeyframeFrequency(config.keyframe_frequency);
         enc->setNumBFrames(0);
         enc->setRateControlMode(dai::VideoEncoderProperties::RateControlMode::CBR);
 
-        auto xout = m_pipeline->create<dai::node::XLinkOut>();
+        auto xout = pipeline.create<dai::node::XLinkOut>();
         xout->setStreamName(stream_name);
 
         source.link(enc->input);
@@ -113,16 +128,9 @@ void OakCamera::create_pipeline(const OakConfig& config,
     // ---- Color camera ----
     if (need_color)
     {
-        auto it = sensors.find(dai::CameraBoardSocket::CAM_A);
-        if (it == sensors.end())
-            throw std::runtime_error("Color stream requested but no sensor found on CAM_A");
-
-        auto resolution = it->second == "OV9782" ? dai::ColorCameraProperties::SensorResolution::THE_800_P :
-                                                   dai::ColorCameraProperties::SensorResolution::THE_1080_P;
-
-        auto camRgb = m_pipeline->create<dai::node::ColorCamera>();
+        auto camRgb = pipeline.create<dai::node::ColorCamera>();
         camRgb->setBoardSocket(dai::CameraBoardSocket::CAM_A);
-        camRgb->setResolution(resolution);
+        camRgb->setResolution(color_resolution);
         camRgb->setFps(config.fps);
         camRgb->setColorOrder(dai::ColorCameraProperties::ColorOrder::BGR);
 
@@ -132,7 +140,7 @@ void OakCamera::create_pipeline(const OakConfig& config,
     // ---- Mono cameras ----
     if (need_mono_left)
     {
-        auto monoLeft = m_pipeline->create<dai::node::MonoCamera>();
+        auto monoLeft = pipeline.create<dai::node::MonoCamera>();
         monoLeft->setBoardSocket(dai::CameraBoardSocket::CAM_B);
         monoLeft->setResolution(dai::MonoCameraProperties::SensorResolution::THE_400_P);
         monoLeft->setFps(config.fps);
@@ -142,13 +150,15 @@ void OakCamera::create_pipeline(const OakConfig& config,
 
     if (need_mono_right)
     {
-        auto monoRight = m_pipeline->create<dai::node::MonoCamera>();
+        auto monoRight = pipeline.create<dai::node::MonoCamera>();
         monoRight->setBoardSocket(dai::CameraBoardSocket::CAM_C);
         monoRight->setResolution(dai::MonoCameraProperties::SensorResolution::THE_400_P);
         monoRight->setFps(config.fps);
 
         create_h264_output(monoRight->out, core::EnumNameStreamType(core::StreamType_MonoRight));
     }
+
+    return pipeline;
 }
 
 // =============================================================================
@@ -183,6 +193,9 @@ void OakCamera::update()
         m_sink->on_frame(frame);
         ++m_frame_counts[type];
     }
+
+    if (m_preview)
+        m_preview->update();
 }
 
 // =============================================================================

src/plugins/oak/core/oak_camera.hpp

@@ -17,8 +17,9 @@ namespace plugins
 namespace oak
 {
 
-// Forward declaration -- FrameSink is defined in frame_sink.hpp.
+// Forward declarations
 class FrameSink;
+class PreviewStream;
 
 // ============================================================================
 // Stream configuration
@@ -41,6 +42,7 @@ struct OakConfig
     int bitrate = 8'000'000;
     int quality = 80;
     int keyframe_frequency = 30;
+    bool preview = false;
 };
 
 struct OakFrame
@@ -75,6 +77,7 @@ class OakCamera
 {
 public:
     OakCamera(const OakConfig& config, const std::vector<StreamConfig>& streams, std::unique_ptr<FrameSink> sink);
+    ~OakCamera();
 
     OakCamera(const OakCamera&) = delete;
     OakCamera& operator=(const OakCamera&) = delete;
@@ -89,15 +92,17 @@ class OakCamera
 
 private:
     dai::DeviceInfo find_device(const std::string& device_id);
-    void create_pipeline(const OakConfig& config,
-                         const std::vector<StreamConfig>& streams,
-                         const std::unordered_map<dai::CameraBoardSocket, std::string>& sensors);
+    dai::Pipeline create_pipeline(const OakConfig& config,
+                                  const std::vector<StreamConfig>& streams,
+                                  dai::ColorCameraProperties::SensorResolution color_resolution);
 
-    std::shared_ptr<dai::Pipeline> m_pipeline;
     std::shared_ptr<dai::Device> m_device;
     std::map<core::StreamType, std::shared_ptr<dai::DataOutputQueue>> m_queues;
+
     std::unique_ptr<FrameSink> m_sink;
     std::map<core::StreamType, uint64_t> m_frame_counts;
+
+    std::unique_ptr<PreviewStream> m_preview;
 };
 
 } // namespace oak