diff --git a/examples/oxr/python/test_oak_camera.py b/examples/oxr/python/test_oak_camera.py index a48129950..6647e4092 100755 --- a/examples/oxr/python/test_oak_camera.py +++ b/examples/oxr/python/test_oak_camera.py @@ -185,6 +185,7 @@ def run_test(duration: float = 10.0, mode: str = MODE_NO_METADATA): extra_args = [ f"--add-stream=camera=Color,output={color_path}", f"--add-stream=camera=MonoLeft,output={mono_left_path}", + "--preview", ] # For plugin-mcap, resolve to absolute path so the plugin can find it diff --git a/src/plugins/oak/CMakeLists.txt b/src/plugins/oak/CMakeLists.txt index cc84e45a8..e98e4af8d 100644 --- a/src/plugins/oak/CMakeLists.txt +++ b/src/plugins/oak/CMakeLists.txt @@ -18,6 +18,7 @@ message(STATUS "Configuring DepthAI...") set(DEPTHAI_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) set(DEPTHAI_BUILD_TESTS OFF CACHE BOOL "" FORCE) set(DEPTHAI_BUILD_DOCS OFF CACHE BOOL "" FORCE) +set(DEPTHAI_OPENCV_SUPPORT OFF CACHE BOOL "" FORCE) FetchContent_Declare( depthai @@ -28,6 +29,23 @@ FetchContent_MakeAvailable(depthai) message(STATUS "Building OAK camera plugin with DepthAI ${depthai_VERSION}") +# ============================================================================== +# SDL2 (for live preview window, bundled via FetchContent) +# ============================================================================== +set(SDL2_VERSION "2.32.10") + +set(SDL_TEST OFF CACHE BOOL "" FORCE) +set(SDL_SHARED OFF CACHE BOOL "" FORCE) +set(SDL_STATIC ON CACHE BOOL "" FORCE) + +FetchContent_Declare( + sdl2 + URL "https://github.com/libsdl-org/SDL/releases/download/release-${SDL2_VERSION}/SDL2-${SDL2_VERSION}.tar.gz" + URL_HASH "SHA256=5f5993c530f084535c65a6879e9b26ad441169b3e25d789d83287040a9ca5165" +) +FetchContent_MakeAvailable(sdl2) +message(STATUS "SDL2 ${SDL2_VERSION} — live preview support enabled") + # ============================================================================== # Build OAK Plugin # ============================================================================== @@ -36,6 +54,7 @@ add_executable(camera_plugin_oak core/oak_camera.cpp core/rawdata_writer.cpp core/frame_sink.cpp + core/preview_stream.cpp ) target_link_libraries(camera_plugin_oak @@ -45,9 +64,10 @@ target_link_libraries(camera_plugin_oak mcap::mcap oxr::oxr_core pusherio::pusherio + SDL2::SDL2-static ) -# Set RPATH to find bundled libraries in same directory +# Set RPATH so the binary finds bundled shared libraries (e.g. libusb) in its own directory set_target_properties(camera_plugin_oak PROPERTIES INSTALL_RPATH "$ORIGIN" ) diff --git a/src/plugins/oak/README.md b/src/plugins/oak/README.md index df14582b3..05ea41a8d 100644 --- a/src/plugins/oak/README.md +++ b/src/plugins/oak/README.md @@ -22,18 +22,30 @@ DepthAI is fetched and built automatically via FetchContent. The first build tak cd IsaacTeleop # Configure and build -cmake -B build +cmake -B build -DBUILD_PLUGIN_OAK_CAMERA=ON cmake --build build --target camera_plugin_oak --parallel ``` ## Usage ```bash -# Record to a file (--output is required) -./build/src/plugins/oak/camera_plugin_oak --output=./recordings/session.h264 +# Record a single color stream +./build/src/plugins/oak/camera_plugin_oak --add-stream=camera=Color,output=./color.h264 -# Custom path and camera settings -./build/src/plugins/oak/camera_plugin_oak --output=/tmp/session.h264 --width=1920 --height=1080 --fps=30 --bitrate=15000000 +# Record multiple streams +./build/src/plugins/oak/camera_plugin_oak \ + --add-stream=camera=Color,output=./color.h264 \ + --add-stream=camera=MonoLeft,output=./left.h264 \ + --add-stream=camera=MonoRight,output=./right.h264 + +# Record with a live preview window +./build/src/plugins/oak/camera_plugin_oak \ + --add-stream=camera=Color,output=./color.h264 --preview + +# Record metadata to MCAP +./build/src/plugins/oak/camera_plugin_oak \ + --add-stream=camera=Color,output=./color.h264 \ + --mcap-filename=./metadata.mcap # Show help ./build/src/plugins/oak/camera_plugin_oak --help @@ -45,13 +57,14 @@ Press `Ctrl+C` to stop recording. | Option | Default | Description | |--------|---------|-------------| -| `--width` | 1920 | Frame width | -| `--height` | 1080 | Frame height | -| `--fps` | 30 | Frame rate | -| `--bitrate` | 8000000 | H.264 bitrate (bps) | -| `--quality` | 80 | H.264 quality (1-100) | -| `--output` | (required) | Full path for recording file | -| `--collection-prefix` | oak_camera | Tensor collection prefix for metadata (per-stream IDs: `prefix/StreamName`) | +| `--add-stream=camera=,output=` | (at least one required) | Add a capture stream. `camera` is one of `Color`, `MonoLeft`, `MonoRight`. Repeatable. | +| `--fps=N` | 30 | Frame rate for all streams | +| `--bitrate=N` | 8000000 | H.264 bitrate (bps) | +| `--quality=N` | 80 | H.264 quality (1-100) | +| `--device-id=ID` | first available | OAK device MxId | +| `--preview` | off | Open a live SDL2 window showing the color camera feed | +| `--collection-prefix=PREFIX` | | Push per-frame metadata via OpenXR tensor extensions | +| `--mcap-filename=PATH` | | Record per-frame metadata to an MCAP file | ## Architecture @@ -70,15 +83,10 @@ Press `Ctrl+C` to stop recording. ## Dependencies -**System dependencies** (install before building): - -```bash -sudo apt install libusb-1.0-0-dev -``` - -**Automatically built via CMake:** +All dependencies are built automatically via CMake: - **DepthAI** - OAK camera interface +- **SDL2** - Live preview window (used by `--preview`) ## Output Format diff --git a/src/plugins/oak/core/oak_camera.cpp b/src/plugins/oak/core/oak_camera.cpp index c746e5118..d9dd88af7 100644 --- a/src/plugins/oak/core/oak_camera.cpp +++ b/src/plugins/oak/core/oak_camera.cpp @@ -4,6 +4,7 @@ #include "oak_camera.hpp" #include "frame_sink.hpp" +#include "preview_stream.hpp" #include #include @@ -42,18 +43,32 @@ OakCamera::OakCamera(const OakConfig& config, const std::vector& s for (const auto& [socket, name] : sensors) std::cout << " Socket " << static_cast(socket) << ": " << name << std::endl; - create_pipeline(config, streams, sensors); + if (sensors.find(dai::CameraBoardSocket::CAM_A) == sensors.end()) + throw std::runtime_error("Color sensor not found on CAM_A"); + auto color_sensor_name = sensors.find(dai::CameraBoardSocket::CAM_A)->second; + auto color_resolution = color_sensor_name == "OV9782" ? dai::ColorCameraProperties::SensorResolution::THE_800_P : + dai::ColorCameraProperties::SensorResolution::THE_1080_P; - m_device->startPipeline(*m_pipeline); + static constexpr const char* kPreviewStreamName = "ColorPreview"; + + auto pipeline = create_pipeline(config, streams, color_resolution); + + if (config.preview) + m_preview = PreviewStream::create(kPreviewStreamName, pipeline, color_resolution); + + m_device->startPipeline(pipeline); for (const auto& s : streams) - { m_queues[s.camera] = m_device->getOutputQueue(core::EnumNameStreamType(s.camera), 8, false); - } + + if (m_preview) + m_preview->setOutputQueue(m_device->getOutputQueue(kPreviewStreamName, 4, false)); std::cout << "OAK camera pipeline started" << std::endl; } +OakCamera::~OakCamera() = default; + dai::DeviceInfo OakCamera::find_device(const std::string& device_id) { auto devices = dai::Device::getAllAvailableDevices(); @@ -83,11 +98,11 @@ dai::DeviceInfo OakCamera::find_device(const std::string& device_id) // Pipeline construction // ============================================================================= -void OakCamera::create_pipeline(const OakConfig& config, - const std::vector& streams, - const std::unordered_map& sensors) +dai::Pipeline OakCamera::create_pipeline(const OakConfig& config, + const std::vector& streams, + dai::ColorCameraProperties::SensorResolution color_resolution) { - m_pipeline = std::make_shared(); + dai::Pipeline pipeline; bool need_color = has_stream(streams, core::StreamType_Color); bool need_mono_left = has_stream(streams, core::StreamType_MonoLeft); @@ -95,7 +110,7 @@ void OakCamera::create_pipeline(const OakConfig& config, auto create_h264_output = [&](dai::Node::Output& source, const char* stream_name) { - auto enc = m_pipeline->create(); + auto enc = pipeline.create(); enc->setDefaultProfilePreset(config.fps, dai::VideoEncoderProperties::Profile::H264_BASELINE); enc->setBitrate(config.bitrate); enc->setQuality(config.quality); @@ -103,7 +118,7 @@ void OakCamera::create_pipeline(const OakConfig& config, enc->setNumBFrames(0); enc->setRateControlMode(dai::VideoEncoderProperties::RateControlMode::CBR); - auto xout = m_pipeline->create(); + auto xout = pipeline.create(); xout->setStreamName(stream_name); source.link(enc->input); @@ -113,16 +128,9 @@ void OakCamera::create_pipeline(const OakConfig& config, // ---- Color camera ---- if (need_color) { - auto it = sensors.find(dai::CameraBoardSocket::CAM_A); - if (it == sensors.end()) - throw std::runtime_error("Color stream requested but no sensor found on CAM_A"); - - auto resolution = it->second == "OV9782" ? dai::ColorCameraProperties::SensorResolution::THE_800_P : - dai::ColorCameraProperties::SensorResolution::THE_1080_P; - - auto camRgb = m_pipeline->create(); + auto camRgb = pipeline.create(); camRgb->setBoardSocket(dai::CameraBoardSocket::CAM_A); - camRgb->setResolution(resolution); + camRgb->setResolution(color_resolution); camRgb->setFps(config.fps); camRgb->setColorOrder(dai::ColorCameraProperties::ColorOrder::BGR); @@ -132,7 +140,7 @@ void OakCamera::create_pipeline(const OakConfig& config, // ---- Mono cameras ---- if (need_mono_left) { - auto monoLeft = m_pipeline->create(); + auto monoLeft = pipeline.create(); monoLeft->setBoardSocket(dai::CameraBoardSocket::CAM_B); monoLeft->setResolution(dai::MonoCameraProperties::SensorResolution::THE_400_P); monoLeft->setFps(config.fps); @@ -142,13 +150,15 @@ void OakCamera::create_pipeline(const OakConfig& config, if (need_mono_right) { - auto monoRight = m_pipeline->create(); + auto monoRight = pipeline.create(); monoRight->setBoardSocket(dai::CameraBoardSocket::CAM_C); monoRight->setResolution(dai::MonoCameraProperties::SensorResolution::THE_400_P); monoRight->setFps(config.fps); create_h264_output(monoRight->out, core::EnumNameStreamType(core::StreamType_MonoRight)); } + + return pipeline; } // ============================================================================= @@ -183,6 +193,9 @@ void OakCamera::update() m_sink->on_frame(frame); ++m_frame_counts[type]; } + + if (m_preview) + m_preview->update(); } // ============================================================================= diff --git a/src/plugins/oak/core/oak_camera.hpp b/src/plugins/oak/core/oak_camera.hpp index f2ab34c60..01f7f1aac 100644 --- a/src/plugins/oak/core/oak_camera.hpp +++ b/src/plugins/oak/core/oak_camera.hpp @@ -17,8 +17,9 @@ namespace plugins namespace oak { -// Forward declaration -- FrameSink is defined in frame_sink.hpp. +// Forward declarations class FrameSink; +class PreviewStream; // ============================================================================ // Stream configuration @@ -41,6 +42,7 @@ struct OakConfig int bitrate = 8'000'000; int quality = 80; int keyframe_frequency = 30; + bool preview = false; }; struct OakFrame @@ -75,6 +77,7 @@ class OakCamera { public: OakCamera(const OakConfig& config, const std::vector& streams, std::unique_ptr sink); + ~OakCamera(); OakCamera(const OakCamera&) = delete; OakCamera& operator=(const OakCamera&) = delete; @@ -89,15 +92,17 @@ class OakCamera private: dai::DeviceInfo find_device(const std::string& device_id); - void create_pipeline(const OakConfig& config, - const std::vector& streams, - const std::unordered_map& sensors); + dai::Pipeline create_pipeline(const OakConfig& config, + const std::vector& streams, + dai::ColorCameraProperties::SensorResolution color_resolution); - std::shared_ptr m_pipeline; std::shared_ptr m_device; std::map> m_queues; + std::unique_ptr m_sink; std::map m_frame_counts; + + std::unique_ptr m_preview; }; } // namespace oak diff --git a/src/plugins/oak/core/preview_stream.cpp b/src/plugins/oak/core/preview_stream.cpp new file mode 100644 index 000000000..50255ceff --- /dev/null +++ b/src/plugins/oak/core/preview_stream.cpp @@ -0,0 +1,142 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "preview_stream.hpp" + +#include +#include +#include + +namespace plugins +{ +namespace oak +{ + +struct PreviewStream::Impl +{ + SDL_Window* window = nullptr; + SDL_Renderer* renderer = nullptr; + SDL_Texture* texture = nullptr; + int tex_width = 0; + int tex_height = 0; + + std::shared_ptr queue; + + ~Impl() + { + if (texture) + SDL_DestroyTexture(texture); + if (renderer) + SDL_DestroyRenderer(renderer); + if (window) + SDL_DestroyWindow(window); + SDL_QuitSubSystem(SDL_INIT_VIDEO); + } +}; + +PreviewStream::~PreviewStream() = default; + +std::unique_ptr PreviewStream::create(const std::string& name, + dai::Pipeline& pipeline, + dai::ColorCameraProperties::SensorResolution resolution) +{ + // Find existing ColorCamera on CAM_A, or create one + std::shared_ptr camRgb; + for (auto& node : pipeline.getAllNodes()) + { + auto cam = std::dynamic_pointer_cast(node); + if (cam && cam->getBoardSocket() == dai::CameraBoardSocket::CAM_A) + { + camRgb = cam; + break; + } + } + + if (!camRgb) + { + std::cout << "Creating new ColorCamera on CAM_A" << std::endl; + camRgb = pipeline.create(); + camRgb->setBoardSocket(dai::CameraBoardSocket::CAM_A); + camRgb->setResolution(resolution); + camRgb->setColorOrder(dai::ColorCameraProperties::ColorOrder::BGR); + } + + int preview_w = 640; + int preview_h = (resolution == dai::ColorCameraProperties::SensorResolution::THE_800_P) ? 400 : 360; + + camRgb->setPreviewSize(preview_w, preview_h); + camRgb->setInterleaved(true); + + auto xout = pipeline.create(); + xout->setStreamName(name); + camRgb->preview.link(xout->input); + + if (SDL_Init(SDL_INIT_VIDEO) != 0) + throw std::runtime_error(std::string("Preview: SDL_Init failed: ") + SDL_GetError()); + + auto impl = std::make_unique(); + + impl->window = SDL_CreateWindow( + name.c_str(), SDL_WINDOWPOS_CENTERED, SDL_WINDOWPOS_CENTERED, preview_w, preview_h, SDL_WINDOW_SHOWN); + + if (!impl->window) + throw std::runtime_error(std::string("Preview: SDL_CreateWindow failed: ") + SDL_GetError()); + + impl->renderer = SDL_CreateRenderer(impl->window, -1, SDL_RENDERER_ACCELERATED | SDL_RENDERER_PRESENTVSYNC); + if (!impl->renderer) + impl->renderer = SDL_CreateRenderer(impl->window, -1, 0); + + if (!impl->renderer) + throw std::runtime_error(std::string("Preview: SDL_CreateRenderer failed: ") + SDL_GetError()); + + auto stream = std::unique_ptr(new PreviewStream()); + stream->m_impl = std::move(impl); + + std::cout << "Color preview enabled (" << preview_w << "x" << preview_h << ")" << std::endl; + return stream; +} + +void PreviewStream::setOutputQueue(std::shared_ptr queue) +{ + m_impl->queue = std::move(queue); +} + +void PreviewStream::update() +{ + if (!m_impl->queue) + throw std::runtime_error("Preview: Output queue not set"); + + auto frame = m_impl->queue->tryGet(); + if (!frame) + return; + + const auto* data = frame->getData().data(); + int width = frame->getWidth(); + int height = frame->getHeight(); + + if (width != m_impl->tex_width || height != m_impl->tex_height) + { + if (m_impl->texture) + SDL_DestroyTexture(m_impl->texture); + + m_impl->texture = + SDL_CreateTexture(m_impl->renderer, SDL_PIXELFORMAT_BGR24, SDL_TEXTUREACCESS_STREAMING, width, height); + + if (!m_impl->texture) + { + std::cerr << "Preview: SDL_CreateTexture failed: " << SDL_GetError() << std::endl; + return; + } + + m_impl->tex_width = width; + m_impl->tex_height = height; + } + + SDL_UpdateTexture(m_impl->texture, nullptr, data, width * 3); + SDL_RenderClear(m_impl->renderer); + SDL_RenderCopy(m_impl->renderer, m_impl->texture, nullptr, nullptr); + SDL_RenderPresent(m_impl->renderer); +} + +} // namespace oak +} // namespace plugins diff --git a/src/plugins/oak/core/preview_stream.hpp b/src/plugins/oak/core/preview_stream.hpp new file mode 100644 index 000000000..f40c3fa4e --- /dev/null +++ b/src/plugins/oak/core/preview_stream.hpp @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include +#include + +namespace plugins +{ +namespace oak +{ + +/** + * @brief Self-contained color preview stream. + * + * Owns the full lifecycle: adds preview nodes to the pipeline, opens an SDL2 + * window, connects to the device output queue, and polls/displays frames. + */ +class PreviewStream +{ +public: + ~PreviewStream(); + + PreviewStream(const PreviewStream&) = delete; + PreviewStream& operator=(const PreviewStream&) = delete; + + /** + * @brief Wire preview nodes into an existing pipeline and create the window. + * + * Searches the pipeline for an existing ColorCamera on CAM_A. If none is + * found, creates and configures one. Then attaches preview output nodes. + * + * @throws std::runtime_error if SDL initialisation or window creation fails. + */ + static std::unique_ptr create(const std::string& name, + dai::Pipeline& pipeline, + dai::ColorCameraProperties::SensorResolution resolution); + + /** @brief Set the output queue to poll frames from. Call after Device::startPipeline. */ + void setOutputQueue(std::shared_ptr queue); + + /** @brief Poll the queue and display a frame if available. */ + void update(); + +private: + PreviewStream() = default; + + struct Impl; + std::unique_ptr m_impl; +}; + +} // namespace oak +} // namespace plugins diff --git a/src/plugins/oak/main.cpp b/src/plugins/oak/main.cpp index 9e7ac3433..62510d8fb 100644 --- a/src/plugins/oak/main.cpp +++ b/src/plugins/oak/main.cpp @@ -105,12 +105,14 @@ void print_usage(const char* program_name) << "\nMetadata (mutually exclusive):\n" << " --collection-prefix=PREFIX Push metadata via OpenXR tensor extensions\n" << " --mcap-filename=PATH Record metadata to an MCAP file\n" + << "\nPreview:\n" + << " --preview Show live color camera preview via SDL2 window\n" << "\nGeneral:\n" << " --help Show this help message\n" << "\nExamples:\n" << " " << program_name << " --add-stream camera=Color,output=./color.h264\n" << " " << program_name - << " --add-stream=camera=Color,output=./color.h264 --add-stream=camera=LeftMono,output=./left.h264 --add-stream=camera=RightMono,output=./right.h264\n"; + << " --add-stream=camera=Color,output=./color.h264 --add-stream=camera=MonoLeft,output=./left.h264 --add-stream=camera=MonoRight,output=./right.h264\n"; } // ============================================================================= @@ -155,6 +157,10 @@ try { camera_config.device_id = arg.substr(12); } + else if (arg == "--preview") + { + camera_config.preview = true; + } else if (arg.find("--collection-prefix=") == 0) { collection_prefix = arg.substr(20);