From 68d654cef78951c4849ecd6fa1a94caf9e029aea Mon Sep 17 00:00:00 2001 From: Raphael Dumusc Date: Thu, 19 Jan 2017 10:36:02 +0100 Subject: [PATCH 1/7] Added stereo 3D streaming support --- CMakeLists.txt | 4 +- apps/SimpleStreamer/main.cpp | 215 ++++++++++++++++++------------- deflect/CMakeLists.txt | 8 +- deflect/Frame.h | 5 +- deflect/FrameDispatcher.cpp | 97 +++++++++++--- deflect/FrameDispatcher.h | 26 ++-- deflect/ImageWrapper.h | 16 ++- deflect/MessageHeader.h | 3 +- deflect/MetaTypeRegistration.cpp | 1 + deflect/ReceiveBuffer.cpp | 99 ++++++++++---- deflect/ReceiveBuffer.h | 70 ++++------ deflect/SegmentParameters.h | 23 ++-- deflect/ServerWorker.cpp | 13 +- deflect/ServerWorker.h | 9 +- deflect/SourceBuffer.cpp | 133 +++++++++++++++++++ deflect/SourceBuffer.h | 93 +++++++++++++ deflect/StreamPrivate.cpp | 10 ++ deflect/StreamPrivate.h | 3 + deflect/types.h | 3 + doc/Changelog.md | 5 + 20 files changed, 616 insertions(+), 220 deletions(-) create mode 100644 deflect/SourceBuffer.cpp create mode 100644 deflect/SourceBuffer.h diff --git a/CMakeLists.txt b/CMakeLists.txt index ba91e70..81d4286 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,8 +4,8 @@ # Daniel Nachbaur cmake_minimum_required(VERSION 3.1 FATAL_ERROR) -project(Deflect VERSION 0.12.1) -set(Deflect_VERSION_ABI 5) +project(Deflect VERSION 0.13.0) +set(Deflect_VERSION_ABI 6) list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/CMake ${CMAKE_SOURCE_DIR}/CMake/common) diff --git a/apps/SimpleStreamer/main.cpp b/apps/SimpleStreamer/main.cpp index ee32627..abc2674 100644 --- a/apps/SimpleStreamer/main.cpp +++ b/apps/SimpleStreamer/main.cpp @@ -1,5 +1,5 @@ /*********************************************************************/ -/* Copyright (c) 2014-2015, EPFL/Blue Brain Project */ +/* Copyright (c) 2014-2017, EPFL/Blue Brain Project */ /* Raphael Dumusc */ /* */ /* Copyright (c) 2011 - 2012, The University of Texas at Austin. */ @@ -57,33 +57,30 @@ bool deflectInteraction = false; bool deflectCompressImage = true; +bool deflectStereoStream = false; unsigned int deflectCompressionQuality = 75; -char* deflectHost = NULL; +std::string deflectHost; std::string deflectStreamId = "SimpleStreamer"; -deflect::Stream* deflectStream = NULL; +std::unique_ptr deflectStream; -void syntax( char* app ); +void syntax( int exitStatus ); void readCommandLineArguments( int argc, char** argv ); void initGLWindow( int argc, char** argv ); void initDeflectStream(); void display(); void reshape( int width, int height ); - void cleanup() { - delete deflectStream; + deflectStream.reset(); } int main( int argc, char** argv ) { readCommandLineArguments( argc, argv ); - if( deflectHost == NULL ) - { - syntax( argv[0] ); - return EXIT_FAILURE; - } + if( deflectHost.empty( )) + syntax( EXIT_FAILURE ); initGLWindow( argc, argv ); initDeflectStream(); @@ -94,16 +91,27 @@ int main( int argc, char** argv ) return EXIT_SUCCESS; } +void syntax( const int exitStatus ) +{ + std::cout << "Usage: simplestreamer [options] " << std::endl; + std::cout << "Stream a GLUT teapot to a remote host\n" << std::endl; + std::cout << "Options:" << std::endl; + std::cout << " -h, --help display this help" << std::endl; + std::cout << " -n set stream identifier (default: 'SimpleStreamer')" << std::endl; + std::cout << " -i enable interaction events (default: OFF)" << std::endl; + std::cout << " -u enable uncompressed streaming (default: OFF)" << std::endl; + std::cout << " -s enable stereo streaming (default: OFF)" << std::endl; + exit( exitStatus ); +} + void readCommandLineArguments( int argc, char** argv ) { for( int i = 1; i < argc; ++i ) { if( std::string( argv[i] ) == "--help" ) - { - syntax( argv[0] ); - ::exit( EXIT_SUCCESS ); - } - else if( argv[i][0] == '-' ) + syntax( EXIT_SUCCESS ); + + if( argv[i][0] == '-' ) { switch( argv[i][1] ) { @@ -120,8 +128,13 @@ void readCommandLineArguments( int argc, char** argv ) case 'u': deflectCompressImage = false; break; + case 's': + deflectStereoStream = true; + break; + case 'h': + syntax( EXIT_SUCCESS ); default: - syntax( argv[0] ); + std::cerr << "Unknown command line option: " << argv[i] << std::endl; ::exit( EXIT_FAILURE ); } } @@ -147,92 +160,120 @@ void initGLWindow( int argc, char** argv ) // the reshape function will be called on window resize glutReshapeFunc( reshape ); - glClearColor( 0.5, 0.5, 0.5, 1.0 ); - glEnable( GL_DEPTH_TEST ); glEnable( GL_LIGHTING ); glEnable( GL_LIGHT0 ); } - void initDeflectStream() { - deflectStream = new deflect::Stream( deflectStreamId, deflectHost ); + deflectStream.reset( new deflect::Stream( deflectStreamId, deflectHost )); if( !deflectStream->isConnected( )) { std::cerr << "Could not connect to host!" << std::endl; - delete deflectStream; - exit( 1 ); + deflectStream.reset(); + exit( EXIT_FAILURE ); } if( deflectInteraction && !deflectStream->registerForEvents( )) { std::cerr << "Could not register for events!" << std::endl; - delete deflectStream; - exit( 1 ); + deflectStream.reset(); + exit( EXIT_FAILURE ); } } - -void syntax( char* app ) -{ - std::cout << "Usage: " << app << " [options] " << std::endl; - std::cout << "Stream a GLUT teapot to a remote host\n" << std::endl; - std::cout << "Options:" << std::endl; - std::cout << " -n set stream identifier (default: 'SimpleStreamer')" << std::endl; - std::cout << " -i enable interaction events (default: OFF)" << std::endl; - std::cout << " -u enable uncompressed streaming (default: OFF)" << std::endl; -} - -void display() +struct Camera { // angles of camera rotation and zoom factor - static float angleX = 0.f; - static float angleY = 0.f; - static float offsetX = 0.f; - static float offsetY = 0.f; - static float zoom = 1.f; + float angleX = 0.f; + float angleY = 0.f; + float offsetX = 0.f; + float offsetY = 0.f; + float zoom = 1.f; - // Render the teapot - glClear( GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT ); - glMatrixMode( GL_PROJECTION ); - glLoadIdentity(); + void apply() + { + glMatrixMode( GL_PROJECTION ); + glLoadIdentity(); - const float size = 2.f; - glOrtho( -size, size, -size, size, -size, size ); + const float size = 2.f; + glOrtho( -size, size, -size, size, -size, size ); - glMatrixMode( GL_MODELVIEW ); - glLoadIdentity(); + glMatrixMode( GL_MODELVIEW ); + glLoadIdentity(); - glTranslatef( offsetX, -offsetY, 0.f ); + glTranslatef( offsetX, -offsetY, 0.f ); - glRotatef( angleX, 0.f, 1.f, 0.f ); - glRotatef( angleY, -1.f, 0.f, 0.f ); + glRotatef( angleX, 0.f, 1.f, 0.f ); + glRotatef( angleY, -1.f, 0.f, 0.f ); - glScalef( zoom, zoom, zoom ); - glutSolidTeapot( 1.f ); + glScalef( zoom, zoom, zoom ); + } +}; - // Grab the frame from OpenGL - const int windowWidth = glutGet( GLUT_WINDOW_WIDTH ); - const int windowHeight = glutGet( GLUT_WINDOW_HEIGHT ); +struct Image +{ + size_t width = 0; + size_t height = 0; + std::vector data; - const size_t imageSize = windowWidth * windowHeight * 4; - unsigned char* imageData = new unsigned char[imageSize]; - glReadPixels( 0, 0, windowWidth, windowHeight, GL_RGBA, GL_UNSIGNED_BYTE, - (GLvoid*)imageData ); + static Image readGlBuffer() + { + Image image; + image.width = glutGet( GLUT_WINDOW_WIDTH ); + image.height = glutGet( GLUT_WINDOW_HEIGHT ); + image.data.resize( image.width * image.height * 4 ); + glReadPixels( 0, 0, image.width, image.height, GL_RGBA, + GL_UNSIGNED_BYTE, (GLvoid*)image.data.data( )); + + deflect::ImageWrapper::swapYAxis( image.data.data(), image.width, + image.height, 4 ); + return image; + } +}; - // Send the frame through the stream - deflect::ImageWrapper deflectImage( (const void*)imageData, windowWidth, - windowHeight, deflect::RGBA ); +bool send( const Image& image, const deflect::View view ) +{ + deflect::ImageWrapper deflectImage( image.data.data(), image.width, + image.height, deflect::RGBA ); deflectImage.compressionPolicy = deflectCompressImage ? deflect::COMPRESSION_ON : deflect::COMPRESSION_OFF; deflectImage.compressionQuality = deflectCompressionQuality; - deflect::ImageWrapper::swapYAxis( (void*)imageData, windowWidth, - windowHeight, 4 ); - const bool success = deflectStream->send( deflectImage ); - deflectStream->finishFrame(); + deflectImage.view = view; + return deflectStream->send( deflectImage ) && + deflectStream->finishFrame(); +} + +void display() +{ + static Camera camera; + camera.apply(); + + bool success = false; + if( deflectStereoStream ) + { + glClearColor( 0.7, 0.3, 0.3, 1.0 ); + glClear( GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT ); + glutSolidTeapot( 1.f ); + const auto leftImage = Image::readGlBuffer(); + + glClearColor( 0.3, 0.7, 0.3, 1.0 ); + glClear( GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT ); + glutSolidTeapot( 1.f ); + const auto rightImage = Image::readGlBuffer(); + + success = send( leftImage, deflect::View::LEFT_EYE ) && + send( rightImage, deflect::View::RIGHT_EYE ); + } + else + { + glClearColor( 0.5, 0.5, 0.5, 1.0 ); + glClear( GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT ); + glutSolidTeapot( 1.f ); + success = send( Image::readGlBuffer(), deflect::View::MONO ); + } - delete [] imageData; glutSwapBuffers(); // increment rotation angle according to interaction, or by a constant rate @@ -254,11 +295,11 @@ void display() { case deflect::Event::EVT_CLOSE: std::cout << "Received close..." << std::endl; - exit( 0 ); + exit( EXIT_SUCCESS ); case deflect::Event::EVT_PINCH: - zoom += std::copysign( std::sqrt( event.dx * event.dx + - event.dy * event.dy ), - event.dx + event.dy ); + camera.zoom += std::copysign( std::sqrt( event.dx * event.dx + + event.dy * event.dy ), + event.dx + event.dy ); break; case deflect::Event::EVT_PRESS: mouseX = event.mouseX; @@ -268,26 +309,26 @@ void display() case deflect::Event::EVT_RELEASE: if( event.mouseLeft ) { - angleX += (event.mouseX - mouseX) * 360.f; - angleY += (event.mouseY - mouseY) * 360.f; + camera.angleX += (event.mouseX - mouseX) * 360.f; + camera.angleY += (event.mouseY - mouseY) * 360.f; } mouseX = event.mouseX; mouseY = event.mouseY; break; case deflect::Event::EVT_PAN: - offsetX += event.dx; - offsetY += event.dy; + camera.offsetX += event.dx; + camera.offsetY += event.dy; mouseX = event.mouseX; mouseY = event.mouseY; break; case deflect::Event::EVT_KEY_PRESS: if( event.key == ' ' ) { - angleX = 0.f; - angleY = 0.f; - offsetX = 0.f; - offsetY = 0.f; - zoom = 1.f; + camera.angleX = 0.f; + camera.angleY = 0.f; + camera.offsetX = 0.f; + camera.offsetY = 0.f; + camera.zoom = 1.f; } break; default: @@ -297,8 +338,8 @@ void display() } else { - angleX += 1.f; - angleY += 1.f; + camera.angleX += 1.f; + camera.angleY += 1.f; } if( !success ) @@ -306,12 +347,12 @@ void display() if( !deflectStream->isConnected( )) { std::cout << "Stream closed, exiting." << std::endl; - exit( 0 ); + exit( EXIT_SUCCESS ); } else { std::cerr << "failure in deflectStreamSend()" << std::endl; - exit( 1 ); + exit( EXIT_FAILURE ); } } } diff --git a/deflect/CMakeLists.txt b/deflect/CMakeLists.txt index b8661be..c039a11 100644 --- a/deflect/CMakeLists.txt +++ b/deflect/CMakeLists.txt @@ -1,7 +1,7 @@ -# Copyright (c) 2013-2016, EPFL/Blue Brain Project -# Raphael Dumusc -# Daniel Nachbaur +# Copyright (c) 2013-2017, EPFL/Blue Brain Project +# Raphael Dumusc +# Daniel Nachbaur set(DEFLECT_PUBLIC_HEADERS config.h @@ -26,6 +26,7 @@ set(DEFLECT_HEADERS ReceiveBuffer.h ServerWorker.h Socket.h + SourceBuffer.h StreamPrivate.h ) @@ -41,6 +42,7 @@ set(DEFLECT_SOURCES Server.cpp ServerWorker.cpp Socket.cpp + SourceBuffer.cpp Stream.cpp StreamPrivate.cpp StreamSendWorker.cpp diff --git a/deflect/Frame.h b/deflect/Frame.h index f7fb3e9..628c9c8 100644 --- a/deflect/Frame.h +++ b/deflect/Frame.h @@ -1,5 +1,5 @@ /*********************************************************************/ -/* Copyright (c) 2014-2016, EPFL/Blue Brain Project */ +/* Copyright (c) 2014-2017, EPFL/Blue Brain Project */ /* Raphael Dumusc */ /* All rights reserved. */ /* */ @@ -62,6 +62,9 @@ class Frame /** The PixelStream uri to which this frame is associated. */ QString uri; + /** The view to which this frame belongs. */ + View view = View::MONO; + /** Get the total dimensions of this frame. */ DEFLECT_API QSize computeDimensions() const; }; diff --git a/deflect/FrameDispatcher.cpp b/deflect/FrameDispatcher.cpp index 1cc85d3..8cd878d 100644 --- a/deflect/FrameDispatcher.cpp +++ b/deflect/FrameDispatcher.cpp @@ -1,5 +1,5 @@ /*********************************************************************/ -/* Copyright (c) 2013-2015, EPFL/Blue Brain Project */ +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project */ /* Raphael Dumusc */ /* All rights reserved. */ /* */ @@ -43,6 +43,7 @@ #include "ReceiveBuffer.h" #include +#include namespace deflect { @@ -52,26 +53,52 @@ class FrameDispatcher::Impl public: Impl() {} - FramePtr consumeLatestFrame( const QString& uri ) + FramePtr consumeLatestMonoFrame( const QString& uri ) { + const View view = deflect::View::MONO; + FramePtr frame( new Frame ); frame->uri = uri; + frame->view = view; ReceiveBuffer& buffer = streamBuffers[uri]; - - while( buffer.hasCompleteFrame( )) - frame->segments = buffer.popFrame(); - + while( buffer.hasCompleteFrame( view )) + frame->segments = buffer.popFrame( view ); assert( !frame->segments.empty( )); // receiver will request a new frame once this frame was consumed - buffer.setAllowedToSend( false ); - + buffer.setAllowedToSend( false, view ); return frame; } - typedef std::map StreamBuffers; - StreamBuffers streamBuffers; + std::pair consumeLatestStereoFrame( const QString& uri ) + { + FramePtr frameLeft( new Frame ); + frameLeft->uri = uri; + frameLeft->view = deflect::View::LEFT_EYE; + + FramePtr frameRight( new Frame ); + frameRight->uri = uri; + frameRight->view = deflect::View::RIGHT_EYE; + + ReceiveBuffer& buffer = streamBuffers[uri]; + + while( buffer.hasCompleteFrame( deflect::View::LEFT_EYE ) && + buffer.hasCompleteFrame( deflect::View::RIGHT_EYE )) + { + frameLeft->segments = buffer.popFrame( deflect::View::LEFT_EYE ); + frameRight->segments = buffer.popFrame( deflect::View::RIGHT_EYE ); + } + assert( !frameLeft->segments.empty( )); + assert( !frameRight->segments.empty( )); + + // receiver will request a new frame once this frame was consumed + buffer.setAllowedToSend( false, deflect::View::LEFT_EYE ); + buffer.setAllowedToSend( false, deflect::View::RIGHT_EYE ); + return std::make_pair( std::move( frameLeft ), std::move( frameRight )); + } + + std::map streamBuffers; }; FrameDispatcher::FrameDispatcher() @@ -85,7 +112,7 @@ void FrameDispatcher::addSource( const QString uri, const size_t sourceIndex ) _impl->streamBuffers[uri].addSource( sourceIndex ); if( _impl->streamBuffers[uri].getSourceCount() == 1 ) - emit openPixelStream( uri ); + emit pixelStreamOpened( uri ); } void FrameDispatcher::removeSource( const QString uri, @@ -102,23 +129,40 @@ void FrameDispatcher::removeSource( const QString uri, void FrameDispatcher::processSegment( const QString uri, const size_t sourceIndex, - deflect::Segment segment ) + deflect::Segment segment, + const deflect::View view ) { if( _impl->streamBuffers.count( uri )) - _impl->streamBuffers[uri].insert( segment, sourceIndex ); + _impl->streamBuffers[uri].insert( segment, sourceIndex, view ); } void FrameDispatcher::processFrameFinished( const QString uri, - const size_t sourceIndex ) + const size_t sourceIndex, + const deflect::View view ) { if( !_impl->streamBuffers.count( uri )) return; ReceiveBuffer& buffer = _impl->streamBuffers[uri]; - buffer.finishFrameForSource( sourceIndex ); + buffer.finishFrameForSource( sourceIndex, view ); - if( buffer.isAllowedToSend() && buffer.hasCompleteFrame( )) - emit sendFrame( _impl->consumeLatestFrame( uri )); + if( view == deflect::View::MONO ) + { + if( buffer.isAllowedToSend( view ) && buffer.hasCompleteFrame( view )) + emit sendFrame( _impl->consumeLatestMonoFrame( uri )); + } + else + { + if( buffer.isAllowedToSend( deflect::View::LEFT_EYE ) && + buffer.isAllowedToSend( deflect::View::RIGHT_EYE ) && + buffer.hasCompleteFrame( deflect::View::LEFT_EYE ) && + buffer.hasCompleteFrame( deflect::View::RIGHT_EYE )) + { + const auto frames = _impl->consumeLatestStereoFrame( uri ); + emit sendFrame( frames.first ); + emit sendFrame( frames.second ); + } + } } void FrameDispatcher::deleteStream( const QString uri ) @@ -126,7 +170,7 @@ void FrameDispatcher::deleteStream( const QString uri ) if( _impl->streamBuffers.count( uri )) { _impl->streamBuffers.erase( uri ); - emit deletePixelStream( uri ); + emit pixelStreamClosed( uri ); } } @@ -136,9 +180,20 @@ void FrameDispatcher::requestFrame( const QString uri ) return; ReceiveBuffer& buffer = _impl->streamBuffers[uri]; - buffer.setAllowedToSend( true ); - if( buffer.hasCompleteFrame( )) - emit sendFrame( _impl->consumeLatestFrame( uri )); + buffer.setAllowedToSend( true, deflect::View::MONO ); + buffer.setAllowedToSend( true, deflect::View::LEFT_EYE ); + buffer.setAllowedToSend( true, deflect::View::RIGHT_EYE ); + + if( buffer.hasCompleteFrame( deflect::View::MONO )) + emit sendFrame( _impl->consumeLatestMonoFrame( uri )); + + if( buffer.hasCompleteFrame( deflect::View::LEFT_EYE ) && + buffer.hasCompleteFrame( deflect::View::RIGHT_EYE )) + { + const auto frames = _impl->consumeLatestStereoFrame( uri ); + emit sendFrame( frames.first ); + emit sendFrame( frames.second ); + } } } diff --git a/deflect/FrameDispatcher.h b/deflect/FrameDispatcher.h index e1c53cc..a34859b 100644 --- a/deflect/FrameDispatcher.h +++ b/deflect/FrameDispatcher.h @@ -1,5 +1,5 @@ /*********************************************************************/ -/* Copyright (c) 2013-2015, EPFL/Blue Brain Project */ +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project */ /* Raphael Dumusc */ /* All rights reserved. */ /* */ @@ -45,7 +45,6 @@ #include #include -#include namespace deflect { @@ -87,17 +86,21 @@ public slots: * @param uri Identifier for the Stream * @param sourceIndex Identifier for the source in this stream * @param segment The segment to process + * @param view to which the segment belongs */ DEFLECT_API void processSegment( QString uri, size_t sourceIndex, - deflect::Segment segment ); + deflect::Segment segment, + deflect::View view ); /** * The given source has finished sending segments for the current frame. * * @param uri Identifier for the Stream * @param sourceIndex Identifier for the source in this stream + * @param view for which the frame is finished */ - DEFLECT_API void processFrameFinished( QString uri, size_t sourceIndex ); + DEFLECT_API void processFrameFinished( QString uri, size_t sourceIndex, + deflect::View view ); /** * Delete an entire stream. @@ -107,9 +110,14 @@ public slots: DEFLECT_API void deleteStream( QString uri ); /** - * Called by the user to request the dispatching of a new frame. + * Request the dispatching of a new frame for any stream (MONO/STEREO). + * + * A sendFrame() signal will be emitted for each of the view for which a + * frame becomes available. + * + * Stereo LEFT/RIGHT frames will only be be dispatched together when both + * are available to ensure that the two eye channels remain synchronized. * - * A sendFrame() signal will be emitted as soon as a frame is available. * @param uri Identifier for the stream */ DEFLECT_API void requestFrame( QString uri ); @@ -120,14 +128,14 @@ public slots: * * @param uri Identifier for the Stream */ - DEFLECT_API void openPixelStream( QString uri ); + DEFLECT_API void pixelStreamOpened( QString uri ); /** - * Notify that a pixel stream has been deleted. + * Notify that a pixel stream has been closed. * * @param uri Identifier for the Stream */ - DEFLECT_API void deletePixelStream( QString uri ); + DEFLECT_API void pixelStreamClosed( QString uri ); /** * Dispatch a full frame. diff --git a/deflect/ImageWrapper.h b/deflect/ImageWrapper.h index 0bdcdd6..8a172ed 100644 --- a/deflect/ImageWrapper.h +++ b/deflect/ImageWrapper.h @@ -1,6 +1,6 @@ /*********************************************************************/ -/* Copyright (c) 2013, EPFL/Blue Brain Project */ -/* Raphael Dumusc */ +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project */ +/* Raphael Dumusc */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -42,6 +42,7 @@ #include #include +#include namespace deflect { @@ -86,9 +87,8 @@ struct ImageWrapper * @version 1.0 */ DEFLECT_API - ImageWrapper( const void* data, const unsigned int width, - const unsigned int height, const PixelFormat format, - const unsigned int x = 0, const unsigned int y = 0 ); + ImageWrapper( const void* data, unsigned int width, unsigned int height, + PixelFormat format, unsigned int x = 0, unsigned int y = 0 ); /** Pointer to the image data of size getBufferSize(). @version 1.0 */ const void* const data; @@ -120,6 +120,12 @@ struct ImageWrapper @version 1.0 */ //@} + /** + * The view that this image represents in stereo 3D streams. + * @version 1.6 + */ + View view = View::MONO; + /** * Get the number of bytes per pixel based on the pixelFormat. * @version 1.0 diff --git a/deflect/MessageHeader.h b/deflect/MessageHeader.h index e6aab26..e3153ea 100644 --- a/deflect/MessageHeader.h +++ b/deflect/MessageHeader.h @@ -67,7 +67,8 @@ enum MessageType MESSAGE_TYPE_EVENT = 9, MESSAGE_TYPE_QUIT = 12, MESSAGE_TYPE_SIZE_HINTS = 13, - MESSAGE_TYPE_DATA = 14 + MESSAGE_TYPE_DATA = 14, + MESSAGE_TYPE_IMAGE_VIEW = 15 }; #define MESSAGE_HEADER_URI_LENGTH 64 diff --git a/deflect/MetaTypeRegistration.cpp b/deflect/MetaTypeRegistration.cpp index bae65bd..b31e120 100644 --- a/deflect/MetaTypeRegistration.cpp +++ b/deflect/MetaTypeRegistration.cpp @@ -59,6 +59,7 @@ struct MetaTypeRegistration qRegisterMetaType< deflect::SizeHints >( "deflect::SizeHints" ); qRegisterMetaType< deflect::Event >( "deflect::Event" ); qRegisterMetaType< deflect::FramePtr >( "deflect::FramePtr" ); + qRegisterMetaType< deflect::View >( "deflect::View" ); } }; diff --git a/deflect/ReceiveBuffer.cpp b/deflect/ReceiveBuffer.cpp index d8934dc..bab5dff 100644 --- a/deflect/ReceiveBuffer.cpp +++ b/deflect/ReceiveBuffer.cpp @@ -1,5 +1,5 @@ /*********************************************************************/ -/* Copyright (c) 2013-2015, EPFL/Blue Brain Project */ +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project */ /* Raphael Dumusc */ /* All rights reserved. */ /* */ @@ -44,23 +44,16 @@ namespace deflect { -ReceiveBuffer::ReceiveBuffer() - : _lastFrameComplete( 0 ) - , _allowedToSend( false ) -{ -} - bool ReceiveBuffer::addSource( const size_t sourceIndex ) { assert( !_sourceBuffers.count( sourceIndex )); // TODO: This function must return false if the stream was already started! - // This requires an full adaptation of the Stream library (DISCL-241) - if ( _sourceBuffers.count( sourceIndex )) + // This requires a full adaptation of the Stream library (DISCL-241) + if( _sourceBuffers.count( sourceIndex )) return false; _sourceBuffers[sourceIndex] = SourceBuffer(); - _sourceBuffers[sourceIndex].segments.push( Segments( )); return true; } @@ -74,56 +67,112 @@ size_t ReceiveBuffer::getSourceCount() const return _sourceBuffers.size(); } -void ReceiveBuffer::insert( const Segment& segment, const size_t sourceIndex ) +void ReceiveBuffer::insert( const Segment& segment, const size_t sourceIndex, + const deflect::View view ) { assert( _sourceBuffers.count( sourceIndex )); - _sourceBuffers[sourceIndex].segments.back().push_back( segment ); + _sourceBuffers[sourceIndex].insert( segment, view ); } -void ReceiveBuffer::finishFrameForSource( const size_t sourceIndex ) +void ReceiveBuffer::finishFrameForSource( const size_t sourceIndex, + const deflect::View view ) { assert( _sourceBuffers.count( sourceIndex )); - _sourceBuffers[sourceIndex].push(); + _sourceBuffers[sourceIndex].push( view ); } -bool ReceiveBuffer::hasCompleteFrame() const +bool ReceiveBuffer::hasCompleteFrame( const View view ) const { assert( !_sourceBuffers.empty( )); + const auto lastCompleteFrameIndex = _getLastCompleteFrameIndex( view ); + // Check if all sources for Stream have reached the same index for( const auto& kv : _sourceBuffers ) { const auto& buffer = kv.second; - if( buffer.backFrameIndex <= _lastFrameComplete ) + if( buffer.getBackFrameIndex( view ) <= lastCompleteFrameIndex ) return false; } return true; } -Segments ReceiveBuffer::popFrame() +Segments ReceiveBuffer::popFrame( const View view ) { Segments frame; for( auto& kv : _sourceBuffers ) { auto& buffer = kv.second; - frame.insert( frame.end(), buffer.segments.front().begin(), - buffer.segments.front().end( )); - buffer.pop(); + const auto& segments = buffer.getSegments( view ); + frame.insert( frame.end(), segments.begin(), segments.end( )); + buffer.pop( view ); } - ++_lastFrameComplete; + _incrementLastFrameComplete( view ); return frame; } -void ReceiveBuffer::setAllowedToSend( const bool enable ) +void ReceiveBuffer::setAllowedToSend( const bool enable, const View view ) +{ + switch( view ) + { + case View::MONO: + _allowedToSend = enable; + break; + case View::LEFT_EYE: + _allowedToSendLeft = enable; + break; + case View::RIGHT_EYE: + _allowedToSendRight = enable; + break; + }; +} + +bool ReceiveBuffer::isAllowedToSend( const View view ) const { - _allowedToSend = enable; + switch( view ) + { + case View::MONO: + return _allowedToSend; + case View::LEFT_EYE: + return _allowedToSendLeft; + case View::RIGHT_EYE: + return _allowedToSendRight; + default: + throw std::invalid_argument( "no such view" ); // keep compiler happy + }; } -bool ReceiveBuffer::isAllowedToSend() const +FrameIndex ReceiveBuffer::_getLastCompleteFrameIndex( const View view ) const { - return _allowedToSend; + switch( view ) + { + case View::MONO: + return _lastFrameComplete; + case View::LEFT_EYE: + return _lastFrameCompleteLeft; + case View::RIGHT_EYE: + return _lastFrameCompleteRight; + default: + throw std::invalid_argument( "no such view" ); // keep compiler happy + }; +} + +void ReceiveBuffer::_incrementLastFrameComplete( const View view ) +{ + switch( view ) + { + case View::MONO: + ++_lastFrameComplete; + break; + case View::LEFT_EYE: + ++_lastFrameCompleteLeft; + break; + case View::RIGHT_EYE: + ++_lastFrameCompleteRight; + break; + }; } } diff --git a/deflect/ReceiveBuffer.h b/deflect/ReceiveBuffer.h index 8244f70..1483711 100644 --- a/deflect/ReceiveBuffer.h +++ b/deflect/ReceiveBuffer.h @@ -1,5 +1,5 @@ /*********************************************************************/ -/* Copyright (c) 2013-2015, EPFL/Blue Brain Project */ +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project */ /* Raphael Dumusc */ /* All rights reserved. */ /* */ @@ -42,48 +42,16 @@ #include #include +#include #include #include -#include #include namespace deflect { -typedef unsigned int FrameIndex; - -/** - * Buffer for a single source of segements. - */ -struct SourceBuffer -{ - SourceBuffer() : frontFrameIndex( 0 ), backFrameIndex( 0 ) {} - - /** The current indexes of the frame for this source */ - FrameIndex frontFrameIndex, backFrameIndex; - - /** The collection of segments */ - std::queue segments; - - /** Pop the first element of the buffer */ - void pop() - { - segments.pop(); - ++frontFrameIndex; - } - - /** Push a new element to the back of the buffer */ - void push() - { - segments.push( Segments( )); - ++backFrameIndex; - } -}; - -typedef std::map SourceBufferMap; - /** * Buffer Segments from (multiple) sources. * @@ -93,9 +61,6 @@ typedef std::map SourceBufferMap; class ReceiveBuffer { public: - /** Construct a Buffer */ - DEFLECT_API ReceiveBuffer(); - /** * Add a source of segments. * @param sourceIndex Unique source identifier @@ -117,34 +82,47 @@ class ReceiveBuffer * Insert a segement for the current frame and source. * @param segment The segment to insert * @param sourceIndex Unique source identifier + * @param view in which the segment should be inserted */ - DEFLECT_API void insert( const Segment& segment, size_t sourceIndex ); + DEFLECT_API void insert( const Segment& segment, size_t sourceIndex, + deflect::View view = deflect::View::MONO ); /** * Call when the source has finished sending segments for the current frame. * @param sourceIndex Unique source identifier + * @param view for which to finish the frame */ - DEFLECT_API void finishFrameForSource( size_t sourceIndex ); + DEFLECT_API void finishFrameForSource( + size_t sourceIndex, deflect::View view = deflect::View::MONO ); /** Does the Buffer have a new complete frame (from all sources) */ - DEFLECT_API bool hasCompleteFrame() const; + DEFLECT_API bool hasCompleteFrame( View view = deflect::View::MONO ) const; /** * Get the finished frame. * @return A collection of segments that form a frame */ - DEFLECT_API Segments popFrame(); + DEFLECT_API Segments popFrame( View view = deflect::View::MONO ); /** Allow this buffer to be used by the next FrameDispatcher::sendLatestFrame */ - DEFLECT_API void setAllowedToSend( bool enable ); + DEFLECT_API void setAllowedToSend( bool enable, View view ); /** @return true if this buffer can be sent by FrameDispatcher */ - DEFLECT_API bool isAllowedToSend() const; + DEFLECT_API bool isAllowedToSend( View view ) const; private: - FrameIndex _lastFrameComplete; - SourceBufferMap _sourceBuffers; - bool _allowedToSend; + std::map _sourceBuffers; + + FrameIndex _lastFrameComplete = 0u; + FrameIndex _lastFrameCompleteLeft = 0u; + FrameIndex _lastFrameCompleteRight = 0u; + + bool _allowedToSend = false; + bool _allowedToSendLeft = false; + bool _allowedToSendRight = false; + + FrameIndex _getLastCompleteFrameIndex( View view ) const; + void _incrementLastFrameComplete( View view ); }; } diff --git a/deflect/SegmentParameters.h b/deflect/SegmentParameters.h index e8d7ec6..9fa9255 100644 --- a/deflect/SegmentParameters.h +++ b/deflect/SegmentParameters.h @@ -1,5 +1,5 @@ /*********************************************************************/ -/* Copyright (c) 2013-2016, EPFL/Blue Brain Project */ +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project */ /* Raphael Dumusc */ /* All rights reserved. */ /* */ @@ -46,6 +46,8 @@ #include #endif +#include + namespace deflect { @@ -56,27 +58,18 @@ struct SegmentParameters { /** @name Coordinates */ //@{ - uint32_t x; /**< The x position in pixels. */ - uint32_t y; /**< The y position in pixels. */ + uint32_t x = 0u; /**< The x position in pixels. */ + uint32_t y = 0u; /**< The y position in pixels. */ //@} /** @name Dimensions */ //@{ - uint32_t width; /**< The width in pixels. */ - uint32_t height; /**< The height in pixels. */ + uint32_t width = 0u; /**< The width in pixels. */ + uint32_t height = 0u; /**< The height in pixels. */ //@} /** Is the image raw pixel data or compressed in jpeg format */ - bool compressed; - - /** Default constructor */ - SegmentParameters() - : x( 0 ) - , y( 0 ) - , width( 0 ) - , height( 0 ) - , compressed( true ) - {} + bool compressed = true; }; } diff --git a/deflect/ServerWorker.cpp b/deflect/ServerWorker.cpp index fd759d4..45bbd46 100644 --- a/deflect/ServerWorker.cpp +++ b/deflect/ServerWorker.cpp @@ -61,6 +61,7 @@ ServerWorker::ServerWorker( const int socketDescriptor ) , _sourceId( socketDescriptor ) , _clientProtocolVersion( NETWORK_PROTOCOL_VERSION ) , _registeredToEvents( false ) + , _activeView( View::MONO ) { if( !_tcpSocket->setSocketDescriptor( socketDescriptor )) { @@ -234,7 +235,7 @@ void ServerWorker::_handleMessage( const MessageHeader& messageHeader, break; case MESSAGE_TYPE_PIXELSTREAM_FINISH_FRAME: - emit receivedFrameFinished( _streamId, _sourceId ); + emit receivedFrameFinished( _streamId, _sourceId, _activeView ); break; case MESSAGE_TYPE_PIXELSTREAM: @@ -253,6 +254,14 @@ void ServerWorker::_handleMessage( const MessageHeader& messageHeader, emit receivedData( _streamId, byteArray ); break; + case MESSAGE_TYPE_IMAGE_VIEW: + { + const auto view = reinterpret_cast( byteArray.data( )); + if( *view >= deflect::View::MONO && *view <= deflect::View::RIGHT_EYE ) + _activeView = *view; + break; + } + case MESSAGE_TYPE_BIND_EVENTS: case MESSAGE_TYPE_BIND_EVENTS_EX: if( _registeredToEvents ) @@ -287,7 +296,7 @@ void ServerWorker::_handlePixelStreamMessage( const QByteArray& message ) segment.imageData = message.right( message.size() - sizeof( SegmentParameters )); - emit( receivedSegment( _streamId, _sourceId, segment )); + emit( receivedSegment( _streamId, _sourceId, segment, _activeView )); } void ServerWorker::_sendProtocolVersion() diff --git a/deflect/ServerWorker.h b/deflect/ServerWorker.h index 4717cbe..1ee8e49 100644 --- a/deflect/ServerWorker.h +++ b/deflect/ServerWorker.h @@ -73,11 +73,12 @@ public slots: void removeStreamSource( QString uri, size_t sourceIndex ); void receivedSegment( QString uri, size_t sourceIndex, - deflect::Segment segment ); - void receivedFrameFinished( QString uri, size_t sourceIndex ); + deflect::Segment segment, deflect::View view ); + void receivedFrameFinished( QString uri, size_t sourceIndex, + deflect::View view ); void registerToEvents( QString uri, bool exclusive, - deflect::EventReceiver* receiver); + deflect::EventReceiver* receiver ); void receivedSizeHints( QString uri, deflect::SizeHints hints ); @@ -101,6 +102,8 @@ private slots: bool _registeredToEvents; QQueue _events; + View _activeView; + void _receiveMessage(); MessageHeader _receiveMessageHeader(); QByteArray _receiveMessageBody( int size ); diff --git a/deflect/SourceBuffer.cpp b/deflect/SourceBuffer.cpp new file mode 100644 index 0000000..6ebb4d6 --- /dev/null +++ b/deflect/SourceBuffer.cpp @@ -0,0 +1,133 @@ +/*********************************************************************/ +/* Copyright (c) 2017, EPFL/Blue Brain Project */ +/* Raphael Dumusc */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of Ecole polytechnique federale de Lausanne. */ +/*********************************************************************/ + +#include "SourceBuffer.h" + +#include + +namespace deflect +{ + +SourceBuffer::SourceBuffer() +{ + _segmentsMono.push( Segments( )); + _segmentsLeft.push( Segments( )); + _segmentsRight.push( Segments( )); +} + +const Segments& SourceBuffer::getSegments( const View view ) const +{ + return _getQueue( view ).front(); +} + +FrameIndex SourceBuffer::getBackFrameIndex( const View view ) const +{ + switch( view ) + { + case View::MONO: + return _backFrameIndexMono; + case View::LEFT_EYE: + return _backFrameIndexLeft; + case View::RIGHT_EYE: + return _backFrameIndexRight; + default: + throw std::invalid_argument( "no such view" ); // keep compiler happy + }; +} + +void SourceBuffer::pop( const View view ) +{ + _getQueue( view ).pop(); +} + +void SourceBuffer::push( const View view ) +{ + _getQueue( view ).push( Segments( )); + + switch( view ) + { + case View::MONO: + ++_backFrameIndexMono; + break; + case View::LEFT_EYE: + ++_backFrameIndexLeft; + break; + case View::RIGHT_EYE: + ++_backFrameIndexRight; + break; + }; +} + +void SourceBuffer::insert( const Segment& segment, const deflect::View view ) +{ + _getQueue( view ).back().push_back( segment ); +} + +std::queue& SourceBuffer::_getQueue( const deflect::View view ) +{ + switch( view ) + { + case View::MONO: + return _segmentsMono; + case View::LEFT_EYE: + return _segmentsLeft; + case View::RIGHT_EYE: + return _segmentsRight; + default: + throw std::invalid_argument( "no such view" ); // keep compiler happy + }; +} + +const std::queue& +SourceBuffer::_getQueue( const deflect::View view ) const +{ + switch( view ) + { + case View::MONO: + return _segmentsMono; + case View::LEFT_EYE: + return _segmentsLeft; + case View::RIGHT_EYE: + return _segmentsRight; + default: + throw std::invalid_argument( "no such view" ); // keep compiler happy + }; +} + +} diff --git a/deflect/SourceBuffer.h b/deflect/SourceBuffer.h new file mode 100644 index 0000000..b711114 --- /dev/null +++ b/deflect/SourceBuffer.h @@ -0,0 +1,93 @@ +/*********************************************************************/ +/* Copyright (c) 2017, EPFL/Blue Brain Project */ +/* Raphael Dumusc */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of Ecole polytechnique federale de Lausanne. */ +/*********************************************************************/ + +#ifndef DEFLECT_SOURCEBUFFER_H +#define DEFLECT_SOURCEBUFFER_H + +#include +#include +#include + +#include + +namespace deflect +{ + +using FrameIndex = unsigned int; + +/** + * Buffer for a single source of segments. + */ +class SourceBuffer +{ +public: + /** Construct an empty buffer. */ + SourceBuffer(); + + /** @return the segments at the front of the queue for a given view. */ + const Segments& getSegments( View view ) const; + + /** @return the frame index of the back of the buffer for a given view. */ + FrameIndex getBackFrameIndex( View view ) const; + + /** Insert a segment into the back frame of the appropriate queue. */ + void insert( const Segment& segment, const View view ); + + /** Push a new frame to the back of given view. */ + void push( const View view ); + + /** Pop the front frame of the buffer for the given view. */ + void pop( const View view ); + +private: + /** The collections of segments for each view. */ + std::queue _segmentsMono, _segmentsLeft, _segmentsRight; + + /** The current indices of the frame for this source. */ + FrameIndex _backFrameIndexMono = 0u; + FrameIndex _backFrameIndexLeft = 0u; + FrameIndex _backFrameIndexRight = 0u; + + std::queue& _getQueue( View view ); + const std::queue& _getQueue( View view ) const; +}; + +} + +#endif diff --git a/deflect/StreamPrivate.cpp b/deflect/StreamPrivate.cpp index 49b1113..8e002b0 100644 --- a/deflect/StreamPrivate.cpp +++ b/deflect/StreamPrivate.cpp @@ -141,6 +141,7 @@ bool StreamPrivate::send( const ImageWrapper& image ) return false; } + sendImageView( image.view ); const auto sendFunc = std::bind( &StreamPrivate::sendPixelStreamSegment, this, std::placeholders::_1 ); return imageSegmenter.generate( image, sendFunc ); @@ -161,6 +162,15 @@ bool StreamPrivate::finishFrame() return socket.send( mh, QByteArray( )); } +bool StreamPrivate::sendImageView( const View view ) +{ + QByteArray message; + message.append( (const char*)( &view ), sizeof(View) ); + + const MessageHeader mh( MESSAGE_TYPE_IMAGE_VIEW, message.size(), id ); + return socket.send( mh, message ); +} + bool StreamPrivate::sendPixelStreamSegment( const Segment& segment ) { // Create message header diff --git a/deflect/StreamPrivate.h b/deflect/StreamPrivate.h index 03ed288..2f03721 100644 --- a/deflect/StreamPrivate.h +++ b/deflect/StreamPrivate.h @@ -99,6 +99,9 @@ class StreamPrivate /** @sa Stream::finishFrame */ bool finishFrame(); + /** Send the view for the image to be sent with sendPixelStreamSegment. */ + bool sendImageView( View view ); + /** * Send a Segment through the Stream. * @param segment An image segment with valid parameters and data diff --git a/deflect/types.h b/deflect/types.h index d51131d..6815997 100644 --- a/deflect/types.h +++ b/deflect/types.h @@ -49,6 +49,9 @@ namespace deflect { +/** The different types of view. */ +enum class View : std::int8_t { MONO, LEFT_EYE, RIGHT_EYE }; + class EventReceiver; class Frame; class FrameDispatcher; diff --git a/doc/Changelog.md b/doc/Changelog.md index c8b830b..df01493 100644 --- a/doc/Changelog.md +++ b/doc/Changelog.md @@ -1,6 +1,11 @@ Changelog {#Changelog} ============ +## Deflect 0.13 (git master) + +* [148](https://github.com/BlueBrain/Deflect/pull/148): + Support for streaming stereo 3D content in a frame-sequential manner. + ## Deflect 0.12 ### 0.12.1 (01-02-2017) From 56253b3250918a636bcece9fe7c1b336bdf04ed7 Mon Sep 17 00:00:00 2001 From: Raphael Dumusc Date: Wed, 1 Feb 2017 15:17:34 +0100 Subject: [PATCH 2/7] Improved Server interface, FrameDispatcher is now private --- deflect/CMakeLists.txt | 2 +- deflect/FrameDispatcher.cpp | 18 +++---- deflect/FrameDispatcher.h | 63 +++++++++++++------------ deflect/Server.cpp | 47 ++++++++++--------- deflect/Server.h | 93 ++++++++++++++++++++++++++++++++----- tests/cpp/ServerTests.cpp | 2 +- 6 files changed, 148 insertions(+), 77 deletions(-) diff --git a/deflect/CMakeLists.txt b/deflect/CMakeLists.txt index c039a11..1760f8d 100644 --- a/deflect/CMakeLists.txt +++ b/deflect/CMakeLists.txt @@ -7,7 +7,6 @@ set(DEFLECT_PUBLIC_HEADERS config.h Event.h EventReceiver.h - FrameDispatcher.h Frame.h ImageWrapper.h MTQueue.h @@ -20,6 +19,7 @@ set(DEFLECT_PUBLIC_HEADERS ) set(DEFLECT_HEADERS + FrameDispatcher.h ImageSegmenter.h MessageHeader.h NetworkProtocol.h diff --git a/deflect/FrameDispatcher.cpp b/deflect/FrameDispatcher.cpp index 8cd878d..cbd7559 100644 --- a/deflect/FrameDispatcher.cpp +++ b/deflect/FrameDispatcher.cpp @@ -165,15 +165,6 @@ void FrameDispatcher::processFrameFinished( const QString uri, } } -void FrameDispatcher::deleteStream( const QString uri ) -{ - if( _impl->streamBuffers.count( uri )) - { - _impl->streamBuffers.erase( uri ); - emit pixelStreamClosed( uri ); - } -} - void FrameDispatcher::requestFrame( const QString uri ) { if( !_impl->streamBuffers.count( uri )) @@ -196,4 +187,13 @@ void FrameDispatcher::requestFrame( const QString uri ) } } +void FrameDispatcher::deleteStream( const QString uri ) +{ + if( _impl->streamBuffers.count( uri )) + { + _impl->streamBuffers.erase( uri ); + emit pixelStreamClosed( uri ); + } +} + } diff --git a/deflect/FrameDispatcher.h b/deflect/FrameDispatcher.h index a34859b..e9d29aa 100644 --- a/deflect/FrameDispatcher.h +++ b/deflect/FrameDispatcher.h @@ -58,56 +58,48 @@ class FrameDispatcher : public QObject public: /** Construct a dispatcher */ - DEFLECT_API FrameDispatcher(); + FrameDispatcher(); /** Destructor. */ - DEFLECT_API ~FrameDispatcher(); + ~FrameDispatcher(); public slots: /** * Add a source of Segments for a Stream. * - * @param uri Identifier for the Stream + * @param uri Identifier for the stream * @param sourceIndex Identifier for the source in this stream */ - DEFLECT_API void addSource( QString uri, size_t sourceIndex ); + void addSource( QString uri, size_t sourceIndex ); /** - * Add a source of Segments for a Stream. + * Remove a source of Segments for a Stream. * - * @param uri Identifier for the Stream + * @param uri Identifier for the stream * @param sourceIndex Identifier for the source in this stream */ - DEFLECT_API void removeSource( QString uri, size_t sourceIndex ); + void removeSource( QString uri, size_t sourceIndex ); /** - * Process a new Segement. + * Process a new Segment. * - * @param uri Identifier for the Stream - * @param sourceIndex Identifier for the source in this stream - * @param segment The segment to process + * @param uri Identifier for the stream + * @param sourceIndex Identifier for the source in the stream + * @param segment to process * @param view to which the segment belongs */ - DEFLECT_API void processSegment( QString uri, size_t sourceIndex, - deflect::Segment segment, - deflect::View view ); + void processSegment( QString uri, size_t sourceIndex, + deflect::Segment segment, deflect::View view ); /** * The given source has finished sending segments for the current frame. * - * @param uri Identifier for the Stream - * @param sourceIndex Identifier for the source in this stream + * @param uri Identifier for the stream + * @param sourceIndex Identifier for the source in the stream * @param view for which the frame is finished */ - DEFLECT_API void processFrameFinished( QString uri, size_t sourceIndex, - deflect::View view ); - - /** - * Delete an entire stream. - * - * @param uri Identifier for the Stream - */ - DEFLECT_API void deleteStream( QString uri ); + void processFrameFinished( QString uri, size_t sourceIndex, + deflect::View view ); /** * Request the dispatching of a new frame for any stream (MONO/STEREO). @@ -120,29 +112,36 @@ public slots: * * @param uri Identifier for the stream */ - DEFLECT_API void requestFrame( QString uri ); + void requestFrame( QString uri ); + + /** + * Delete all the buffers for a Stream. + * + * @param uri Identifier for the stream + */ + void deleteStream( QString uri ); signals: /** * Notify that a PixelStream has been opened. * - * @param uri Identifier for the Stream + * @param uri Identifier for the stream */ - DEFLECT_API void pixelStreamOpened( QString uri ); + void pixelStreamOpened( QString uri ); /** * Notify that a pixel stream has been closed. * - * @param uri Identifier for the Stream + * @param uri Identifier for the stream */ - DEFLECT_API void pixelStreamClosed( QString uri ); + void pixelStreamClosed( QString uri ); /** * Dispatch a full frame. * - * @param frame The frame to dispatch + * @param frame The latest frame available for a stream */ - DEFLECT_API void sendFrame( deflect::FramePtr frame ); + void sendFrame( deflect::FramePtr frame ); private: class Impl; diff --git a/deflect/Server.cpp b/deflect/Server.cpp index 6dca945..ed946ce 100644 --- a/deflect/Server.cpp +++ b/deflect/Server.cpp @@ -54,7 +54,7 @@ const int Server::defaultPortNumber = DEFAULT_PORT_NUMBER; class Server::Impl { public: - FrameDispatcher pixelStreamDispatcher; + FrameDispatcher frameDispatcher; }; Server::Server( const int port ) @@ -65,6 +65,14 @@ Server::Server( const int port ) const auto err = QString( "could not listen on port: %1" ).arg( port ); throw std::runtime_error( err.toStdString( )); } + + // Forward FrameDispatcher signals + connect( &_impl->frameDispatcher, &FrameDispatcher::pixelStreamOpened, + this, &Server::pixelStreamOpened ); + connect( &_impl->frameDispatcher, &FrameDispatcher::pixelStreamClosed, + this, &Server::pixelStreamClosed ); + connect( &_impl->frameDispatcher, &FrameDispatcher::sendFrame, + this, &Server::receivedFrame ); } Server::~Server() @@ -77,21 +85,20 @@ Server::~Server() workerThread->wait(); } } - - delete _impl; } -FrameDispatcher& Server::getPixelStreamDispatcher() +void Server::requestFrame( const QString uri ) { - return _impl->pixelStreamDispatcher; + _impl->frameDispatcher.requestFrame( uri ); } -void Server::onPixelStreamerClosed( const QString uri ) +void Server::closePixelStream( const QString uri ) { - emit _pixelStreamerClosed( uri ); + emit _closePixelStream( uri ); + _impl->frameDispatcher.deleteStream( uri ); } -void Server::onEventRegistrationReply( const QString uri, const bool success ) +void Server::replyToEventRegistration( const QString uri, const bool success ) { emit _eventRegistrationReply( uri, success ); } @@ -121,26 +128,20 @@ void Server::incomingConnection( const qintptr socketHandle ) this, &Server::receivedSizeHints ); connect( worker, &ServerWorker::receivedData, this, &Server::receivedData ); - connect( this, &Server::_pixelStreamerClosed, + connect( this, &Server::_closePixelStream, worker, &ServerWorker::closeConnection ); connect( this, &Server::_eventRegistrationReply, worker, &ServerWorker::replyToEventRegistration ); - // PixelStreamDispatcher + // FrameDispatcher connect( worker, &ServerWorker::addStreamSource, - &_impl->pixelStreamDispatcher, &FrameDispatcher::addSource ); - connect( worker, - &ServerWorker::receivedSegment, - &_impl->pixelStreamDispatcher, - &FrameDispatcher::processSegment ); - connect( worker, - &ServerWorker::receivedFrameFinished, - &_impl->pixelStreamDispatcher, - &FrameDispatcher::processFrameFinished ); - connect( worker, - &ServerWorker::removeStreamSource, - &_impl->pixelStreamDispatcher, - &FrameDispatcher::removeSource ); + &_impl->frameDispatcher, &FrameDispatcher::addSource ); + connect( worker, &ServerWorker::receivedSegment, + &_impl->frameDispatcher, &FrameDispatcher::processSegment ); + connect( worker, &ServerWorker::receivedFrameFinished, + &_impl->frameDispatcher, &FrameDispatcher::processFrameFinished ); + connect( worker, &ServerWorker::removeStreamSource, + &_impl->frameDispatcher, &FrameDispatcher::removeSource ); workerThread->start(); } diff --git a/deflect/Server.h b/deflect/Server.h index 6b901bc..b7e77e2 100644 --- a/deflect/Server.h +++ b/deflect/Server.h @@ -1,5 +1,5 @@ /*********************************************************************/ -/* Copyright (c) 2013-2016, EPFL/Blue Brain Project */ +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project */ /* Raphael Dumusc */ /* Daniel.Nachbaur@epfl.ch */ /* All rights reserved. */ @@ -51,7 +51,12 @@ namespace deflect { /** - * Listen to incoming PixelStream connections from Stream clients. + * Listen to incoming connections from multiple Stream clients. + * + * Both mono and frame-sequential stereo 3D streams are supported. + * + * The server integrates a flow-control mechanism to ensure that new frames are + * dispatched only as fast as the application is capable of processing them. */ class DEFLECT_API Server : public QTcpServer { @@ -63,38 +68,104 @@ class DEFLECT_API Server : public QTcpServer /** * Create a new server listening for Stream connections. + * * @param port The port to listen on. Must be available. * @throw std::runtime_error if the server could not be started. */ explicit Server( int port = defaultPortNumber ); - /** Destructor */ + /** Stop the server and close all open pixel stream connections. */ ~Server(); - /** Get the PixelStreamDispatcher. */ - FrameDispatcher& getPixelStreamDispatcher(); +public slots: + /** + * Request the dispatching of the next frame for a given pixel stream. + * + * A receivedFrame() signal will subsequently be emitted for each of the + * view(s) (MONO or STEREO) for which a frame is or becomes available. + * + * To ensure that the two eye channels remain synchronized, stereo + * LEFT/RIGHT frames are dispatched together only when both are available. + * + * @param uri Identifier for the stream + */ + void requestFrame( QString uri ); + + /** + * Reply to an event registration request after a registerToEvents() signal. + * + * @param uri Identifier for the stream + * @param success Result of the registration operation + */ + void replyToEventRegistration( QString uri, bool success ); + + /** + * Close a pixel stream, disconnecting the remote client. + * + * @param uri Identifier for the stream + */ + void closePixelStream( QString uri ); signals: + /** + * Notify that a pixel stream has been opened. + * + * @param uri Identifier for the stream + */ + void pixelStreamOpened( QString uri ); + + /** + * Notify that a pixel stream has been closed. + * + * @param uri Identifier for the stream + */ + void pixelStreamClosed( QString uri ); + + /** + * Emitted when a full frame has been received from a pixel stream. + * + * This signal is only emitted after the application signals that it is + * ready to handle a new frame by calling requestFrame(). + * + * @param frame The latest frame that was received for a stream. + */ + void receivedFrame( deflect::FramePtr frame ); + + /** + * Emitted when a remote client wants to register for receiving events. + * + * @param uri Identifier for the stream + * @param exclusive true if the receiver should receive events exclusively + * @param receiver the event receiver instance + */ void registerToEvents( QString uri, bool exclusive, deflect::EventReceiver* receiver ); + /** + * Emitted when a remote client sends size hints for displaying the stream. + * + * @param uri Identifier for the stream + * @param hints The size hints to apply + */ void receivedSizeHints( QString uri, deflect::SizeHints hints ); + /** + * Emitted when a remote client sends generic data. + * + * @param uri Identifier for the stream + * @param data A streamer-specific message + */ void receivedData( QString uri, QByteArray data ); -public slots: - void onPixelStreamerClosed( QString uri ); - void onEventRegistrationReply( QString uri, bool success ); - private: class Impl; - Impl* _impl; + std::unique_ptr _impl; /** Re-implemented handling of connections from QTCPSocket. */ void incomingConnection( qintptr socketHandle ) final; signals: - void _pixelStreamerClosed( QString uri ); + void _closePixelStream( QString uri ); void _eventRegistrationReply( QString uri, bool success ); }; diff --git a/tests/cpp/ServerTests.cpp b/tests/cpp/ServerTests.cpp index 73fd020..abfb617 100644 --- a/tests/cpp/ServerTests.cpp +++ b/tests/cpp/ServerTests.cpp @@ -142,7 +142,7 @@ BOOST_AUTO_TEST_CASE( testRegisterForEventReceivedByServer ) streamId = id; exclusiveBind = exclusive; eventReceiver = receiver; - server->onEventRegistrationReply( id, true ); // send reply to Stream + server->replyToEventRegistration( id, true ); // send reply to Stream mutex.lock(); receivedState = true; received.wakeAll(); From 7676567e71a2506ab561d5702832936ed3702625 Mon Sep 17 00:00:00 2001 From: Raphael Dumusc Date: Wed, 1 Feb 2017 16:48:44 +0100 Subject: [PATCH 3/7] Server: added safety check to close likely pixels streams Prior to this, the queue of unprocessed segments could grow indefinitely if one of multiple sources of segments stopped sending, or if the application stopped requesting frames (after a JPEG decompression error from a corrupted segment, for instance). --- deflect/FrameDispatcher.cpp | 10 +++++++- deflect/FrameDispatcher.h | 7 ++++++ deflect/ReceiveBuffer.cpp | 8 ++++++ deflect/ReceiveBuffer.h | 1 + deflect/Server.cpp | 2 ++ deflect/SourceBuffer.cpp | 5 ++++ deflect/SourceBuffer.h | 3 +++ tests/cpp/ReceiveBufferTests.cpp | 42 ++++++++++++++++++++++++++++++++ 8 files changed, 77 insertions(+), 1 deletion(-) diff --git a/deflect/FrameDispatcher.cpp b/deflect/FrameDispatcher.cpp index cbd7559..5224f42 100644 --- a/deflect/FrameDispatcher.cpp +++ b/deflect/FrameDispatcher.cpp @@ -144,7 +144,15 @@ void FrameDispatcher::processFrameFinished( const QString uri, return; ReceiveBuffer& buffer = _impl->streamBuffers[uri]; - buffer.finishFrameForSource( sourceIndex, view ); + try + { + buffer.finishFrameForSource( sourceIndex, view ); + } + catch( const std::runtime_error& ) + { + emit bufferSizeExceeded( uri ); + return; + } if( view == deflect::View::MONO ) { diff --git a/deflect/FrameDispatcher.h b/deflect/FrameDispatcher.h index e9d29aa..81b0698 100644 --- a/deflect/FrameDispatcher.h +++ b/deflect/FrameDispatcher.h @@ -143,6 +143,13 @@ public slots: */ void sendFrame( deflect::FramePtr frame ); + /** + * Notify that a pixel stream has exceeded its maximum allowed size. + * + * @param uri Identifier for the stream + */ + void bufferSizeExceeded( QString uri ); + private: class Impl; std::unique_ptr _impl; diff --git a/deflect/ReceiveBuffer.cpp b/deflect/ReceiveBuffer.cpp index bab5dff..adb689e 100644 --- a/deflect/ReceiveBuffer.cpp +++ b/deflect/ReceiveBuffer.cpp @@ -41,6 +41,11 @@ #include +namespace +{ +const size_t MAX_QUEUE_SIZE = 150; // stream blocked for ~5 seconds at 30Hz +} + namespace deflect { @@ -80,6 +85,9 @@ void ReceiveBuffer::finishFrameForSource( const size_t sourceIndex, { assert( _sourceBuffers.count( sourceIndex )); + if( _sourceBuffers[sourceIndex].getQueueSize( view ) > MAX_QUEUE_SIZE ) + throw std::runtime_error( "maximum queue size exceeded" ); + _sourceBuffers[sourceIndex].push( view ); } diff --git a/deflect/ReceiveBuffer.h b/deflect/ReceiveBuffer.h index 1483711..3442332 100644 --- a/deflect/ReceiveBuffer.h +++ b/deflect/ReceiveBuffer.h @@ -91,6 +91,7 @@ class ReceiveBuffer * Call when the source has finished sending segments for the current frame. * @param sourceIndex Unique source identifier * @param view for which to finish the frame + * @throw std::runtime_error if the buffer exceeds its maximum size */ DEFLECT_API void finishFrameForSource( size_t sourceIndex, deflect::View view = deflect::View::MONO ); diff --git a/deflect/Server.cpp b/deflect/Server.cpp index ed946ce..6cb7a48 100644 --- a/deflect/Server.cpp +++ b/deflect/Server.cpp @@ -73,6 +73,8 @@ Server::Server( const int port ) this, &Server::pixelStreamClosed ); connect( &_impl->frameDispatcher, &FrameDispatcher::sendFrame, this, &Server::receivedFrame ); + connect( &_impl->frameDispatcher, &FrameDispatcher::bufferSizeExceeded, + this, &Server::closePixelStream ); } Server::~Server() diff --git a/deflect/SourceBuffer.cpp b/deflect/SourceBuffer.cpp index 6ebb4d6..ea05ca7 100644 --- a/deflect/SourceBuffer.cpp +++ b/deflect/SourceBuffer.cpp @@ -99,6 +99,11 @@ void SourceBuffer::insert( const Segment& segment, const deflect::View view ) _getQueue( view ).back().push_back( segment ); } +size_t SourceBuffer::getQueueSize( const View view ) const +{ + return _getQueue( view ).size(); +} + std::queue& SourceBuffer::_getQueue( const deflect::View view ) { switch( view ) diff --git a/deflect/SourceBuffer.h b/deflect/SourceBuffer.h index b711114..0862938 100644 --- a/deflect/SourceBuffer.h +++ b/deflect/SourceBuffer.h @@ -75,6 +75,9 @@ class SourceBuffer /** Pop the front frame of the buffer for the given view. */ void pop( const View view ); + /** @return the size of the queue for the given view. */ + size_t getQueueSize( const View view ) const; + private: /** The collections of segments for each view. */ std::queue _segmentsMono, _segmentsLeft, _segmentsRight; diff --git a/tests/cpp/ReceiveBufferTests.cpp b/tests/cpp/ReceiveBufferTests.cpp index 47c5aab..07fe247 100644 --- a/tests/cpp/ReceiveBufferTests.cpp +++ b/tests/cpp/ReceiveBufferTests.cpp @@ -247,3 +247,45 @@ BOOST_AUTO_TEST_CASE( TestRemoveSourceWhileStreaming ) BOOST_CHECK_EQUAL( frameSize.width(), 192 ); BOOST_CHECK_EQUAL( frameSize.height(), 256 ); } + +BOOST_AUTO_TEST_CASE( TestOneOfTwoSourceStopsSendingSegments ) +{ + const size_t sourceIndex1 = 46; + const size_t sourceIndex2 = 819; + + deflect::ReceiveBuffer buffer; + buffer.addSource( sourceIndex1 ); + buffer.addSource( sourceIndex2 ); + + deflect::Segments testSegments = generateTestSegments(); + + // First Frame - 2 sources + buffer.insert( testSegments[0], sourceIndex1 ); + buffer.insert( testSegments[1], sourceIndex1 ); + buffer.insert( testSegments[2], sourceIndex2 ); + buffer.insert( testSegments[3], sourceIndex2 ); + BOOST_CHECK( !buffer.hasCompleteFrame( )); + buffer.finishFrameForSource(sourceIndex1); + BOOST_CHECK( !buffer.hasCompleteFrame( )); + buffer.finishFrameForSource( sourceIndex2 ); + BOOST_CHECK( buffer.hasCompleteFrame( )); + + deflect::Segments segments = buffer.popFrame(); + + BOOST_CHECK_EQUAL( segments.size(), 4 ); + BOOST_CHECK( !buffer.hasCompleteFrame( )); + + // Next frames - one source stops sending segments + for( int i = 0; i < 150; ++i ) + { + buffer.insert( testSegments[0], sourceIndex1 ); + buffer.insert( testSegments[1], sourceIndex1 ); + BOOST_REQUIRE_NO_THROW( buffer.finishFrameForSource( sourceIndex1 )); + BOOST_REQUIRE( !buffer.hasCompleteFrame( )); + } + // Test buffer exceeds maximum allowed size + buffer.insert( testSegments[0], sourceIndex1 ); + buffer.insert( testSegments[1], sourceIndex1 ); + BOOST_CHECK_THROW( buffer.finishFrameForSource( sourceIndex1 ), + std::runtime_error ); +} From d94bedeaa183db485e5e89ec523eae33e8b9f090 Mon Sep 17 00:00:00 2001 From: Raphael Dumusc Date: Fri, 3 Feb 2017 17:41:09 +0100 Subject: [PATCH 4/7] Server: support independant Streams for LEFT/RIGHT stereo views --- apps/SimpleStreamer/main.cpp | 75 +++++++--- deflect/FrameDispatcher.cpp | 47 +++---- deflect/ReceiveBuffer.cpp | 54 ++++++-- deflect/ReceiveBuffer.h | 7 +- deflect/ServerWorker.cpp | 2 +- tests/cpp/ReceiveBufferTests.cpp | 230 ++++++++++++++++++++++++++----- 6 files changed, 325 insertions(+), 90 deletions(-) diff --git a/apps/SimpleStreamer/main.cpp b/apps/SimpleStreamer/main.cpp index abc2674..7022f66 100644 --- a/apps/SimpleStreamer/main.cpp +++ b/apps/SimpleStreamer/main.cpp @@ -57,7 +57,8 @@ bool deflectInteraction = false; bool deflectCompressImage = true; -bool deflectStereoStream = false; +bool deflectStereoStreamLeft = false; +bool deflectStereoStreamRight = false; unsigned int deflectCompressionQuality = 75; std::string deflectHost; std::string deflectStreamId = "SimpleStreamer"; @@ -100,7 +101,9 @@ void syntax( const int exitStatus ) std::cout << " -n set stream identifier (default: 'SimpleStreamer')" << std::endl; std::cout << " -i enable interaction events (default: OFF)" << std::endl; std::cout << " -u enable uncompressed streaming (default: OFF)" << std::endl; - std::cout << " -s enable stereo streaming (default: OFF)" << std::endl; + std::cout << " -s enable stereo streaming, equivalent to -l -r (default: OFF)" << std::endl; + std::cout << " -l enable stereo streaming, left image only (default: OFF)" << std::endl; + std::cout << " -r enable stereo streaming, right image only (default: OFF)" << std::endl; exit( exitStatus ); } @@ -129,7 +132,14 @@ void readCommandLineArguments( int argc, char** argv ) deflectCompressImage = false; break; case 's': - deflectStereoStream = true; + deflectStereoStreamLeft = true; + deflectStereoStreamRight = true; + break; + case 'l': + deflectStereoStreamLeft = true; + break; + case 'r': + deflectStereoStreamRight = true; break; case 'h': syntax( EXIT_SUCCESS ); @@ -245,26 +255,48 @@ bool send( const Image& image, const deflect::View view ) deflectStream->finishFrame(); } +bool timeout( const float sec ) +{ + using clock = std::chrono::system_clock; + static clock::time_point start = clock::now(); + return std::chrono::duration{ clock::now() - start }.count() > sec; +} + void display() { static Camera camera; camera.apply(); bool success = false; - if( deflectStereoStream ) + bool waitToStart = false; + static bool deflectFirstEventReceived = false; + if( deflectStereoStreamLeft || deflectStereoStreamRight ) { - glClearColor( 0.7, 0.3, 0.3, 1.0 ); - glClear( GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT ); - glutSolidTeapot( 1.f ); - const auto leftImage = Image::readGlBuffer(); - - glClearColor( 0.3, 0.7, 0.3, 1.0 ); - glClear( GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT ); - glutSolidTeapot( 1.f ); - const auto rightImage = Image::readGlBuffer(); - - success = send( leftImage, deflect::View::LEFT_EYE ) && - send( rightImage, deflect::View::RIGHT_EYE ); + // Poor man's attempt to synchronise the start of separate stereo + // streams (waiting on first event from server or 5 sec. timeout). + // This does not prevent applications from going quickly out of sync. + // Real-world applications need a dedicated synchronization channel to + // ensure corresponding left-right views are rendered and sent together. + if( !(deflectStereoStreamLeft && deflectStereoStreamRight )) + waitToStart = !(deflectFirstEventReceived || timeout( 5 )); + + if( deflectStereoStreamLeft && !waitToStart ) + { + glClearColor( 0.7, 0.3, 0.3, 1.0 ); + glClear( GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT ); + glutSolidTeapot( 1.f ); + const auto leftImage = Image::readGlBuffer(); + success = send( leftImage, deflect::View::LEFT_EYE ); + } + if( deflectStereoStreamRight && !waitToStart ) + { + glClearColor( 0.3, 0.7, 0.3, 1.0 ); + glClear( GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT ); + glutSolidTeapot( 1.f ); + const auto rightImage = Image::readGlBuffer(); + success = (!deflectStereoStreamLeft || success) && + send( rightImage, deflect::View::RIGHT_EYE ); + } } else { @@ -291,6 +323,8 @@ void display() { const deflect::Event& event = deflectStream->getEvent(); + deflectFirstEventReceived = true; + switch( event.type ) { case deflect::Event::EVT_CLOSE: @@ -338,8 +372,11 @@ void display() } else { - camera.angleX += 1.f; - camera.angleY += 1.f; + if( !waitToStart ) + { + camera.angleX += 1.f; + camera.angleY += 1.f; + } } if( !success ) @@ -349,7 +386,7 @@ void display() std::cout << "Stream closed, exiting." << std::endl; exit( EXIT_SUCCESS ); } - else + else if( !waitToStart ) { std::cerr << "failure in deflectStreamSend()" << std::endl; exit( EXIT_FAILURE ); diff --git a/deflect/FrameDispatcher.cpp b/deflect/FrameDispatcher.cpp index 5224f42..ac73337 100644 --- a/deflect/FrameDispatcher.cpp +++ b/deflect/FrameDispatcher.cpp @@ -55,19 +55,17 @@ class FrameDispatcher::Impl FramePtr consumeLatestMonoFrame( const QString& uri ) { - const View view = deflect::View::MONO; - FramePtr frame( new Frame ); frame->uri = uri; - frame->view = view; + frame->view = View::MONO; ReceiveBuffer& buffer = streamBuffers[uri]; - while( buffer.hasCompleteFrame( view )) - frame->segments = buffer.popFrame( view ); + while( buffer.hasCompleteMonoFrame( )) + frame->segments = buffer.popFrame( View::MONO ); assert( !frame->segments.empty( )); // receiver will request a new frame once this frame was consumed - buffer.setAllowedToSend( false, view ); + buffer.setAllowedToSend( false, View::MONO ); return frame; } @@ -75,26 +73,25 @@ class FrameDispatcher::Impl { FramePtr frameLeft( new Frame ); frameLeft->uri = uri; - frameLeft->view = deflect::View::LEFT_EYE; + frameLeft->view = View::LEFT_EYE; FramePtr frameRight( new Frame ); frameRight->uri = uri; - frameRight->view = deflect::View::RIGHT_EYE; + frameRight->view = View::RIGHT_EYE; ReceiveBuffer& buffer = streamBuffers[uri]; - while( buffer.hasCompleteFrame( deflect::View::LEFT_EYE ) && - buffer.hasCompleteFrame( deflect::View::RIGHT_EYE )) + while( buffer.hasCompleteStereoFrame( )) { - frameLeft->segments = buffer.popFrame( deflect::View::LEFT_EYE ); - frameRight->segments = buffer.popFrame( deflect::View::RIGHT_EYE ); + frameLeft->segments = buffer.popFrame( View::LEFT_EYE ); + frameRight->segments = buffer.popFrame( View::RIGHT_EYE ); } assert( !frameLeft->segments.empty( )); assert( !frameRight->segments.empty( )); // receiver will request a new frame once this frame was consumed - buffer.setAllowedToSend( false, deflect::View::LEFT_EYE ); - buffer.setAllowedToSend( false, deflect::View::RIGHT_EYE ); + buffer.setAllowedToSend( false, View::LEFT_EYE ); + buffer.setAllowedToSend( false, View::RIGHT_EYE ); return std::make_pair( std::move( frameLeft ), std::move( frameRight )); } @@ -154,17 +151,16 @@ void FrameDispatcher::processFrameFinished( const QString uri, return; } - if( view == deflect::View::MONO ) + if( view == View::MONO ) { - if( buffer.isAllowedToSend( view ) && buffer.hasCompleteFrame( view )) + if( buffer.isAllowedToSend( view ) && buffer.hasCompleteMonoFrame( )) emit sendFrame( _impl->consumeLatestMonoFrame( uri )); } else { - if( buffer.isAllowedToSend( deflect::View::LEFT_EYE ) && - buffer.isAllowedToSend( deflect::View::RIGHT_EYE ) && - buffer.hasCompleteFrame( deflect::View::LEFT_EYE ) && - buffer.hasCompleteFrame( deflect::View::RIGHT_EYE )) + if( buffer.isAllowedToSend( View::LEFT_EYE ) && + buffer.isAllowedToSend( View::RIGHT_EYE ) && + buffer.hasCompleteStereoFrame( )) { const auto frames = _impl->consumeLatestStereoFrame( uri ); emit sendFrame( frames.first ); @@ -179,15 +175,14 @@ void FrameDispatcher::requestFrame( const QString uri ) return; ReceiveBuffer& buffer = _impl->streamBuffers[uri]; - buffer.setAllowedToSend( true, deflect::View::MONO ); - buffer.setAllowedToSend( true, deflect::View::LEFT_EYE ); - buffer.setAllowedToSend( true, deflect::View::RIGHT_EYE ); + buffer.setAllowedToSend( true, View::MONO ); + buffer.setAllowedToSend( true, View::LEFT_EYE ); + buffer.setAllowedToSend( true, View::RIGHT_EYE ); - if( buffer.hasCompleteFrame( deflect::View::MONO )) + if( buffer.hasCompleteMonoFrame( )) emit sendFrame( _impl->consumeLatestMonoFrame( uri )); - if( buffer.hasCompleteFrame( deflect::View::LEFT_EYE ) && - buffer.hasCompleteFrame( deflect::View::RIGHT_EYE )) + if( buffer.hasCompleteStereoFrame( )) { const auto frames = _impl->consumeLatestStereoFrame( uri ); emit sendFrame( frames.first ); diff --git a/deflect/ReceiveBuffer.cpp b/deflect/ReceiveBuffer.cpp index adb689e..5711e14 100644 --- a/deflect/ReceiveBuffer.cpp +++ b/deflect/ReceiveBuffer.cpp @@ -40,6 +40,7 @@ #include "ReceiveBuffer.h" #include +#include namespace { @@ -85,37 +86,72 @@ void ReceiveBuffer::finishFrameForSource( const size_t sourceIndex, { assert( _sourceBuffers.count( sourceIndex )); - if( _sourceBuffers[sourceIndex].getQueueSize( view ) > MAX_QUEUE_SIZE ) + auto& buffer = _sourceBuffers[sourceIndex]; + + if( buffer.getQueueSize( view ) > MAX_QUEUE_SIZE ) throw std::runtime_error( "maximum queue size exceeded" ); - _sourceBuffers[sourceIndex].push( view ); + buffer.push( view ); } -bool ReceiveBuffer::hasCompleteFrame( const View view ) const +bool ReceiveBuffer::hasCompleteMonoFrame() const { assert( !_sourceBuffers.empty( )); - const auto lastCompleteFrameIndex = _getLastCompleteFrameIndex( view ); - // Check if all sources for Stream have reached the same index for( const auto& kv : _sourceBuffers ) { const auto& buffer = kv.second; - if( buffer.getBackFrameIndex( view ) <= lastCompleteFrameIndex ) + if( buffer.getBackFrameIndex( View::MONO ) <= _lastFrameComplete ) return false; } return true; } +bool ReceiveBuffer::hasCompleteStereoFrame() const +{ + std::set leftSources; + std::set rightSources; + + for( const auto& kv : _sourceBuffers ) + { + const auto& buffer = kv.second; + if( buffer.getBackFrameIndex( View::LEFT_EYE ) > _lastFrameCompleteLeft ) + leftSources.insert( kv.first ); + if( buffer.getBackFrameIndex( View::RIGHT_EYE ) > _lastFrameCompleteRight ) + rightSources.insert( kv.first ); + } + + if( leftSources.empty() || rightSources.empty( )) + return false; + + std::set leftAndRight; + std::set_intersection( leftSources.begin(), leftSources.end(), + rightSources.begin(), rightSources.end(), + std::inserter( leftAndRight, leftAndRight.end( ))); + + // if at least one source sends both left AND right, assume all sources do. + if( !leftAndRight.empty( )) + return leftAndRight.size() == _sourceBuffers.size(); + + // otherwise, assume all streams send either left OR right. + return rightSources.size() + leftSources.size() == _sourceBuffers.size(); +} + Segments ReceiveBuffer::popFrame( const View view ) { + const auto lastCompleteFrameIndex = _getLastCompleteFrameIndex( view ); + Segments frame; for( auto& kv : _sourceBuffers ) { auto& buffer = kv.second; - const auto& segments = buffer.getSegments( view ); - frame.insert( frame.end(), segments.begin(), segments.end( )); - buffer.pop( view ); + if( buffer.getBackFrameIndex( view ) > lastCompleteFrameIndex ) + { + const auto& segments = buffer.getSegments( view ); + frame.insert( frame.end(), segments.begin(), segments.end( )); + buffer.pop( view ); + } } _incrementLastFrameComplete( view ); return frame; diff --git a/deflect/ReceiveBuffer.h b/deflect/ReceiveBuffer.h index 3442332..3b88642 100644 --- a/deflect/ReceiveBuffer.h +++ b/deflect/ReceiveBuffer.h @@ -96,8 +96,11 @@ class ReceiveBuffer DEFLECT_API void finishFrameForSource( size_t sourceIndex, deflect::View view = deflect::View::MONO ); - /** Does the Buffer have a new complete frame (from all sources) */ - DEFLECT_API bool hasCompleteFrame( View view = deflect::View::MONO ) const; + /** Does the Buffer have a new complete mono frame (from all sources) */ + DEFLECT_API bool hasCompleteMonoFrame() const; + + /** Does the Buffer have a new complete stereo frame (from all sources) */ + DEFLECT_API bool hasCompleteStereoFrame() const; /** * Get the finished frame. diff --git a/deflect/ServerWorker.cpp b/deflect/ServerWorker.cpp index 45bbd46..5da1a94 100644 --- a/deflect/ServerWorker.cpp +++ b/deflect/ServerWorker.cpp @@ -228,7 +228,7 @@ void ServerWorker::_handleMessage( const MessageHeader& messageHeader, return; } _streamId = uri; - // The version is only sent by deflect clients since v. 0.13.0 + // The version is only sent by deflect clients since v. 0.12.1 if( !byteArray.isEmpty( )) _parseClientProtocolVersion( byteArray ); emit addStreamSource( _streamId, _sourceId ); diff --git a/tests/cpp/ReceiveBufferTests.cpp b/tests/cpp/ReceiveBufferTests.cpp index 07fe247..dab60fe 100644 --- a/tests/cpp/ReceiveBufferTests.cpp +++ b/tests/cpp/ReceiveBufferTests.cpp @@ -1,5 +1,5 @@ /*********************************************************************/ -/* Copyright (c) 2013-2015, EPFL/Blue Brain Project */ +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project */ /* Raphael Dumusc */ /* All rights reserved. */ /* */ @@ -45,6 +45,12 @@ namespace ut = boost::unit_test; #include #include +inline std::ostream& operator << ( std::ostream& str, const QSize& s ) +{ + str << s.width() << 'x' << s.height(); + return str; +} + BOOST_AUTO_TEST_CASE( TestAddAndRemoveSources ) { deflect::ReceiveBuffer buffer; @@ -64,6 +70,33 @@ BOOST_AUTO_TEST_CASE( TestAddAndRemoveSources ) buffer.removeSource( 888 ); buffer.removeSource( 11981 ); BOOST_CHECK_EQUAL( buffer.getSourceCount(), 0 ); + + buffer.removeSource( 7777 ); + BOOST_CHECK_EQUAL( buffer.getSourceCount(), 0 ); +} + +BOOST_AUTO_TEST_CASE( TestAllowedToSend ) +{ + deflect::ReceiveBuffer buffer; + + BOOST_CHECK( !buffer.isAllowedToSend( deflect::View::MONO )); + BOOST_CHECK( !buffer.isAllowedToSend( deflect::View::LEFT_EYE )); + BOOST_CHECK( !buffer.isAllowedToSend( deflect::View::RIGHT_EYE )); + + buffer.setAllowedToSend( true, deflect::View::MONO ); + BOOST_CHECK( buffer.isAllowedToSend( deflect::View::MONO )); + buffer.setAllowedToSend( false, deflect::View::MONO ); + BOOST_CHECK( !buffer.isAllowedToSend( deflect::View::MONO )); + + buffer.setAllowedToSend( true, deflect::View::LEFT_EYE ); + BOOST_CHECK( buffer.isAllowedToSend( deflect::View::LEFT_EYE )); + buffer.setAllowedToSend( false, deflect::View::LEFT_EYE ); + BOOST_CHECK( !buffer.isAllowedToSend( deflect::View::LEFT_EYE )); + + buffer.setAllowedToSend( true, deflect::View::RIGHT_EYE ); + BOOST_CHECK( buffer.isAllowedToSend( deflect::View::RIGHT_EYE )); + buffer.setAllowedToSend( false, deflect::View::RIGHT_EYE ); + BOOST_CHECK( !buffer.isAllowedToSend( deflect::View::RIGHT_EYE )); } BOOST_AUTO_TEST_CASE( TestCompleteAFrame ) @@ -80,15 +113,15 @@ BOOST_AUTO_TEST_CASE( TestCompleteAFrame ) segment.parameters.height = 256; buffer.insert( segment, sourceIndex ); - BOOST_CHECK( !buffer.hasCompleteFrame( )); + BOOST_CHECK( !buffer.hasCompleteMonoFrame( )); buffer.finishFrameForSource( sourceIndex ); - BOOST_CHECK( buffer.hasCompleteFrame( )); + BOOST_CHECK( buffer.hasCompleteMonoFrame( )); deflect::Segments segments = buffer.popFrame(); BOOST_CHECK_EQUAL( segments.size(), 1 ); - BOOST_CHECK( !buffer.hasCompleteFrame( )); + BOOST_CHECK( !buffer.hasCompleteMonoFrame( )); deflect::Frame frame; frame.segments = segments; @@ -146,20 +179,18 @@ BOOST_AUTO_TEST_CASE( TestCompleteACompositeFrameSingleSource ) buffer.insert( testSegments[1], sourceIndex ); buffer.insert( testSegments[2], sourceIndex ); buffer.insert( testSegments[3], sourceIndex ); - BOOST_CHECK( !buffer.hasCompleteFrame( )); + BOOST_CHECK( !buffer.hasCompleteMonoFrame( )); buffer.finishFrameForSource( sourceIndex ); - BOOST_CHECK( buffer.hasCompleteFrame( )); + BOOST_CHECK( buffer.hasCompleteMonoFrame( )); deflect::Segments segments = buffer.popFrame(); BOOST_CHECK_EQUAL( segments.size(), 4 ); - BOOST_CHECK( !buffer.hasCompleteFrame( )); + BOOST_CHECK( !buffer.hasCompleteMonoFrame( )); deflect::Frame frame; frame.segments = segments; - const QSize frameSize = frame.computeDimensions(); - BOOST_CHECK_EQUAL( frameSize.width(), 192 ); - BOOST_CHECK_EQUAL( frameSize.height(), 768 ); + BOOST_CHECK_EQUAL( frame.computeDimensions(), QSize( 192, 768 )); } BOOST_AUTO_TEST_CASE( TestCompleteACompositeFrameMultipleSources ) @@ -178,27 +209,25 @@ BOOST_AUTO_TEST_CASE( TestCompleteACompositeFrameMultipleSources ) buffer.insert( testSegments[0], sourceIndex1 ); buffer.insert( testSegments[1], sourceIndex2 ); buffer.insert( testSegments[2], sourceIndex3 ); - BOOST_CHECK( !buffer.hasCompleteFrame( )); + BOOST_CHECK( !buffer.hasCompleteMonoFrame( )); buffer.finishFrameForSource( sourceIndex1 ); - BOOST_CHECK( !buffer.hasCompleteFrame( )); + BOOST_CHECK( !buffer.hasCompleteMonoFrame( )); buffer.finishFrameForSource( sourceIndex2 ); - BOOST_CHECK( !buffer.hasCompleteFrame( )); + BOOST_CHECK( !buffer.hasCompleteMonoFrame( )); buffer.insert( testSegments[3], sourceIndex3 ); buffer.finishFrameForSource( sourceIndex3 ); - BOOST_CHECK( buffer.hasCompleteFrame( )); + BOOST_CHECK( buffer.hasCompleteMonoFrame( )); deflect::Segments segments = buffer.popFrame(); BOOST_CHECK_EQUAL( segments.size(), 4 ); - BOOST_CHECK( !buffer.hasCompleteFrame( )); + BOOST_CHECK( !buffer.hasCompleteMonoFrame( )); deflect::Frame frame; frame.segments = segments; - const QSize frameSize = frame.computeDimensions(); - BOOST_CHECK_EQUAL( frameSize.width(), 192 ); - BOOST_CHECK_EQUAL( frameSize.height(), 768 ); + BOOST_CHECK_EQUAL( frame.computeDimensions(), QSize( 192, 768 )); } BOOST_AUTO_TEST_CASE( TestRemoveSourceWhileStreaming ) @@ -217,35 +246,33 @@ BOOST_AUTO_TEST_CASE( TestRemoveSourceWhileStreaming ) buffer.insert( testSegments[1], sourceIndex1 ); buffer.insert( testSegments[2], sourceIndex2 ); buffer.insert( testSegments[3], sourceIndex2 ); - BOOST_CHECK( !buffer.hasCompleteFrame( )); + BOOST_CHECK( !buffer.hasCompleteMonoFrame( )); buffer.finishFrameForSource(sourceIndex1); - BOOST_CHECK( !buffer.hasCompleteFrame( )); + BOOST_CHECK( !buffer.hasCompleteMonoFrame( )); buffer.finishFrameForSource( sourceIndex2 ); - BOOST_CHECK( buffer.hasCompleteFrame( )); + BOOST_CHECK( buffer.hasCompleteMonoFrame( )); deflect::Segments segments = buffer.popFrame(); BOOST_CHECK_EQUAL( segments.size(), 4 ); - BOOST_CHECK( !buffer.hasCompleteFrame( )); + BOOST_CHECK( !buffer.hasCompleteMonoFrame( )); // Second frame - 1 source buffer.removeSource( sourceIndex2 ); buffer.insert( testSegments[0], sourceIndex1 ); buffer.insert( testSegments[1], sourceIndex1 ); - BOOST_CHECK( !buffer.hasCompleteFrame( )); + BOOST_CHECK( !buffer.hasCompleteMonoFrame( )); buffer.finishFrameForSource( sourceIndex1 ); - BOOST_CHECK( buffer.hasCompleteFrame( )); + BOOST_CHECK( buffer.hasCompleteMonoFrame( )); segments = buffer.popFrame(); BOOST_CHECK_EQUAL( segments.size(), 2 ); - BOOST_CHECK( !buffer.hasCompleteFrame( )); + BOOST_CHECK( !buffer.hasCompleteMonoFrame( )); deflect::Frame frame; frame.segments = segments; - const QSize frameSize = frame.computeDimensions(); - BOOST_CHECK_EQUAL( frameSize.width(), 192 ); - BOOST_CHECK_EQUAL( frameSize.height(), 256 ); + BOOST_CHECK_EQUAL( frame.computeDimensions(), QSize( 192, 256 )); } BOOST_AUTO_TEST_CASE( TestOneOfTwoSourceStopsSendingSegments ) @@ -264,16 +291,16 @@ BOOST_AUTO_TEST_CASE( TestOneOfTwoSourceStopsSendingSegments ) buffer.insert( testSegments[1], sourceIndex1 ); buffer.insert( testSegments[2], sourceIndex2 ); buffer.insert( testSegments[3], sourceIndex2 ); - BOOST_CHECK( !buffer.hasCompleteFrame( )); + BOOST_CHECK( !buffer.hasCompleteMonoFrame( )); buffer.finishFrameForSource(sourceIndex1); - BOOST_CHECK( !buffer.hasCompleteFrame( )); + BOOST_CHECK( !buffer.hasCompleteMonoFrame( )); buffer.finishFrameForSource( sourceIndex2 ); - BOOST_CHECK( buffer.hasCompleteFrame( )); + BOOST_CHECK( buffer.hasCompleteMonoFrame( )); deflect::Segments segments = buffer.popFrame(); BOOST_CHECK_EQUAL( segments.size(), 4 ); - BOOST_CHECK( !buffer.hasCompleteFrame( )); + BOOST_CHECK( !buffer.hasCompleteMonoFrame( )); // Next frames - one source stops sending segments for( int i = 0; i < 150; ++i ) @@ -281,7 +308,7 @@ BOOST_AUTO_TEST_CASE( TestOneOfTwoSourceStopsSendingSegments ) buffer.insert( testSegments[0], sourceIndex1 ); buffer.insert( testSegments[1], sourceIndex1 ); BOOST_REQUIRE_NO_THROW( buffer.finishFrameForSource( sourceIndex1 )); - BOOST_REQUIRE( !buffer.hasCompleteFrame( )); + BOOST_REQUIRE( !buffer.hasCompleteMonoFrame( )); } // Test buffer exceeds maximum allowed size buffer.insert( testSegments[0], sourceIndex1 ); @@ -289,3 +316,140 @@ BOOST_AUTO_TEST_CASE( TestOneOfTwoSourceStopsSendingSegments ) BOOST_CHECK_THROW( buffer.finishFrameForSource( sourceIndex1 ), std::runtime_error ); } + +void _insert( deflect::ReceiveBuffer& buffer, const size_t sourceIndex, + const deflect::Segments& frame, const deflect::View view ) +{ + for( const auto& segment : frame ) + buffer.insert( segment, sourceIndex, view ); + buffer.finishFrameForSource( sourceIndex, view ); +} + +void _testStereoBuffer( deflect::ReceiveBuffer& buffer ) +{ + const auto leftSegments = buffer.popFrame( deflect::View::LEFT_EYE ); + BOOST_CHECK_EQUAL( leftSegments.size(), 4 ); + BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); + + const auto rightSegments = buffer.popFrame( deflect::View::RIGHT_EYE ); + BOOST_CHECK_EQUAL( rightSegments.size(), 4 ); + BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); + + deflect::Frame frame; + frame.segments = leftSegments; + BOOST_CHECK_EQUAL( frame.computeDimensions(), QSize( 192, 768 )); + frame.segments = rightSegments; + BOOST_CHECK_EQUAL( frame.computeDimensions(), QSize( 192, 768 )); +} + +BOOST_AUTO_TEST_CASE( TestStereoOneSource ) +{ + const size_t sourceIndex = 46; + + deflect::ReceiveBuffer buffer; + buffer.addSource( sourceIndex ); + + deflect::Segments testSegments = generateTestSegments(); + + _insert( buffer, sourceIndex, testSegments, deflect::View::LEFT_EYE ); + BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); + + _insert( buffer, sourceIndex, testSegments, deflect::View::RIGHT_EYE ); + BOOST_CHECK( buffer.hasCompleteStereoFrame( )); + + _testStereoBuffer( buffer ); +} + +BOOST_AUTO_TEST_CASE( TestStereoTwoSourcesScreenSpaceSplit ) +{ + const size_t sourceIndex1 = 46; + const size_t sourceIndex2 = 819; + + deflect::ReceiveBuffer buffer; + buffer.addSource( sourceIndex1 ); + buffer.addSource( sourceIndex2 ); + + const auto testSegments = generateTestSegments(); + const auto segmentsScreen1 = deflect::Segments{ testSegments[0], + testSegments[1] }; + const auto segmentsScreen2 = deflect::Segments{ testSegments[2], + testSegments[3] }; + + _insert( buffer, sourceIndex1, segmentsScreen1, deflect::View::LEFT_EYE ); + BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); + _insert( buffer, sourceIndex1, segmentsScreen1, deflect::View::RIGHT_EYE ); + BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); + + _insert( buffer, sourceIndex2, segmentsScreen2, deflect::View::LEFT_EYE ); + BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); + _insert( buffer, sourceIndex2, segmentsScreen2, deflect::View::RIGHT_EYE ); + BOOST_CHECK( buffer.hasCompleteStereoFrame( )); + + _testStereoBuffer( buffer ); +} + +BOOST_AUTO_TEST_CASE( TestStereoTwoSourcesStereoSplit ) +{ + const size_t sourceIndex1 = 46; + const size_t sourceIndex2 = 819; + + deflect::ReceiveBuffer buffer; + buffer.addSource( sourceIndex1 ); + buffer.addSource( sourceIndex2 ); + + const auto testSegments = generateTestSegments(); + + _insert( buffer, sourceIndex1, testSegments, deflect::View::LEFT_EYE ); + BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); + _insert( buffer, sourceIndex2, testSegments, deflect::View::RIGHT_EYE ); + BOOST_CHECK( buffer.hasCompleteStereoFrame( )); + + _testStereoBuffer( buffer ); +} + +BOOST_AUTO_TEST_CASE( TestStereoFourSourcesScreenSpaceAndStereoSplit ) +{ + const size_t sourceIndex1 = 46; + const size_t sourceIndex2 = 819; + const size_t sourceIndex3 = 489; + const size_t sourceIndex4 = 113; + + deflect::ReceiveBuffer buffer; + buffer.addSource( sourceIndex1 ); + buffer.addSource( sourceIndex2 ); + buffer.addSource( sourceIndex3 ); + buffer.addSource( sourceIndex4 ); + + const auto testSegments = generateTestSegments(); + const auto segmentsScreen1 = deflect::Segments{ testSegments[0], + testSegments[1] }; + const auto segmentsScreen2 = deflect::Segments{ testSegments[2], + testSegments[3] }; + + _insert( buffer, sourceIndex1, segmentsScreen1, deflect::View::LEFT_EYE ); + BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); + _insert( buffer, sourceIndex2, segmentsScreen1, deflect::View::RIGHT_EYE ); + BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); + _insert( buffer, sourceIndex3, segmentsScreen2, deflect::View::LEFT_EYE ); + BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); + _insert( buffer, sourceIndex4, segmentsScreen2, deflect::View::RIGHT_EYE ); + BOOST_CHECK( buffer.hasCompleteStereoFrame( )); + + _testStereoBuffer( buffer ); + + // Random insertion order + _insert( buffer, sourceIndex1, segmentsScreen1, deflect::View::LEFT_EYE ); + BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); + _insert( buffer, sourceIndex3, segmentsScreen2, deflect::View::LEFT_EYE ); + BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); + _insert( buffer, sourceIndex2, segmentsScreen1, deflect::View::RIGHT_EYE ); + BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); + _insert( buffer, sourceIndex1, segmentsScreen1, deflect::View::LEFT_EYE ); + BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); + _insert( buffer, sourceIndex2, segmentsScreen1, deflect::View::RIGHT_EYE ); + BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); + _insert( buffer, sourceIndex4, segmentsScreen2, deflect::View::RIGHT_EYE ); + BOOST_CHECK( buffer.hasCompleteStereoFrame( )); + + _testStereoBuffer( buffer ); +} From 81e18870dd588be3b5f81e2b5c6037881b099536 Mon Sep 17 00:00:00 2001 From: Raphael Dumusc Date: Mon, 6 Feb 2017 13:55:28 +0100 Subject: [PATCH 5/7] Added stereo streaming documentation --- README.md | 7 +- doc/StereoStreaming.md | 68 +++ doc/stereo.png | Bin 0 -> 38333 bytes doc/stereo.svg | 1260 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 1332 insertions(+), 3 deletions(-) create mode 100644 doc/StereoStreaming.md create mode 100644 doc/stereo.png create mode 100644 doc/stereo.svg diff --git a/README.md b/README.md index 0e37538..399f01b 100644 --- a/README.md +++ b/README.md @@ -2,15 +2,16 @@ Welcome to Deflect, a C++ library for streaming pixels to other Deflect-based applications, for example [Tide](https://github.com/BlueBrain/Tide). -Deflect offers a stable API marked with version 1.5 (for the client part). +Deflect offers a stable API marked with version 1.6 (for the client part). ## Features Deflect provides the following functionality: * Stream pixels to a remote Server from one or multiple sources -* Register for receiving events from the Server -* Receive keyboard, mouse and multi-point touch gestures from the Server +* Stream stereo images from a distributed 3D application +* Receive input events from the Server and send data to it +* Transmitted events include keyboard, mouse and multi-point touch gestures * Compressed or uncompressed streaming * Fast multi-threaded JPEG compression (using libjpeg-turbo) diff --git a/doc/StereoStreaming.md b/doc/StereoStreaming.md new file mode 100644 index 0000000..cc7a4f0 --- /dev/null +++ b/doc/StereoStreaming.md @@ -0,0 +1,68 @@ +Stereo Streaming +============ + +This document describes the stereo streaming support introduced in Deflect 0.13. + +## Requirements + +* Simple extension of the monoscopic Stream API +* No network protocol changes that break current Deflect clients or servers +* Support both screen space decompostion (sort-first) and left-right stereo + decomposition modes for distributed rendering. + +![Stereo streaming overview](stereo.png) + +## API + +New view enum in deflect/types.h: + + enum class View : std::int8_t { MONO, LEFT_EYE, RIGHT_EYE }; + +On the client side, no changes to the Stream API. The ImageWrapper takes an +additional View parameter. + +On the server side, no changes to the Server API (except some cleanups). Each +Frame dispatched now contains the View information. + +## Protocol + +The Stream send an additional View information message before each image +payload. This message is silently ignored by older Servers. + +## Examples + +Example of a stereo 3D client application: + + deflect::Stream stream( ... ); + + /** ...synchronize start with other render clients (network barrier)... */ + + renderLoop() + { + /** ...render left image... */ + + deflect::ImageWrapper leftImage( data, width, height, deflect::RGBA ); + leftImage.view = deflect::View::LEFT_EYE; + deflectStream->send( leftImage ) && deflectStream->finishFrame(); + + /** ...render right image... */ + + deflect::ImageWrapper rightImage( data, width, height, deflect::RGBA ); + rightImage.view = deflect::View::RIGHT_EYE; + deflectStream->send( rightImage ) && deflectStream->finishFrame(); + + /** ...synchronize with other render clients (network barrier)... */ + } + +For a more complete example, please refer to the SimpleStreamer application +source code. + +## Issues + +### 1: Should clients be notified if they attempt to stream stereo content to a server which can only handle monoscopic content? + +_Resolution: Not yet:_ +For legacy implementation reasons, it is not possible to +send information from the Server to the Stream about the capabilities of the +server (such as stereo support) without rejecting all clients based on Deflect +< 0.12.1. A transition period is required before implementing this feature. diff --git a/doc/stereo.png b/doc/stereo.png new file mode 100644 index 0000000000000000000000000000000000000000..0d93b2a0913eb7e2c7af71a5702dbaf46b0cb4a1 GIT binary patch literal 38333 zcmc$GcRZK>`|oYkUb)@BeIoBWo0B1B9!b+W@cp1?7h9u_5S`I zzdz3T<2=qE=W))b2l>3a-Q#*)<9S`L=kt!%zo|h&$ViAF2#J=a>MaDp#DK3Pd|ddX zJH($8{=&7^)=))I=s%e?A78*H1n!!qUI;=&h5o`o-oB@Y5AnRUbk*=?FmZ`#nf2y_ z+7N^t(Na}1^82&)_pwRb3JP!Q&`~CZc0lkRweqXASi7}dn$`rXR(uxASAYEY^2$(d$t&aZ{h^n_;di$r zc6`jJs3^^q{1R+cRaIgP_|{CW`2Xj}|NYfHXZBY|+GWhW0GB>xPwzl@kXzEgv6Jb9_BhysK>ur@?yMlmsJR?w_ueo*W-*1-JF|d{I8R zZMq+P>xomW(({Ks@1;#jtt0jxCTTuh?0%PR)fCw`GNR$+#AjaPAMsAoZMOTJBoeZ> z=WT0erxmPWXh?-R3`QM}iKaC-t0Ic4!$k&#w~G=cD|FvUu$=|LbLyAb>#mGQOG`(^ z#kDMSy{2GeBSY|sh*SipUupY{^X^R&Y7Z(_IDZEo`{^7)Ryn+N_r;SNU zP31kG48g#09{b+( z`^#(QDk@ly9zB9nMtu1qXkHf-$D^0d&dG_1Z0#&)>%5nuINY3Sdn^9XHc#kf+ngdf z_ODb?yr7_<-0OKe8~5z`QPEr&+#i*dm9djFxpj}QaEZy&WgqV}jNjmX^?M$ko+q<)r@3G5S6@D7Q#jiS@5@rb z!wAVNDBx37SASK2`?uQnbx;sIQ18XY-pX%f z#cpzJ3}kEQjzvRT8=lXVi`LRvLc1Y1I&hGf<32!uXU)($(^4Yo>2t;ADF9&AWFti@#F+cIFB4^fE}3G;{jS z3oY+;r;2_YU`u%Z2leSp1p7+<>PWG+0-LD4PHD!=p~1n-yvqT1q9e)J1kYs!1xdjG z1?4tcdU|b>H31liy}kW%e>PquDGOn#2{AHS8yN8E!ZjS11Z`oYe&cU-Qd-)1|2V76 zBjRFDW=y9gbA+>>#I^F1nrT|$DE=jQvP<@Fs6LT?`*5!?2}Yt?S@^72)_o9TP20{1Fj z{!`1U8kt&1dHa?T)`aiefAZ(!3}Tlx6r^dI&WQ=LljEsil)&L34BSiPI86G?LaNXuL_JIx!B?o#XT_>VQm?YWxEndk@DSmzqbpiqm(PK6_RRhhLGu3Y+hd8H5B(9zikaA-E>(7 z$aoFEKk9KLX-?12KRw=E%7zr+@H>xWZEdaZgBsED@^bs$Ii&0c$nd3)ckNdW)nn-~ zXi{buMj)TqEcK+d{3y!K=JG#SJMI{Fo~pg?U-NVN3X&+PdUk#|)ig9j-2`T-fMZGg@ksI&>W#tj~-{Mn)#Wzf`cRO{v!P z?~g3K40$P^f7I4CHrq=X!L)CUIxh6CCAcN(nSV7E=%P#btk7#|-$kWgyVMtFXDNEaI$OUU8{N#m)c+vLkGYo(2ijf}Hu$dF;* zzkf#*^&zQ#$j*+LxGW}SbaHsN()IoaMzb1!8ALTmqvdxR`+DiEm?I`q<-gD8 z3hOb8;(`p_bT%_HQ}yK7Gb;%n(Kj|05W~y*td2QZC#IA!^4V1MY^#Y3*||I~VE*&X zdB!}hO2`MvlIMgMo2Uyict8znYfebW)BpZG@6XoAHpo_wa1B`e#Zc}vL`cReoA~{- zzwCkD=eyqunlb7}M(D!WsaI_ppAux8-NWHIjiVQt&91!amzr5lg{=|K%v&>`q;Qsz zoQ&z_=7v1pCHwX3S7&*vzrTM(?W>g4al0&jHaxZ+00c@x=dTxJC#$8+O{uD((bU^Z7I?CUktnJ4 z1*%+1g}kDUPONt}qnw-^F|*WGc4c?)ID6t@5Nd$G$;rx!qegIjVIcxatY1KYw#Az+ ze`R)k@9z_52J|OCulxMdeC^-U)6-g9chUKu9CZp&@hpx>lJ@S>ITWTU@0DA*EH5;< z!=j@J{P&haU-0PN$S`A*^M4A(oF+iB;p_a&498y%eTZDD%R2S>2HEmQGiz)7EH{F8 z7fIrorEe9WQtRv$!Q7;5^87W_mJSY47w4z1j{^sP|Gv66{I#!7|4}045k!1eVc}I0 zR~))ED-=&8lI&=hrxTi#l(e@RLnqYvs}xXAeQPT&Le9)g?CX0-s1O;}bOQfA`t9Ke zkHQWel|RZ?+qymkW1x#o`?a8&yL*Xuj?(@6HD((RDqa8PWMoKtMY3dL6O)i^@AfKS z9r=2390XhAvmg7h=jT)3y(i1Tp=2Mg3o8VuBRQzXq5p%j(ANb4DJgm=@Iw2Wr>CcL zkWwyP$d#J#r@6nX)3#`!2;13OubjG}Kt@d5oF?NtDt*HeO4I;B_Jf}H4P#{w#!%55 zA+{Op_qt4oK|=1I`;6U)NHeH*qX7w2G7bYrQV` zdm@{4)0NI+lob^f3yheIVh-%*{u5Om16}NwC|FpEUY_UQNa1^<5L~;r3YY1cSXAB_ zMPKX2a(yV)rR(<&c6Rv6#Wqg^UUY>$-d)sE@b&Qclz5JXE<>U_7%v?Lj2gx_MoY7e zgm4A!me&i_oni9u@JLBZr<}IRLjEx+vyb!sFbW7n*m+bq=i^7Z)kD#1*Mz+me<{1< zI=3dU3H=?~1~-XdRrs$$Gn=4rp;&bH8#?!9W@aYb%+rQ43&@DDTf*&6`KP_l_g?*{ zMi{RpCnpo$=e#0hZf>rrtBZq-{`vF4W3IL3;x7PC?Ml1AgEfY;^K)$<;=1z#riqD( z>BYs{4$`AL%s!^&4&W3j&q%Z2u|@&jYBVX@+H&76HYWjq{n@lkU9cyptLyX~QOct` zS_i=eiEUT4Gppc=C00$i&Z8w5Q2vx+Xay{-tU{saa&mED^`^@;rChy}Bjkx>78fVL zLUyhNZ9$=WQ7h;6_lM6LX|K!RI12u!$^3Lg$iv~!O5Jb8b1mq8QtJ;?-u#rGZ#i0G zHBj=^WxT>#NaHGz&+y=2^Tc*wpSknK7BQjI8y1!P!M{uKXj+%iv=_Y5pY@pG!A~f!*bo*Awq!Z z>|!pn++|^N(_((}_TbkB8u05b+!w@m%N-~HDL_!hUebP*8H~Cpsi=sAif?3WoK;jr zA}T8Cy)nTcYX7SqLdRo!RxNev`fYAXnje!tgU(Mp-#xwUPCZ>%T+26BL9ta;Qnt3XijadB7Z<_Bnx-4V`-?3U@(T*S zI(YsAyE_cz5TN6JiinU<`QyzrllOWQ%i*`bnDHV<^BR_x>_nIFAlh(bWU6~Bco3*k zDkeYuO4*Y%O{8%20Wn*DlR>h;uh70`A|^dl_Mn&3g^`-Nx$z0E2BtiOPbeKVR^7->;eE>YN0SB=-$)a$qT);td6a`lw*v1IbRA2Ar z-|AXd9#Rg1pv&J((=%9}rbg<+?*`=9p>|hl(KS{DS;B6|3l0vZXJUe( z!oiD5$ch@!s(I`#>LQAplQkZIvuX7Sj*gGB0Im+y)6>x4P*G9Qs*~>h8zO>EGT*f9 z^2THh(bcP09r35%hxE=>V``ctW#6+vs ziY{JST3W&q5|J@6L`1i4yrvxTc&C}GZ@YzqUL8y+RyOdR4!@QC!gtq@RQ)tl${*s#; z0T0mF&=A^^Y=t9)wYNFN1~tLXpcZMn9*yMW<;^~03sSVVzl>PIq5~Y~BFg552}s0i zA&YBBdnkI{d3gy=PEOo?eIF=EQ7Z1)!JucmiA-U-HI4}h!6jj|1kdG<4RZtp+ZxM& zg|zqdr~_Vw5T;;aih;yB3nqtYQ%bFIqHQnqI36c^jz~QKc7>_~EJ!jm?^HP(02R5f zjfxv*xc(V;6N7w@XKER0X{ol4Ma9E%XMm~*~pF{#(NoGQfPGe<0&7Gea(eN zGB;aG;-}xX^2sqo1UqH4{E314R7i2{$&Hh9aKy4iwt8ud93vQOi1=+;GAjfzBj|yS zIVQWK)UVesUNnV~s~}07=++z{Om0l<;FtSMp`oGq_bLTrRPYH2lY?BXU^LK94h|Q~ zEdAA_@^4*mo%huv=!+UhN9h$Ot5u>nLS7odjA?Csor0bopWRKey}n*?@RQ-Q76bj% z)KsX#Ek9fU9PaIqBelBvtq@{`qb{Bhf{ys^j~*=whGjIsZEoGVb*m5BJV&IYyu2Bj z=WZl(`h8m3_Fvzr2IvS@R#w=4LK<@0YGVt|gpRJgyBh}?EV@TG6?n)(q{&R6Zx6$b zql?qcm?sK?f;Uf&!*GZ!ZEcN~m!Q!?2Px4N75kp|rVjZdFb))b(8E8x2?L&2M9J-+ zpkoKb-ZVbW1o^Gx>sMuPoy_d)@Yq;l+29Liz@x8gkFXSBXg34%h5Oim`}EkqM@YOl zPK0VncWe%gBt53B4m?Cwv5;96O!&?LrqW~18#U1FynoLOvs)NpK^L<*@m*8`I(CSR zQa|)im%I0sz`e-LIut7)SS3wOe8|_&AT4o-i_=(_(nGvJg;$5+9LyoeZ!&YiGz448 zk^$z!FUfdtDgRGvYtmW%APpAkc!`Z!{>hsneF9ZZilCDf9(qQ`uX1DQ&TbWw(&Yj!!{~lWqsi}j8Jdz4)Op8-lBe?Sm#;^$*3s6MeFDa#Ot ze*6Du{-;BtywDSDzUUu6{BtjYQ^(Xw%(=8j1MPU|;ubgO5?-aD(Haig+QQOh?x&Cq=s>qOp z{QGsZv@UnAyyj3$|05|TrDLxga(Wh_Z)luSeLzY?);6&n@9;P0(=WEd-)=Ei`LTb! zlOm`XCK*j%uw8f6Gpq8b8Au3!jfFNo^?!iP|0Q%|OfCLOaof($er8~-3tx(|-kQ9F zL5ZB4F^EnB^UPyy&6M^jJ?8Rm^3>(IvkrD_W`g?T7fPDCH&ZTjxG9y$v1y|JCCU^ zVIfEsPcK8==e!`_%DS-?iihd`?6x-c3GQt9p`%%ydv-fRZ9&0xViFfi>jCDxJOK{c z)tRBfu7w~ogWhc6tSRp0+e2!H7#7*RiIc?Gs+-X<32hyjyIb;sgh6yd?4nl-HQkGE z7p0e@{w=?$U9DT7!-RR z+Jr3XC56c+KkChZA7v112(Z(x-dWx@y?dZ;J5Q7Iu}uXoG4D^W{N| zixrr#wY5Uukm}B#?cw}>S_ui2r1xipNlACM{|%|-kN;Nkt>+W!cSe})?Tf6O{xF}M_}KnC8XEd$Y1L0b zOUnY&*NX;4_g}m8@sC+;*x4gqB&9}Am*^j*cgf5pJdb&?GJV1E0@$c3AO#G5o z!=0So3QM>W8vcGIEKw2h<41+rlm#y4E$#%1+WP)73@LvJq`}GxDuckD7Y&WLQSl_X zy}bZf&&)MU1`7kM@d7gaM3p<%g$Abk!Lq4r@ZN`Zmu73ZH*YJResUzDXZ)k6rY7?4 zlRLYniq5ZnJzUJ2d^w$$@i7}4n{tKDDp@B`({q1DM-T41yAi^sM?{fE5PL}lYsk~5 z&!V3Hc)q->pOf1{M~JO0;X0k89EmKXB&CPOT)ZZC=SILy$l2&D<|VFQ&l{k-C@@}C z;-=&^jvE~MroeNaJ#wArmbd_7pEI_QX=^5M+4rR{@o*6o8A# zm((jX!4;Nn?(*i1m`WU@OIEf`>jxSYM?r=&i(>NGwWp?OMtn;a?U8sNUV<0AVVm4jZ``}7Rr5#Jv_gue`afF%@^ z6XUC@WTK+Khr&|L&)d?na`cU;RbCKd?`=l}5SlLuRn6K@Hp}bMeu0}Htm|SIBBy5N z_>X1z?l;h!-ei@E80jzGo;RPNL!Ct^{OZ1!efQhR4aw5tuUMgQsWE%TI}KZdO*%EKpKuD|nw=~+ z*Zy1$|2C;h>;&7XUbMQn`2|it$3?TGu|g@eZ)k<;g($?QlhmgsTIS?^tn($JD7alj z0xR{6XK(e;{Al;~nzmlTv!Wub z*tmD4CZ3e|U(14D-sF37ej@1URYrt^8Hg3QR$YSU!?YElI0>U=;&c(3AUeS(C8*lE zS$l79`k2eRp_a((92?CWqoX{1c9XSQ)?J?KR$0GoY2!GiyVgzX?l2;RLT6S);^Mm4 zI09jjk*q|QG_0yU?9Oe@1}bI+c%-4elm*^sKbQ78JeEp{wwbd=flL1Bq7 zAw9m+&_K)fT#w(u8pf>`zfG8NGH+ROr3>CySJo>QKP?W!4GYEk$R#2|4_irH@5=Ab ze-Z}e}L#^+VA=Z=gZvx)l1?*QG?97h>HrK7V zKXO=)HJSf67eH}gitb5Zpj^3w!L#!_p{JWkTqxI~9{FG7`mTC>;!baxCd?GQy@|uZ zO{pH|eEJlMfrZ{2Iae;L+qDqm-gLQ8-qA8P&ehKhu0yG-=UpSdVuU(B^op)0uP4`! z2cY)rmrhoZKxAqvE;JEIY27?jrVm`rNy2fzsTJM5eAGAarJ&%k%bRyyJmeAh>o|gV z)Uu+#HtX{89$QTh6&f8_;QXswYa+&=>Fg`H?lf33pBh2@@b4eWnt;lX+FHTMd+pmb z?ynC&aJXMFfXuS3m`2aVDeEr^>Cf*NS(0iB@*G~n ze!0#Nw4#A~WP7Xjfj`W!)R%fBV0Y91&=!Rw#JJhRE%h>=okQ`P-)pSVV46bI;=WZV zj^N7Nee$O_KD`Ap!pY&L(D~is0zqP361xvqwmk^^P8S7``u6-~|3LzR{=OmckhgcY zN8a{!cdI>mB#520BT3uYp{1d&K6xG!6{QG0ZCAP6w{Ij_A3m51>aqU#@fL>g8m1e) z)GX(ej*h%g`|CfrlKk{%Bv3I_i4wO977C@)Gv|~My?5ec`t5<%sSqNTe~*sS=weRh z4rhZxZ#pIdGfhI5b-m&Z)Atv@F0NLeO9}|!^!E#4Am6&XDPS`|hlC{S6UVES+INXB zquxlm5e622I4RbN$M#+uqPR(vW;<*0sAvplQvi>HGFte_(JZIJRHYk5?ZqFA%ajA`F5^e>1bBU?s znRj%__8Cn(n@f#~%#BaJmwK#!P&53k_{}Mg?QCp`o@oTUDn%w#Sfuocg;+TlK6`gT zj}zw(&Q-l1KOV0aefbjUAIHxzOr5Qs#&~Vl?~K~?&v8js?xjJatHGl`rw2;!-u2H| z|9EMocdhR0NU)8Kg_c6GqdlQ+DZP)=pdwe7|Fyrqx&>XW?UGB_Mqy_U<6%PTcYY#n z{!>#`9k*B-lg)zWv#f`8rZw++v*Llp@mk$``gU^m5g2M)IXE0`bj%A}yY{k7OTLoN zIB+rs`- z7#ArsfhE3viE!qdrr6`cFrVch;jt=sk?OPV3G@z|Ptehj7uQ=Rx2IIl_U+3fMJ zV1*GA^DZc)a&tGs&I)!;ZU^c`$Lr68w?()xWMlN-^dmGhP+mRjnCmaDKFmtsfZCyB z!K`|%Z<|=Z=xwN(Lh8$w0!x~0?#Ad)LNUfIL2RRgS&_9m8oJrt6X9@Km5?yLBio%> zHOyOvcBdUVpTXu+HVX@g8Z$wAz^{d`C_X>b!XySU#tkOibcFm%t9I3*SvoDylO z*GSianYpEnA8etnG(-OwF=ApLSVWLLG2;O&`dl`(xvh=s z`gH<2;@1;&M3~$vW}LPh=73j-ufvGQ~8?* zls}m>{kSk_YD~ys;o`A{+h2bO;&1pEWrAtv*D$?+M7S7+KJ2)vd(Vp-E|u?Xu=%%H zQK^xRIO4H&mz!az5aDwL%KDZtmu|cIn&M)#I2az+GAmvpz`*gPuwAynXXh<7 z30J!^lZsu;%cqI2pM-+kS(nQ|V2QGWf5gI4q0M><_rjfd#f8tG4wHMWu2YP%p1hYF z=5P-vndNUsB_S)9SSvn$Ef=qk4h7m5GXXpGPv&|$oJc=w{41Y)M=0#9u38mLixS93 zUuh=_U;e9!sH9zs@ViND3&XkilyBKao_Vx0a*TwS z^c5S?L=rq-NcUkWcaDNMG-U)8IUyeA2J~nGOfy zMLVF5DA3sreLMk%>Ci+nRH=DKx7CzPmFug!FyNFSU+XLJJ-I{Xe*#oV%u1lgtUV)6 zV#4QbGsTAA{g=w267p5{j+ZsN3erzkYl%Q)i>f3q9V|90!u{M*2%WES*t(UdT+XY3 z`I5g~bPsMSg>s5BKT3$uj7O1S{2dh*15AM(xI;#$XzrW5 z##vvYif*iwoMOq$7b1jyHYQ6o@C`Z$>^^Mv$?F>%MzG^cJ@({!wEi|`aIlg;7PGUn zv-x;8_?!vT?{XITo{u*M5*_WawR0C(4MwzT7z>MwjZIBBrKL%T)J_w*zB=Q$cxx?D zfWhwp;9gBP*j4?wcp7~1y>s?T;?+AnR@3$Nqh)O*2s$4R>lT^P;|TA!Q3y-qlOk0< zSIJ#n=eBniYc= z7&5*+t=n^}h_Fj%+0&;Z?vGgG46`Tcns7js3#E6-OSt%$+Fc5Ja9rt!07Tiah4VZxSSXks( zJ|EnTSg3xbWO$DXruK~yO;Irc@$>T|{E5uheL9)WPsZxo9`Il*h2*`u{GL{)F-zY~ z%x&sD&Mo#<+@bA})^7LIIbt7iaZT>;`1qp{>*K-do;ysmnxr_mCykS}1hR4?%A6e$ zZwvVHE~GFI<1?BwS>;SKZ;=zEcvG~%I1|VuuY-#27}G`OMP9p?|M{0Q3=cT3ie4>t z=cMrVDOBMfel=!u+t@g?qQX9D^J5OTR&kb^s%j<>0qp*yi#rX5ntbLp{00NB#?=uK zO+D1%pOU)!q6|vR_mCv=Zgy z=+rpaM{d2}xPBnEL-W5NjEE)&H7HAiwz=#q0X|QEEAynfmVb`bSu?94YI}65OpeWc zsYX3pou$9#R5LXID7kWSF6EBDbJ`DkGf+gqXCj3Y7n{8b#j58gadsbYzkipO^0ehS zSTjxd{8tA-w6=a53%2`=5;T*dAY+?M33Aiily^V% zN%tODdgk!FCEqX`m(g_i#Y)=;EuS&^v9o^zair=c*{6uyWPH}5?g+I&p8z!OB_e2~g(F;?}-6J{jy z;i^PcP&*z`+GUl`4HsSH>`gz$d?R|}njD6{(;)1E8#ohxd1e|TRLmL*-Y?AC+N=)K zEKk)DC@b#>wk zd~au))70(4flFM{lB*3{k%lTNYj-P{LuKUpO1@l*X(nf9(-F`q*HLeg3v+6_cFuf)_;4B!f1aGlJ}pHm&Vo{>kXa?(aWZ~4_u z9vraN8m9FET9&Z2wa>oizVfGT(%-#T_-oHAq=$BdY+nxC)SynNxqE4$t6c9rMC;M1p11=cVz zct7@)#9mz;390rCpI($8?ai373+>c2Go!ZeeN?~P|95+D1q)Fu!zJ;sN}!VsvIzZW zT=-R!7-OeMcVu|$y9fULi{CiN)`1Iug=<>FCGD>IPll7rzdtrSn-)+JjiL@AW zmoQEQj@W$ZV$Oe8wt!)!8N6#lWz!; zz-H0*!^dVDsw{0j{nxUx>5YqfS;;UeXn(E-Aw)`LZXQEPD6?B_+VZRx8=weX{n$mET@}1*tjRN@7x%zz@H-|PNTQZiNXW+?L%P*8S|x9 z@OD5^<7{`#2rZkx)Q)cS4qmOxCB1_BK+-KQ3o<=v-wk-tWSsDrz zqvQh~;Q>N%KRuAnE!MqxQ$kkOOvExo0EjsRNPa;f-fQWes@VmS>ejep$YSgW`s`r5 z5DLR!Yy{X@mYb6}H)?EGc9#-_UClC-sZe))TqCk8E`dGSFe(N8-G`mZJ>N8f2&I=sznHi6Jr=fnHUm3 z82N4z&`txz2ALN!y{`1>6@Rxv0KHTRLs+;Tu<2S961-%-!*#e_bakCg%dwxCjbyzw zH%Ha@?@LWgv|b%boaFj3iF{3k!(U19(YiV-u zG@dOA$ui;od(Cl88vUPgS^`nA_JWHI>8t zBOa~J+cVdWE&i3{8AnGgLf_yFxl=rw);+%RF}>HrMIOJGuDt3!6N1z$e+70Ca*T#{ zUNKMl`-{T-1p@(6GQ`Q;)~s4p<3w+Q?%8gQ7xj8T`4U#>{H=@Rrl5P3*^yc^mzpwU zzvq%4iUgNsK<%BKRSQN!wD$i!es{jMh3?;^<)mW!X*Q~EghxXBS_IfSLQ zD=~Bl3R2sf?Ye9?&$uG`i*)ZzNO@zQ|M12^xG%R_jaM*u)?IF2=nAD5%U4V44%%Rq z!FYy&^>`(oBc$_|j}4Zy{C5Jc`-_UY_elW4py45lj6K(-OG;lhv>Pf(7WB@37v}M# z9o;FWs3g49Vr^2y7+>W3tc&?Ul(-ms(tHuY*T=hJ+}x@iLxGRauQQfCkg*)hhkB>j`8or=~+071yu%$rX3<2^-x($ms5MhEAVpZos7$1{_0 z$JhP*89{{1pV~mK0AQ}am|($~1jV}>^f{s4Wo+O-{BOhRFqmGwRHnLm)oh}gjSG($ z8?m*`Y3zC^63=>{tL!16>|;-$iBzX%RyFIiw3ID!^-Y8-5@YF?J}N;*GiNHOs~;z~bVvW>&!rSvclIkdIipLfuubwram^ZXf^xL&nuP7OU{SyX{~9!E2&6HrKHA?HH$pk$^BcgYklc z>gFJ-)#137ngBYxe^Ojr85qc2U&lBuNiv&(v*`Cn8aLeBizllV1IgFlien+oZ5mcx zuU7DhCj<<${X05<9_0~U*A!lN(*}T+vn()YX(R+(+^-DY`w?UAtSQTU&2l=0)d3znkstFV|CbcZ$pmMW&8kG`~8p3Pt?#w_5J%bLK*qmfM=k^ z!Q*Edju^=eJPH~(L@v?-*XGIyJ36*U%N-f~_e!2tpHk-JUB*Iy4Rm#WREjKQ`FDYX zW5fE$W2wsDSi?U55adHPo6X3V?<7kFYQ1-c`%|t7vtJU$c87NS zsfl6i@oKQz>Bjoj>gZ$58=8$ng*@)-e>#*4i6xx+@hL{Q_|~e#fPG}dZ!z|*F-}FJ z#IsG}^Pf_dj!&Qb4)`%ySy2Zg!@6%dExX+YpuI~AXH9t`hG4OD7bh5rs<&nL<@a+3sYxbva;*iC}9`=#Tyt;_j zK0V0(5`mMP{6d;ik(nTqOtjb~{_m1pN#GFanb#5J@@cxy?x^0eiJcZz*{9WD@uSdM(As-{3om? ze0=&NFje~SHHUX`nNb8gks0(92GX(EM(`;O4Gf@Q)F6TFp2g1jofHEGJmiWnwc#}N zZ@m4f>Z5ZcdcZP}QM#3eq3Rj8NeWjy2_vtEBj9NC#F3p(z?_sl`H;Mx@xRUl3@Hi? zOyC-TRY9!C{m$0DrT*P(x0zn)!i5hG{X#Y+Z7yW7zhaN@Poml%BzgW$-CnDeffQdYp*T=QC9ZJQOU$bE$k=^3=GU45U5W6xud5?W?*2Da)1Kr z-g6Kxl+|2~0!HPH8#lh6p9|jqF%A4{=U&20z{S8FY8)B4bF8e`u&N8ZNzX$?AR4|N zzj;d-c{$`$7Z%I|G^CB*u=!7chK7b+KjpLsCV|UnAJA35>tK4G1}czu>!zWExFI)E zpK%x^!!8fZR6;S4Afth2+1c5!qdA)HtXCmwKmqy_oSr4gAz1A_!M1@&j7Eaa@LJ=R8F zKY!-;UjIYG!NI{Fi3tS7P|z6wH53Ttkx!pu(?)P8201)@IM<%U1jwrXK}e`x7j(V_q*U4*$|Ca`MhOXt<-t!>F}-kqjIznyyNy#* zY`~4f$Hxx|4aJBt?5XlqlU@gmX{rX3eU;SjeYy}ZT=|c0MNRbnVC(mtV9D{x?yaL0i;#7R_MK{W^0j1 zVumoF=QVb95y5HH&QJDdmY1Vsd^buIJ)M9Yrf*=-04rh=djnYeIy!HJLopQN{{mBr zl$6wAsDK8U26p(?)|Mp#gL^X&Il|BNASJYNbc_a0-3K671MgM+=1o!}ZVi}Dm-=p+ zGfTN6r-z#ll=$?490np9;eZ2YS`|_pJiI25uK`~)A~*LE@REt|+_?j^tum>Ur?0X2 zv~J&~jc1j64wN}A&`zT50CZZ|duX3+?;q2F1$GDSWV!QLG7j$pXoS$W7j(Qp2Xw|a zI7w8L(b%AYt9PqAk1Q`2(@DLC0|M5kCj9mu^qBZaV`C#)R0EW`Y|s#*$@*_4-Fh0^ zLxB-G0}`uN;P%r`caBApFu5=HabRO(m(Y8%i;7aggR}vK)5_NNDTpZ0zf2yj%1^3- z1hxZtzVzXUAW#``BO@c(qc$iu9pY{m@zwUdhl@MOA8o!W_mgV7Qd;n zF{~x2|KQ(9F&l`T!axfZj>C(V)P&^aU552HA!gWzp1G>J+6IWj&`FY^Pl)F1315q+ z!m=whFfy`qb0bqwP{78;<&=|S0iGOmUYpaKo4uGFmI3QyWq6<%9R2RaLv*VZgwFA> z0#+~r;EyE!5qK)W%*@>P)lM6bHy1xYF)J%8F#r0>q3keBTRaGh8yT>%wme2CD{Rmf3dToZ+M63b?&=psq7Xyap=qACK0bCvj=ae#{86R*(4MmwxPeD z0t_?9cm)hF2?9!xdc~l9;mQ?!Sjpz*Xexv1M=rFmGkSD{R?t*=E)c>T*d$SCer1IK zT*>UaCk`JUbJfd;r3@;bo0el=?V@60nm|H5y}XRCtE)@<3^+P}K>u9ovwGVZM86

JcD4l>q+6e0`{b`qon}??f@u)mxIvr@Dl0X5Z>9?ojq(9mE4s$)z{Okp6z{(2)4 zuI+Jq;Kc)w$qGLl<~Q#Wi~udz2XJCE)gN>bCW(Z!EG%(Ae71&NT(ld2L^%%F%4NcY zQFLR}I>7CBpYHUkd-dO3fO-rxiykd<9mpl}z4jR!L}?)93T939%J10pe%NaE0p47PX{naKS7E1>pmYG}X!?W<*I zXX2MHU%Wq9g7*{cU;weV8J>gz=`?Y6=BE`ji-K1Hpi2dWBR(mqJ;EM|oZRVpB~bcb zvOs!B!}L<_?03I8;lK*mK6p^3XjlK|4`X_II*fAZK>^zmMomsl{S=fz({-qGp%?>o zR*>WYOFaun?P$xj#L&&GuEqd=q*RK#r3+(EX`-X#j34x}AYE4lx_Gm7>A(mV4-Xz( z%mgGnxTMVOc~Cb;fePEJlHl!Z64kFS9`Jf(uP zw6qkhNP*EtZ;F66hl~spB()}B&7)<|siF_i%>8MZKtOPvWWLw~Nv}Zz{0vrNwBUlR0qDL3-pmYDv7Imy>cYx6C%hP3|jf9SNr&p)yh1Ci=CrJ5i zv4@3+rz{RUXUAXe%fh;>`}QdXm-?2{76aU0D40!FRu*z_f<^E-4Vo8E94K;=IOkdx z(6&LKGzYBY(H}lnp3-Gl1nkJ$RaCAA`p3Po-NzO~By?NYR7= z3J-q035D{CR&>XiD_;Cz206Qux;ic(s|a9(r_1nCsvx|iU{ps83=DetCsKmuKhgfo zH*roxMy3HO1$XF?=@}SG8>S-WAurBBkcO9*iaa=>g5Q*f-}C@=Wg}3!$tfu@U0hru zez<|6H56Xg0RohlwP{d-Ng&42GCdHXwSX!y48nf917>)dkEtg*gk@!!=UU_DZbl;N z39NBIW%u*<=ld0S9o7dOT#%El_>2ae?D0Z6#sZlMI0G4^ezPmld?Y-$w{rQaUSt7{ z#`oHjw!eQ_06K!l$0$!^fJpZ2Y`53l%ggW&g3cdc{Q90AQUpeFE`EM~!U(7nDC*OQ zi3UL02eOtD>lVWI@_~0ZHhV#g)eoRCB7N&f67AodFdcSXNoW32?V0!FMaU3i*6li>GHUh1J7{ zF;MYl6i~+r$`#s>yU`j0fUanz#2e8Eq#z(4Rnl^Gw`T8H!n&h|Bj>*dzKREF4I4p2 zKd6y2o14Y%T~rU9oMr)=gUSXClkGN+KsF!qq!)_IH>aUn?f+RG^in}FCm^1=!dld#*Q zELwjA%|%2|1qf^)-l`fCR^Y8CfEwJgHqSSMQBCkd02=E`nP^o^3kwUhiI?o|5Z|=IOcl-7$^u??IPz`KCsnhO*H{aYcG`zZJQhW06D(sP|fkKK7 zrXdkfs`1eZ%ke*#FI^%4Z9iI<@)SxSA$W)l9%n>L5^{?WR3d0+Ko=nppMya61GFyg z-ri_^^z(CHpbSDM$#;GMSQ@Qxfmas5OIWZ&ev&w1D_yxJW|0m@xbOcn*5syShPQ5j z{1K?t*8#gxPl}5+LU>_CT zTcMi~wBGt=+Vuvw7%2HN&hCqgi=$&5#Aq(LQ)JK#uT9mlp_c>nnXuQ9l%5_BZ-clq zDuM=`jEsy(NMBhz3Duq%O%Inv4@R+iF<$|CzAAz zYQxn7PXh&_e0GVxLQ55WY;1$)>zD(SYKr~@0#9k3LXedjx2fZc(Gqeh_RyDq=g;Nz z(s}GeWxVWr)?}96OYg3g0aR6nZQ9kf;cm1H58edw+r{EVVq(Z0i$EFxB^tnI1bw~r zDlWUVFJd6~ab&dxA}$&WkCgiS{Je!s{zwrJ-lyA3Ssl zNQ)q)fJmx@bcu*ciIjAUv~+iuA_5`}(jd|e($do1N_Xe%^S1nhIM#e{4Y zr2LDX*e(UZ^OPFs>yy8j^CfHn)|d*WnKf7)OXAzNABbFE?5BaI5b|f>`h`>PfByU_ z75|bcS2o3pp4hm6)`g?=)yP@{m{=wW=e0t@ngQHkym2lJZ^WoxJ>@9bFL6szLQ zRJvboHCr#&rnO=xU<2WEnldZd+cL8^9MtAhm5Lj~d9h=)&L3>HrmD(B0(>Bx=;(!> zk`SPhl7W$Am6nnU>Wby0xPj@rxee=;!h-Fyvb1zpE=frJaY9d7wZ{XMe4@m}L<`!B z>kFkEWqlxHg%|p&@ig_EQ3B+ zpMvc?fHb(dxcWzgQs`HBj6=|c^3=`05N1tY-@Ja8nb+^ zM;kY%r>3NGl+E5IgNF$L@M8h;!xl(bb`eauh;0J8+zjwc-3fez@ZO~%@Z8(mGnYRY zTtAeNlamJJl-=W5&(WDpVPc#ar^Oo^6J>Z1SQs%CbH~4gLakPKoujhy=S@poP$mGO zBc#B9mr_pt@Z}z?FWtk3#HXjHTnRnpv*4vtA%;Q5m=ISvAxL(&5NS2B2?-B4G)iU6 zNAlxW78V#}?|#BPU*w?*KN)yNJA8H)^Z9d7pYZXw>;Y5rbz*XI6~^tD*jVP@%eIq8 zBBx8yWg;9Bqy>&hQG_>@w1}}bWFQb^Rex$+P%%VQEa=tI1R>)Kw}XLa7PKxgr?5vB zz|5~_Sh+!hg52e>rC6dnTOe~*J$HqmM*OpEW|~AQ5d?{c8ne>AO>}Z`T;j?($`w;s z@Y_J4CPOZ1uoMFj?LvUiwgBQB8qD7Q9d~fj)cTUDWR-hxadE9-N8Dv%V&LY!3Cwt8 zPHVIJ=N^z=CGv)m+~kMx_?W&ZV7ZB&-t?hs1konQDJ5iNWC{vD06DTPf+jRBjuM2T zkpG@LvK&eMgm_?Jz-hB=6shtwyjH%vw3Gs@KTzGFmQwV4$>b}Y?%iqVGhm8LA3#SY z9N@+@i0ZL%aF8Uyf9K4(2ES@W87DM63Jxh*SzMrXD_e9~LQ)_dK^=^NM?SR1dyk1J4fF&m9X6@H0{96Q zDIHEOQcVUtK!54G*OUA=7{CqZeh^3`=XC|NKzRV%${rDuM3Nkp{mC)4El7q5YoA61{w>pkF_A)#t}%|`qK3dA|G_|>^)Uo-8n7x*Br6UgM%a> zl+_X~FA@y)I06XKy5^}QJPRni37RDMgOelJxd+q^d9{C*0EE*;^nHb6; z$}^p}Ui7EX{&8su0af`Xniph%&2TOe zQE)E6d#k#fLK-BYs0bZJ5T;Rrsc$ytQmid$NYD> z9;$``R@clJDdYoCBmw~u;O=@+!^GFld&D@%1Q`U27QtaW1yjpfcL7H4%~=_&LH-cd zJiKEf8XB#?vcjMu!_Cdz(cN8oSr54R2?$U<&wMiyl4G|$T zp-*yIB*&YR%)QWQ}3E4=w!BirQa8A}E^aw(~b3}sfNMpo_K?*}LIfwV~U{_fqobI4ua z!w4wpo4@r36QpEh{(p1{LtA~2umUK4L1KV{xH$cnyfU0OO0yjuE1T)G5X{VKe$=o9 zsaC1G&L|pc-us$hD%g=}u1HLpO`{ZB$Ney7(-EY$njlGBv#WznSXf!Gfk9w{^!{W# zM|eRUL?0wH9o=$gK$Xqn{0bbWgoeh~&3^z1FG8&Wfe4rO#UWuD>$$2N{qK`!J#V-^ z7X_ej(jx*oCG!pw1RXh`qK1V)qP+koI8BaLfFu|^atvJ>BhUQ;FTM&?p@V)&NJq<{ zNoe@Luam2KH#noa1OsxUO|wfUtpFmhc$0;&Df$-+3 zw)P!Fiv9WsK);dG+X=ar$P=iD2KSiDs>OYEG-+k@JqtpTJT1#nh!aiDJs|w6NPMPK zpn}}N?1NO+`*ckS?Kj41a>1@6fA@IYB$*yic;wML+8F=Pg<=GOH!S#@c&vYrrZzb& zzq5kmD**6*gqXOvc>MRh5CFoj;pC3p;lAh-#NN<)Q<0FJN+czsnUMYMe5-~K5Wezc zWENZX!2}8tX&|uHziNlX^q~mix3Hi;xehP!!~NXBFEB9ntuYuwYtRNlh=JD3Z{g`J z2V~|6i1AW+R~X=#5L9ke<^y~%_Wi`(!J!QT3`i~QyG8^3{pX>aX5_&K3WgB$7ynt; znV!_X4f~$XIu|`{4{1yWCMRoTOD192Hp?IVo8HWKMznyEnabo{A@CC$Lpjl#*ARV% zPu5?-vFE4|0W__sxYM1rWV5VzTIw5HP`3A1s3}2Cj4Xg5qdiI+;+ ztBOrc@E8p;_BbSeixeq2s1bOV{|ZlGV<~!;tF-2l9C8wWHCHFuc_RV$HvqkNd4KE zqy1b~*5@UYBnTB!y9kcaDBwV7ia>}HsZlQNSc*gx(N6N@0M+p@a4mDaomWx}jXs0{ z&)bQ*0B1dAs?B1f10=Clt> zNJnp%q>>BOr$2+KgX{CFBiP;^9@u76l(HD>K>mzM99Tc-km(eYw#EGU$X2&l?e~Q+ z-vh!bb!UFh1)foxmXu^^-wohaFKaSN> zmnD;bjGWAb_nzJ(dRp8)iy@9$rHK|Sg_~YXY5ZZF1oUjY69R8f)loGKPv0PZ-aJ7u z?P?#G6>AdS5IJt%1}Jyx==%1%C$LY%^z`P#AE*=%$X0zLo9DO{CcU)uk6w_I5aa37 z9~bFY2-GE8o(!c^_VviXyxX-?)AL(vxWO5Ehq#V7gn^!mZ9-9NuGo;?og56xBa>rS zZ57DCZ+>GI>o}NCdmmq&-~*H& zeKqwHgSLL3z|GywNq6`kkuDcy?ma~RwCB$4L+O=L0r*hsUVdwUE^a#*ygs5DKpcz@ zAGTGPK&cnSA1>rI z@jU#;9X-dxhU*Q2of|twj$-#qT2Xy_5n90nE{RvXsth9}v2;UH*Xh_L<1P{b=2{rZBw(?Rb@>TIXpEEBI z{i9JNsubKZA&{c<#GH=bX#pKKG}O6~XNhQTqJsP!oP?VXqv{&1A0>eRf+=T zJZt9iy_8vtQs>-ay~SAlznA)paoB&0j0_zu1QU2bBzA18Q9001G`gzCJDuyPWYd0i z`PI=~n3U*gXhGF$(wo|sGi`rHTUjA*DC+t6PvE8GnN)`>z{`CAa z!RPFFdCub@Kyo*#QyD(Uuu#)+{7?N^x;gzc=>ugAHXrKJrK|<@y7;1K)E$O2q>Xc8 zi2u;q9JR2ncfUkoEfkfG#yNfZw1bn?M}mX%20E+5As2d7IvBT=G8aJCDD46!59GD2 zzT18Kt<6gmXC;uT-W)!l>o=e)rKe3#D(lZq^tkj1_R5S;339=*eC3LY=AhFRtBsZJ z<52RM0T-1X7u82qHSA0YJv?+`Z||@4UKD0!=CPNEmSB1<$@E&) zO=9r2ZvSnCsJ94zC6uygD%qG%-d?}4Oa@RBucWU%rtvpQiq6vcC{y?-quwgOGb3Ow z`-+t5*a+@4t-Dr(rP0$(PK!8aet+!e;I70Fx-9*m<4_EN0;z2#JGyKy<#^P3(RN0uX`n=AtV2&VSABcXGRmhqv~bzzmYg#_){*|;27XnVA^MfmUuk?HSOvUfG4Ohe21$=}n*$QCSOJ!b7#&Nn`~Xwu0al*k)= zljhmrfge^@boRkODJ>z3{fq3~{_G)zyf+FRW52->lO;DImW=+Xaz#_alb8vA{)Y8y zX1_UadgTwwR{vJ(cT1?uCO1O636{y1(wDxs`^Dbq#T`l3$P*3tBc`C?$wp54g*5#~ z24$g~oVC4!-bZe9g(?+9Nh>+R@}t=vXw)Hbd)&pv#H zlcTJz;TiTr?Vwy87cV4qeqp0~7aAw+1Z6WxGL$gb$ZwL_87WyL_R2E9QOrx!ef&i% z>p_(W|JRM`t8Dt{95v64v?~_+m|Xf07swln6`&yeNANY1KZ`~}n)AcF+pICvvYFIA zbSWuf?WlT}sjLbth6*{PQ8MVHWVh7mU_}q__C%{&M9ZLOE8})7KkU8LcqaLV;{yu( zCa!-NS}ZUsnIE-R+bZXgxy>mE7m1RG-ZFl)E^i+#dru*cjNniP*rNKBmHuH%_}_%w zh)p6A{iTTd>)&Jf*>9t(I*z)ie&^#;SGoinm_thZyeGhWqgInz#2&YJcbDsH*b49x zT>R_<0u$Z}7l_)*+{N$1M_KxzCr0{Iv&tHSnK1cRPIFe&$4`oXw<#7#e(0TImYPqZ zCc4^7XIavu+Su66?HNShYZf~U@XpK0;dfiROOPxO-1Ypq5fih3`!0hD$RF{tYow^E z@mD*pp{tshJmYBkv->b#6d%QqPcJ2x`BC&h5MCCs^VgegwPv zpWL@+ZstOAG$I&@((S*G|gF{ne?7_MS(AD3+(}glzsDBa(!*Nit* zFNbL(Vu6N?B@(}EygUyVZfS2@+}QoFoFu@4jr#4|#oX2mG2E<|gkpXCpYba0Z;7`V zxqig56D|E^YSd-%?lhHf+j!T~p466m?>*5}D$myXAO=l!X<(^bQRTSd^J8sK+|U=+ zd%v7p+S(U)I!XH14jr+3XO^!D2b?6+cR5~r0uGQe8rx|4kp9R{NMF9)pc9nL`voYc z1^rW9)hIvjRVS{<*1;o(i?PyKnIdV}7Q`p7?)&r$An8U_)9OMQBZhR zR-%9I-NIP<`~8jCscjW)ZJoP<=C^=fzwVNffB!7<2A}`OBRnjJHnAL)G^5Ec*jxQn zJ(6F=qlV<@;K;GW?3r2ZadL3@$*H2Ep#FSR($A5vRbUQBTiNwqZs@zK$E(YtNJvpK7)4u5_zFfaQ;iv0<8L=w;IO8!&}c0~A7I6Ne&Jtg;$Yf~lllRZmX@7jOT8x~H3>=C|OB z3n@~v%!xa{O7?Zh9qvpK#wZ0lu)j@}T-Wt`{2uc;yOuv5QHruUocAR17p1NJUr5mq ze;z%$d1oSNzLwq(0aci1cbw6o)RuQ-RCH!Uc3hb_Sz+Q@H(hSREb8mn;Fi^f)|gOK z>)*Ohp?dx^9oh!5vJ)q3impbIv)YBj?v;obQE1ugppQ@D(m<6-8Wo|ab?(utv@~T$ zrcJ_ePc!!;LQHwoL#+Ca8YDR?M1GVvzp=7#Jv-+aiVf(} zL%)$QZrC=j6=-v{4YvDYNZXP44g}Ez z66jx+Hx%2nJ7gFdA`o(*KarfCK9K&JvW@U|gw03Exqr*8sEiPNxQ?%mkJZ~?m0OHO zpiL5p6%DGM2*BAI6D|&#CJBwyiTB?^tJWJk(W5*l@&>2+%O?Z#e!~x5QpBL;iQbN=g2ELNN@j2 zO$^lKjqke9V))oRSiuNj8B|JWo2#kcoT}Px;!=}f6_i;b`JtN zmu^R;4F1Ka7=IOiu~&|ES9fRAcj201mwlZvsuEfk-_-x)l#a(=b`Eby%Jn)sxdb$t z^V0F~U}9jbLasOFK!-U=&@v}V+bh;kIL&3V#1eLki>R=-mrYk6J7q1FDA}t)#j34$ zMJp&Wob%YRY6@p&W^-5>XfpH4X%(7%&0E&avD0QA8_s``_V&;^pH48F@L*W2&+V5)ly^?6#TJP^osw(s(#fd3#r`B?kYO+)=%vGUVhKFy?p*kIPCLhUvR!D zgX=fz-ZQ5;Znk)eeDl8PPTSa`n0|9SkB`*$i6bsIdjICT&@%oXmvmPeb+-h0x5V^3 zz;{}s}%oPFmxkAuBd$`y*Oy%UEP9wPdD2!Rd^d-eSPBm`fW_k zB3>f&PT|qyatm5%_gdU8H<&!L;J3e9?{oG1o}1f5a~~%)ymh+R(HULg9A!Ph)LY@LGOxlx zV2s|-ILVtuBP!`zM2yb0AM!5~J1V#N1sDEpQPL(FpDKeK70p+#T6J8H>CD<4%x5Yn zq@#0w%|UI5;^cf5E9Q5d+?u@Uih+#0XvokIV@C%+MrfCw|Q~;JLXAN^*>jiC-#0Ktq9w=GtV+Chc;@o$>vSxXT$%P#VVQAEn(x z`3K0q18j7NNc>v?%xz)fljQ7G&B64I1FNM1Ef?;HNnZ2z-fWtV-j4gnRLsn@a8!|t zy0ulXQTfzjVSJ4v~cHZ}FvNEfj>Nko(QqUQI766Am#`DVBh6+K^O2Lh;3v zHX+LF-*8g(BWNWLdK1+uEFSbSGpSg=tZr=2Lc6%EX`HMPzDs;AyxF-dSALd0xXf^G z8T-#40!BeW6tAF`B}2DrI~)Of%|AW#N#MwA~X&-nyP%$YCx! z?;T(&oh%e^4pf;?!fgkD1)E=8@AUPve4aNrzm3Kn#IZRm-`C55JW{uj2laSc|G4Q? zMVZyH)RY^WT?5}oB_VuhC_U4A4izzX(2Cxz$wm8!qWf9Rmx7&>m4P^Z+s79GVMPIj zzV3O|C!e~6W4&iX1RK+}giHocFQeJ}D{V$Kwv+R3$jNzN6Wj@(n;T$Zdnqj&a`+`D zE$uOQ$@KFDYdalk5odeyUoLhMOO-NVPIml4wNEU#Q2>iQNk1sBe&yaoOMWuod8q=du&MOxVBoz3&oNSioNoX;^7_sq2&Q zhgygfOu)_lc;rIiAhjtrb!9*>00@t!CMqC%w2eR3b?wxjv8gTlly=M}n_e?5Dh?)wHl`>`3KlL;bQ zYmuT75^ZB^^0L46VAH2m@}mJ9e!dW)uQR3i^7P`h-S!wxWaJ&+UCBV3FA^X$@y@v9 zR#p6ylf_X{F3+|K8KGpgEuM%WQT4apDHfA9rh|X!A%g#0PDJgaoouzaeJO~^?b$At zqcmZBpIXR)i?G~<6?}Mu)p!_Z?4{S(Nr-XnP~)Uvsa;!_b%)^^rjBPJ)*3{anlZ7{BmYx44pXe%jvj>0rCRIG50?R^CGH-iY;=CvMAz0HvV8Cm%UNMD5pzUqZZ zt)X>2*?)7Lfc}0QK8!WA{&l=mKFZ(SN-Kcj>JZ9@(L9bTAqA@In`NVadZV->gc;Kb zM=2RABbU)AC|dIy-Ty*W*@>p&j0)V(JsBVElPW0@FtTs5yktV6pBY~=<^wrBTE1pB zEPCH0wEiN2rIa?PH|b4@^Z;N*#I*IqF;pwVEgl_BcZJ2`4=sL-u?PA2b8~<|OFVkc z-(h^QQj7d}nO{Omu<9Vhhfhe zZgR&z@YYPx1h2jxNbLXhRZ|-#&91APY?`d0={83DGZ;z#o;38cJ`4ZVoTxS31 zAn5K!HvY4!ZM`3gq#UouS~Y%|t(fqr7qN*<)-eHZ?WLUCGLRlh7yV96jmPH4@6XuS zD-m8^Tl}_$BBZz!pFbNiJm>-($k>dDyidcY@vk@K@suPRmtYBmdfnH{|8AeZ&h?5t z4!^}ohHFACT#b$E{{Sx!QzjGlASAF&}nM!lJ?TpaP+?ArVTeT6Pw9l5h3=HwEh zdzjP3Mb{L^eQdD&yLIPqdF);}xxII!jP2>u{kso*xMV6RCzmOhJi`A6~vqr zhajN(Fyx*sZf*U+v`j;@=1aho=9lLdAGQE9a$8ldoW^NguBQOTPRo!>W-!uDOK;zZ5z? zWJ)S!X}${>bg&tb_J)O6{mbV9Xc=-yf}U>dPvez+ww7nyJ6TfFrQqqL!g zZV1XGDy~c=f0cr-sn1D-3$lwvDS{f?9olkA$&APh(}>q21^24 zjt^IgA!W87hn5E2!=H~pw3sd=U$litw_UGs;#DITqO$58m#rZ>bW0Ef`z7VyzRhAL{ z6g>zuZ%0+$_J~Ua@Z(LVPU@kYx;6b4go%!<9Y&@GJI@mxlcbYHnu%^X8@SBXyQ3*6 zDfwX~pHa-5n9C;8aypMYQw}lwlp9Tbmc?~ZydzkXEh_r-*?eZCF)BjaY5u*F6T0$? zOK!>o8M$6DhjQq?Db*fLmP`B!O?MI2jtvz#F%NP@2}LL)fCZ^9HH-D*B6{4xrrYUY z1G8kjyX!E38r9JD);F!hO?@rP`f6oX9<klCY#`4;#?UKR;uP`87N) zvewt58jI^+t+6hjz&mml{rPi5e*Od@8jrdsFbD(i@4JSYzZyzjpSH1$@3xG%rH^f@ zP&(c9r+Zt(>epPCI%dfFg8anjA>5U^E5_*A5AZw+BA18e=ys)6ozqIzB> zXCKr3yuZJCRq4QvvATLhg2|sp%Z<*TUG3!dL-^3yRzV zLRy+bl)`5sy=%`7*Oyc|@Z)rOUznvN!9^g8*1$#^p8hvpT}CS%76f!WK6wr0?Pf8B zC>+(I@5*l}clO`=4W!qk7`^bct_5^Mf&E)|ZdP9jyfNT8+~nYpGyj=^Dajf${!7f( zmg>(Rd`B|OU(E>yJ9AwBSs>*KRo*u^Xu#TqS)Xm8oN}c-Ywd)dX4!glEG$GQC?i}^ zNtRxnc3E6>bZq(ewcYJetv=ohK>Qr#QmBuAo&J%Yt(%tVHOXVEnaYXphxc+h^%`eq z{XTyVY4%dOPY3@gZ9oK#I=>NO05mt~&HWXT=j1d2J}N4(YJD_0+LMG|s3}O1jU@ln zqhs$uP*Csy1uP;-SWpuBxUa8h)f8Z*jy#P9Yn_SQ+;Dfs4ARx;q#SQz*4BpXuB11w z85Rc)fhx)!lP{-?P0h z(f_ITVPu0(8ocIDSy_S5zdeG*blI{mrAb~&4+>xFK;;G3wkyyp4Yi&Bd1i*DS4|Bq zA$BgY?*MjPlfv@#;B9=PD^6xn+ST!K$GST{bYl6;j9Rx)b5jf|BFeX8ITAquJ6U43 zo`ep2<4x-CkFuFjvYE#ho@QlKa&mIZ+F2@jn+}uPcotTI&R2U~_y?PUC}>zB7{OA% z(R^8-D!n>FeHIkC}54 z)C4S41W4ieobD$#w}#rVeG~ZMdYnwj%|QAh)`K9l6*WgWM3Lm#gq44^Oz!dvj#x?X z_vfFwmL2LEl4y^)2(GGeExuMDKP0Uu*BqeU$+nqq#zo{if^7QgjVQst)L(x z!>L<+a?pWU$aE`-nV*+J8;xd9zJ1$t%^PB_&V;nm5?!(k5ox|_R9W$$k z0vBiN5-n|8#g~Pz)NXfQSYk>rN^CmRT>=4gepzw(djI%1FnUdT-w4zXpg%3pwn)(P zs&;z22AXg9tBu*eyC2@Gzj}8MTDX7MF4yFXmUS3D1a2}JOh|peM*awqViZm1c0O|+ zv@tyCA?kGL219w↦f@%?tcC!Zg#2SU5?z8Hco$U$Wt+RuDN+|QgV?SmtdGvU}pWW7Hetjy{!o%wUp{oUhuC>>j#?NHD~cfyOS zdzuA}L$`-&`J@betR%gv*pHI&Z+M}hT^DX}+J8_~3y+zF z9R-QQF^gV}7LM-jjd#TzE!k-be85R#f%^lBctwA;Jw=VC4l^^)*=#F^NahqE_LJ$R{|u$ zE*E%`fpR}7#pa@C?XgNu`6PzCCfsK%>tybdN5b;CjGA?2g3-*ItZ+q2#4n)>-X;#H=y{>t$lr$S6GooMUpd(>qd=MPrXovttc!Em?t=hFb$aD9DAY3oUg ziCCCM8D>vWr_(+^dE;k?kBPW@FR}UJv~RH`n1<~eYsYWd=AoQOX!Q=5L={W;!F$5< zmjZc=3?Kd-J!9$bBP`KQ_PL{{hwusulLC!R=tzE6Vx61vgNruVZI87reBBHW!JdGw zxBn?qjmLyTS@s?txD~UI$OghI#~r?xZp+*VFUXf>JJ#fvU%i&{q7amjO9_LV>Sur*T!9aFRpw@DOIE zZ1C#-OxtKfpYIDs3WS*E6givP4sAvj^G`2U>#ZrAmvQ|hgWrX{FWlCJW(mVqLPh(e z7RPh?hC+j%WZN1l-8Zg1csSy@&n|SE=jU)Th6jM91eY-AO|JsxZ4+fdW&QpJ;u-<5||$$Ul6zqW(ZyEDTedW&5TipoR0dFB#^z zRnML5q9O7N?tf4Pa=Id3KYuniUXC}_shBAB+nPId%>|AnhQ`{hX-_%9^UtdDXLcKt zsN}~54x|g}A8)-vcu8{<#tB*oII!yk_p21B$-tFkQNLo$WlcZW*%&ebVT*$^;(U!m zn|J%)3&mr54sTu8Upg>_Nuxm=ddI_q`by|_ct;@S{v18}*_mGZPclAxLP{bUVOr7a z3lzKRG)yhtEP2{i#z?#NeD$Jn>8nUUu0MEM{>mev{nrQ~*$;|`Rrs%s<=ZX}XB&H( zZ!{kM?V*VJ;*ir6)1HWV^`we_HzoDi_=K zsgu)Q3Vi^=b;wg$G+(rwe7;Tq?FX+FIG~)S$5twjcjPxS{x*u{YF@66(6o;fR=l!E zNJt214b^Ea)N0_3yC3*u2ehN9zv+;O=8siTbGq;7$N%lo@yMC&NvNPYIc{HNA-G7F z-QV45`M&){cdK+2nv=K^UIEXJ8>V*&o0j<>I@CO%I&t)Dapy1pf_5E5-#9O0q*N=k zc#kbi8$oEbuDO>bx7p@{5gql(HdWgqPvvb`pQtB^LA&%A<`oZtS(p`HH>^qZ15iR9 z4vsW{_xo$M9SdRow_Bt;`C6z@JG@AB((K`_-OJ--{iTkha&OP1a4NmkXQ0Wo7h^j5 z=|5N1JG|v0Izn?!qec5e!~|$KIHBZqVR3O~hiT489#I(O%@A6T6IA089Hn?t`PwV~ zv_-zO>ue7p#7fe^<{yl951q)=T9d04ZdySgqw`=-M1>B;$AU_<^?>+B29mJs#iUz5$gf=QI9Ya}^C6T&;q zvoGcstt@7Q++8^T_IVPh7Jjph+x#(8B1FL`UD`ZS@E!%xTi6>XvTbqUFsE9&IrT*_ zAvY0b74NT_fZ}VPDLEJn7`w^!*K?nmm{_02SWrr_1#Awto#}0uj!g*&sH{Kgw!8}t z^N5AiwdHhG36u{+*3ZN!@(-2>m!rhiRHh`2Y~5CDt==$hRA+XKDp`oH{=;&|5W zNh#jG<+?*TKyh2Xu)Rg~&LV71n`uG&U%O?uxje?K*U9Ylcs(w5Ur9alL@OCL99{P~ zG($~t=OuofB@*%N8$LoT`Mi2=E5-?9PKc+h<__Vm1~uO-cW_w>7hIlDc5^U zS!^+B3n5fvrOh^^rS)4V9Bkd~e6zKYNZ&&qdN@Ao-GjC}bK%%;qDH&bpJ1+Dg3-(I z#dIMe_nAq21W#ikE$*9<@~1DTFYL**oy{2#=gw*W4sSh5c1lZTN7&&L zG|eP?G{Lrfh#2yc&Ll*p5P^$J3x0i`SNJ8do~Y2)m&O7nxeSzwt|jHs>-VX6cC^==v!NnoFAvi3d>goU>Y|za%z@IuxDU zy4ETsqGi`Wq$@_Qtencoi9`;h>-b8f79JfBnSm;G(=#B}aq9BE#p*pfKhYg|AC25d zQ}^HOoUIq715@I-G|bBj5z(cGZEY?aLp=DeE}X-?W^USKR#My&1sOrzTNY}I^h8Eb zdCTsFvn9YKG@OhHl`Z)7$Vxc;->e4#Zl;gmB8Nz0U2Uxzk-hVu2ks1dMZNQJ{F|>b zOiUO#Eb%^l%6B9@Ij<-zKc0Y{PV0W~Q0aBi#LRgsPUJ#EQ7t9~v&j9DTo4NyNPCJ$ zF2$pNUVwJ{U`{MQ^qeNqRRxC3gP_%K9<3vq84AVS+dF$s9!M$E$v z97IQVZNSq|?Y67ShS~kk(<6bG`0b&b5kE1!g8CI&F3v=|UC(Kd%HSz$^Rq*=U5;$c zuU{wm0i*D5&v2fox3-Sb@^Z{)()mI8EC7tKsRhZ=5YW@yTFdjHn8gJJa=f4YIWs$l z2QA;9zSwCB<(&M!QGX@XR}=qp60)J)1@%nA^HUPU+}2viQ`+v9)LE6q`u-y}JLDzr zsdns+P*~}&ez~`^SHjEF-fGbbc^;S+IaQY@$lcWOaT927)NwsbAc!DzaltQ@JQLJp zfK{B5qy6cN1EKlbYJ3zVaq{Fu80N?Ri;8nf`_uJYu0HJ9oz;k<`mY3uy;P0uPYIgh zS6(^I&p+e0{sOIx#)p(+XbtU!(vjZ62*i^oIch(b4h4Cwj7_o@$BHjuaA}`8;rZ%V z8QKW7x_|s^k5@_FBfQwPx^?-x)Wb!h2-w~p<97YJIp~6TK{jDiSo3jH& zgqRErXFqCwnE&+=1&|QfG~vxo%r9>HX1@KR``7VNii6VRSH4(w*Ssl`VJXGZ>5^}s z^fxOG@D5tLw~gmwFGctDM2F_UQW-Q~V5nVnnpg5afzCXTDyM}6oRpN*Fvj(=H*tWH zCl{?yvkIX|IzRt6U_3f&?quSI+ZfUlPs5yPWO{O2F8`Sm=lT*O1qXGL55_eRlt=#d z06ldf-UQQNqLeu*Aps-jWe6;)-r%X|?#}+A6CM?hDYBYJvd)`k70ZF}dJ!9;2bLA$ zT@(bL@kPSpcXwg0N-%uXmDxVGvDsI7>HhZ#3mG;4iwN%u>=A=k8tD41y~}DRvrp9y zb{#vqx*Gpv7Nx(=y5a|SeBeT}g!_c)B~|25UJfM``;^Vd@FK9qO1u!r=c&VXjSKGQpY zsKG7U*mySU_A~CKHU&k@237o%^ZizFaWe)QMelzX*{3GBwBiv!$C4y{N-bD^!{7gO zWwt<&;M(ejO79guRoPenr{*|x=flPCvGRqhF?C*E5}QOOw%$&DC9W39g=~bf!+q!5 zeEk-h(2_zMj<%XMMzeICX0g<`09K2T%v7A8wwlosCz{5{l<@AGb;3(-*d@pFUAy6c z1^Ctf3B>w8;9387UyeViN#^Df6vl(vD6#=9%Vhpn4jw^SZ$#xx#E*La)2H_J1kiQ? zy*CZb)Pbw&fbh%;UCZt0U@5>KH1U^(=?4H6R+Lp78hSLZ$F6bQa@SR0V1+#B!36*QCxhX@iQwa_pmL7{TM7iP zA3t(taPaUX6%>f@(!P#Q#~~0hFJ3f*dQg>P_DodQa@^y0#6xEG;SH*w?(v09{g?O~ zWlxc8bQB~TxCWC-_CGqBU)!Dk|L2MQ5B3ST=l^_ZCIK`)$d%B=VY=LbFlRce4oq(z zuI@$>{olQN=UB#3aptm7-0l?h!$bMxUToJq$XbGW%t9w~GIHj9xABMYks8oSFaRV7 zNsdQS>{eIby^RD~5g_@rEkFUb`FlNC(-8Ijj6Y&Axr$UWth*8S2E*OeU-1L9f5fAt zrKMAJxvFohUI_>qY0qXH3zQcZtaKQ`w#bqaauk*5b(*KZ7NHvM(x zXlXWBPA-2}^6wIL{#(iHspW--;8w}zx7n*x6AjvDuC6szv!IyLvq|lV-AY}JXf!9m zMt}VHaRw+W342~?f=nU%!^>w}u4Gs6KlZsL2tlp`qd7e7>>>a}Z--=6M0| zP~QU`TmXpaAt51@Y(%JBk3j0C6!anQm%Wmci!kj04sUCFyZo7aQCZm=&`bV!w?Mr_ z8dPU!DuC7b{+6=LC(vohhl%gp^eII}@-#FwAW!q@u5+EX#YE|I&?=FA{yezja%6Ng zFD*^NTdZ-)xJ)A^K0aq?NcrBqd#gZgq@twk2OT32e_(#Dq^cSl5*oT}TV)4=hPPH3 zBSigL8flTDC6b@$J54PuuMBvM)Bds~^cauSgA#*0s^6i|#nojEaIJAl67L!S`TnM} zN6qvWxBcMYU^v|BEs^By*iGN26qu6mx_Q-(3WP}l?ko7W8(QyN0S|9Gvq0gCsqHU7xWNmHjeSZWU)i+U3kM4YzCS3)0 zcXyY)rQoyk^PWxK1or&(Jx@39%k40?hP~5$lQ$|Va$|o%xKZP$;q#UVEiWRkTaRKo zeP9CnD$C0Uk?oJ>H-V`Ok{tt}4OFVz zJv~D2y`|*{Fvl4*Kyd2e0*|*$S6`nhh`6bth*3RBo@=K#H5GK2oSZbOn7m~th0ctL z8b^jQ$^}hZsq^!C?iVMD?Ck6lKaE1~uEI0^4T1>oJrJw^R!gxkF`3MhzkW3nDZIY& z|6q?Q22-<PiA$6zj9v&T;{p(G9eZ|wXD}n zRB_N8`TTfK@f38Se&tr!EJ}Z_X1R_WU0|P^&!J{tY@i0*1S+z9x&IAf03E=@~Q zQ|!}Yw0xC3xnvROa%XsM_Wpfz^E^z9pItE|LWPH9F;O-k$RT9)4)$w}w8gt$RIA_bp=k_3I zJ3KVxr&$En15|izt2AA9=DagL-Vd$QE~?eut~`g6np0b=Nf78OjsmM{?9;0Mv1?h$ z#-{l0D(KzY3M59$xeI>;vi1sWoVB&JPwk5!%n>;>IhohgBz}770m3e + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + 2 + 3 + 4 + + + + + SERVER + STREAM CLIENTS + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + LEFT + RIGHT + + From e0fc612c688e60795c9ba9003ab80194dca81512 Mon Sep 17 00:00:00 2001 From: Raphael Dumusc Date: Tue, 7 Feb 2017 10:52:25 +0100 Subject: [PATCH 6/7] CR#1: allow single finishFrame after sending left + right image --- deflect/FrameDispatcher.cpp | 28 ++++++++----------- deflect/FrameDispatcher.h | 4 +-- deflect/ReceiveBuffer.cpp | 19 +++++++++---- deflect/ReceiveBuffer.h | 4 +-- deflect/ServerWorker.cpp | 2 +- deflect/ServerWorker.h | 3 +-- deflect/SourceBuffer.cpp | 5 ++++ deflect/SourceBuffer.h | 3 +++ doc/StereoStreaming.md | 46 ++++++++++++++++++++++++++++---- tests/cpp/ReceiveBufferTests.cpp | 25 ++++++++++++++++- 10 files changed, 102 insertions(+), 37 deletions(-) diff --git a/deflect/FrameDispatcher.cpp b/deflect/FrameDispatcher.cpp index ac73337..207a085 100644 --- a/deflect/FrameDispatcher.cpp +++ b/deflect/FrameDispatcher.cpp @@ -134,8 +134,7 @@ void FrameDispatcher::processSegment( const QString uri, } void FrameDispatcher::processFrameFinished( const QString uri, - const size_t sourceIndex, - const deflect::View view ) + const size_t sourceIndex ) { if( !_impl->streamBuffers.count( uri )) return; @@ -143,7 +142,7 @@ void FrameDispatcher::processFrameFinished( const QString uri, ReceiveBuffer& buffer = _impl->streamBuffers[uri]; try { - buffer.finishFrameForSource( sourceIndex, view ); + buffer.finishFrameForSource( sourceIndex ); } catch( const std::runtime_error& ) { @@ -151,21 +150,16 @@ void FrameDispatcher::processFrameFinished( const QString uri, return; } - if( view == View::MONO ) - { - if( buffer.isAllowedToSend( view ) && buffer.hasCompleteMonoFrame( )) - emit sendFrame( _impl->consumeLatestMonoFrame( uri )); - } - else + if( buffer.isAllowedToSend( View::MONO ) && buffer.hasCompleteMonoFrame( )) + emit sendFrame( _impl->consumeLatestMonoFrame( uri )); + + if( buffer.isAllowedToSend( View::LEFT_EYE ) && + buffer.isAllowedToSend( View::RIGHT_EYE ) && + buffer.hasCompleteStereoFrame( )) { - if( buffer.isAllowedToSend( View::LEFT_EYE ) && - buffer.isAllowedToSend( View::RIGHT_EYE ) && - buffer.hasCompleteStereoFrame( )) - { - const auto frames = _impl->consumeLatestStereoFrame( uri ); - emit sendFrame( frames.first ); - emit sendFrame( frames.second ); - } + const auto frames = _impl->consumeLatestStereoFrame( uri ); + emit sendFrame( frames.first ); + emit sendFrame( frames.second ); } } diff --git a/deflect/FrameDispatcher.h b/deflect/FrameDispatcher.h index 81b0698..caf1f9c 100644 --- a/deflect/FrameDispatcher.h +++ b/deflect/FrameDispatcher.h @@ -96,10 +96,8 @@ public slots: * * @param uri Identifier for the stream * @param sourceIndex Identifier for the source in the stream - * @param view for which the frame is finished */ - void processFrameFinished( QString uri, size_t sourceIndex, - deflect::View view ); + void processFrameFinished( QString uri, size_t sourceIndex ); /** * Request the dispatching of a new frame for any stream (MONO/STEREO). diff --git a/deflect/ReceiveBuffer.cpp b/deflect/ReceiveBuffer.cpp index 5711e14..34261fa 100644 --- a/deflect/ReceiveBuffer.cpp +++ b/deflect/ReceiveBuffer.cpp @@ -39,12 +39,16 @@ #include "ReceiveBuffer.h" +#include #include #include namespace { const size_t MAX_QUEUE_SIZE = 150; // stream blocked for ~5 seconds at 30Hz +const auto views = std::array{{ deflect::View::MONO, + deflect::View::LEFT_EYE, + deflect::View::RIGHT_EYE }}; } namespace deflect @@ -81,17 +85,22 @@ void ReceiveBuffer::insert( const Segment& segment, const size_t sourceIndex, _sourceBuffers[sourceIndex].insert( segment, view ); } -void ReceiveBuffer::finishFrameForSource( const size_t sourceIndex, - const deflect::View view ) +void ReceiveBuffer::finishFrameForSource( const size_t sourceIndex ) { assert( _sourceBuffers.count( sourceIndex )); auto& buffer = _sourceBuffers[sourceIndex]; - if( buffer.getQueueSize( view ) > MAX_QUEUE_SIZE ) - throw std::runtime_error( "maximum queue size exceeded" ); + for( const auto view : views ) + { + if( buffer.isBackFrameEmpty( view )) + continue; + + if( buffer.getQueueSize( view ) > MAX_QUEUE_SIZE ) + throw std::runtime_error( "maximum queue size exceeded" ); - buffer.push( view ); + buffer.push( view ); + } } bool ReceiveBuffer::hasCompleteMonoFrame() const diff --git a/deflect/ReceiveBuffer.h b/deflect/ReceiveBuffer.h index 3b88642..efbfad1 100644 --- a/deflect/ReceiveBuffer.h +++ b/deflect/ReceiveBuffer.h @@ -90,11 +90,9 @@ class ReceiveBuffer /** * Call when the source has finished sending segments for the current frame. * @param sourceIndex Unique source identifier - * @param view for which to finish the frame * @throw std::runtime_error if the buffer exceeds its maximum size */ - DEFLECT_API void finishFrameForSource( - size_t sourceIndex, deflect::View view = deflect::View::MONO ); + DEFLECT_API void finishFrameForSource( size_t sourceIndex ); /** Does the Buffer have a new complete mono frame (from all sources) */ DEFLECT_API bool hasCompleteMonoFrame() const; diff --git a/deflect/ServerWorker.cpp b/deflect/ServerWorker.cpp index 5da1a94..5a09526 100644 --- a/deflect/ServerWorker.cpp +++ b/deflect/ServerWorker.cpp @@ -235,7 +235,7 @@ void ServerWorker::_handleMessage( const MessageHeader& messageHeader, break; case MESSAGE_TYPE_PIXELSTREAM_FINISH_FRAME: - emit receivedFrameFinished( _streamId, _sourceId, _activeView ); + emit receivedFrameFinished( _streamId, _sourceId ); break; case MESSAGE_TYPE_PIXELSTREAM: diff --git a/deflect/ServerWorker.h b/deflect/ServerWorker.h index 1ee8e49..809e8ca 100644 --- a/deflect/ServerWorker.h +++ b/deflect/ServerWorker.h @@ -74,8 +74,7 @@ public slots: void receivedSegment( QString uri, size_t sourceIndex, deflect::Segment segment, deflect::View view ); - void receivedFrameFinished( QString uri, size_t sourceIndex, - deflect::View view ); + void receivedFrameFinished( QString uri, size_t sourceIndex ); void registerToEvents( QString uri, bool exclusive, deflect::EventReceiver* receiver ); diff --git a/deflect/SourceBuffer.cpp b/deflect/SourceBuffer.cpp index ea05ca7..7e89e07 100644 --- a/deflect/SourceBuffer.cpp +++ b/deflect/SourceBuffer.cpp @@ -71,6 +71,11 @@ FrameIndex SourceBuffer::getBackFrameIndex( const View view ) const }; } +bool SourceBuffer::isBackFrameEmpty( const View view ) const +{ + return _getQueue( view ).back().empty(); +} + void SourceBuffer::pop( const View view ) { _getQueue( view ).pop(); diff --git a/deflect/SourceBuffer.h b/deflect/SourceBuffer.h index 0862938..65b9cfd 100644 --- a/deflect/SourceBuffer.h +++ b/deflect/SourceBuffer.h @@ -66,6 +66,9 @@ class SourceBuffer /** @return the frame index of the back of the buffer for a given view. */ FrameIndex getBackFrameIndex( View view ) const; + /** @return true if the back frame of the given view has no segments. */ + bool isBackFrameEmpty( View view ) const; + /** Insert a segment into the back frame of the appropriate queue. */ void insert( const Segment& segment, const View view ); diff --git a/doc/StereoStreaming.md b/doc/StereoStreaming.md index cc7a4f0..3f05095 100644 --- a/doc/StereoStreaming.md +++ b/doc/StereoStreaming.md @@ -31,7 +31,7 @@ payload. This message is silently ignored by older Servers. ## Examples -Example of a stereo 3D client application: +Example of a stereo 3D client application using the blocking Stream API: deflect::Stream stream( ... ); @@ -43,19 +43,55 @@ Example of a stereo 3D client application: deflect::ImageWrapper leftImage( data, width, height, deflect::RGBA ); leftImage.view = deflect::View::LEFT_EYE; - deflectStream->send( leftImage ) && deflectStream->finishFrame(); + deflectStream->send( leftImage ); /** ...render right image... */ deflect::ImageWrapper rightImage( data, width, height, deflect::RGBA ); rightImage.view = deflect::View::RIGHT_EYE; - deflectStream->send( rightImage ) && deflectStream->finishFrame(); + deflectStream->send( rightImage ); + + deflectStream->finishFrame(); + + /** ...synchronize with other render clients (network barrier)... */ + } + +Example of a stereo 3D client application using the asynchronous Stream API: + + deflect::Stream stream( ... ); + + /** ...synchronize start with other render clients (network barrier)... */ + + delfect::Stream::Future leftFuture, rightFuture; + leftFuture = deflect::qt::make_ready_future( true ); + rightFuture = deflect::qt::make_ready_future( true ); + + ImageData leftData, rightData; // must remain valid until sending completes + + renderLoop() + { + if( !leftFuture.valid() || !leftFuture.get( )) + return; + + /** ...render left image... */ + + deflect::ImageWrapper leftImage( leftData, width, height, deflect::RGBA ); + leftImage.view = deflect::View::LEFT_EYE; + leftFuture = deflectStream->asyncSend( leftImage ); + + if( !rightFuture.valid() || !rightFuture.get( )) + return; + + /** ...render right image... */ + + deflect::ImageWrapper rightImage( rightData, width, height, deflect::RGBA ); + rightImage.view = deflect::View::RIGHT_EYE; + rightFuture = deflectStream->send( rightImage ); /** ...synchronize with other render clients (network barrier)... */ } -For a more complete example, please refer to the SimpleStreamer application -source code. +For a complete code example, please refer to the SimpleStreamer application. ## Issues diff --git a/tests/cpp/ReceiveBufferTests.cpp b/tests/cpp/ReceiveBufferTests.cpp index dab60fe..bf0bf35 100644 --- a/tests/cpp/ReceiveBufferTests.cpp +++ b/tests/cpp/ReceiveBufferTests.cpp @@ -322,7 +322,7 @@ void _insert( deflect::ReceiveBuffer& buffer, const size_t sourceIndex, { for( const auto& segment : frame ) buffer.insert( segment, sourceIndex, view ); - buffer.finishFrameForSource( sourceIndex, view ); + buffer.finishFrameForSource( sourceIndex ); } void _testStereoBuffer( deflect::ReceiveBuffer& buffer ) @@ -360,6 +360,29 @@ BOOST_AUTO_TEST_CASE( TestStereoOneSource ) _testStereoBuffer( buffer ); } +BOOST_AUTO_TEST_CASE( TestStereoSingleFinishFrame ) +{ + const size_t sourceIndex = 46; + + deflect::ReceiveBuffer buffer; + buffer.addSource( sourceIndex ); + + const deflect::Segments testSegments = generateTestSegments(); + + for( const auto& segment : testSegments ) + buffer.insert( segment, sourceIndex, deflect::View::LEFT_EYE ); + BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); + + for( const auto& segment : testSegments ) + buffer.insert( segment, sourceIndex, deflect::View::RIGHT_EYE ); + BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); + + buffer.finishFrameForSource( sourceIndex ); + BOOST_CHECK( buffer.hasCompleteStereoFrame( )); + + _testStereoBuffer( buffer ); +} + BOOST_AUTO_TEST_CASE( TestStereoTwoSourcesScreenSpaceSplit ) { const size_t sourceIndex1 = 46; From e3a2e668b0ecbdf4e70d49a7b1ee3e23e7edf4ac Mon Sep 17 00:00:00 2001 From: Raphael Dumusc Date: Thu, 16 Feb 2017 15:51:05 +0100 Subject: [PATCH 7/7] CR#2: various style cleanups --- CMakeLists.txt | 2 +- apps/SimpleStreamer/main.cpp | 6 +-- deflect/Frame.h | 2 +- deflect/FrameDispatcher.cpp | 30 ++++++------ deflect/FrameDispatcher.h | 4 +- deflect/ImageWrapper.h | 4 +- deflect/ReceiveBuffer.cpp | 69 ++++++-------------------- deflect/ReceiveBuffer.h | 15 +++--- deflect/Server.h | 4 +- deflect/ServerWorker.cpp | 4 +- deflect/SourceBuffer.cpp | 62 ++++-------------------- deflect/SourceBuffer.h | 19 ++++---- deflect/StreamPrivate.cpp | 4 +- deflect/types.h | 9 +++- doc/StereoStreaming.md | 10 ++-- tests/cpp/ReceiveBufferTests.cpp | 83 +++++++++++++++++--------------- 16 files changed, 128 insertions(+), 199 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 81d4286..b018c23 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ -# Copyright (c) 2013-2016, EPFL/Blue Brain Project +# Copyright (c) 2013-2017, EPFL/Blue Brain Project # Raphael Dumusc # Daniel Nachbaur diff --git a/apps/SimpleStreamer/main.cpp b/apps/SimpleStreamer/main.cpp index 7022f66..c50e9a1 100644 --- a/apps/SimpleStreamer/main.cpp +++ b/apps/SimpleStreamer/main.cpp @@ -286,7 +286,7 @@ void display() glClear( GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT ); glutSolidTeapot( 1.f ); const auto leftImage = Image::readGlBuffer(); - success = send( leftImage, deflect::View::LEFT_EYE ); + success = send( leftImage, deflect::View::left_eye ); } if( deflectStereoStreamRight && !waitToStart ) { @@ -295,7 +295,7 @@ void display() glutSolidTeapot( 1.f ); const auto rightImage = Image::readGlBuffer(); success = (!deflectStereoStreamLeft || success) && - send( rightImage, deflect::View::RIGHT_EYE ); + send( rightImage, deflect::View::right_eye ); } } else @@ -303,7 +303,7 @@ void display() glClearColor( 0.5, 0.5, 0.5, 1.0 ); glClear( GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT ); glutSolidTeapot( 1.f ); - success = send( Image::readGlBuffer(), deflect::View::MONO ); + success = send( Image::readGlBuffer(), deflect::View::mono ); } glutSwapBuffers(); diff --git a/deflect/Frame.h b/deflect/Frame.h index 628c9c8..9bbf2d8 100644 --- a/deflect/Frame.h +++ b/deflect/Frame.h @@ -63,7 +63,7 @@ class Frame QString uri; /** The view to which this frame belongs. */ - View view = View::MONO; + View view = View::mono; /** Get the total dimensions of this frame. */ DEFLECT_API QSize computeDimensions() const; diff --git a/deflect/FrameDispatcher.cpp b/deflect/FrameDispatcher.cpp index 207a085..c979aff 100644 --- a/deflect/FrameDispatcher.cpp +++ b/deflect/FrameDispatcher.cpp @@ -57,15 +57,15 @@ class FrameDispatcher::Impl { FramePtr frame( new Frame ); frame->uri = uri; - frame->view = View::MONO; + frame->view = View::mono; ReceiveBuffer& buffer = streamBuffers[uri]; while( buffer.hasCompleteMonoFrame( )) - frame->segments = buffer.popFrame( View::MONO ); + frame->segments = buffer.popFrame( View::mono ); assert( !frame->segments.empty( )); // receiver will request a new frame once this frame was consumed - buffer.setAllowedToSend( false, View::MONO ); + buffer.setAllowedToSend( false, View::mono ); return frame; } @@ -73,25 +73,25 @@ class FrameDispatcher::Impl { FramePtr frameLeft( new Frame ); frameLeft->uri = uri; - frameLeft->view = View::LEFT_EYE; + frameLeft->view = View::left_eye; FramePtr frameRight( new Frame ); frameRight->uri = uri; - frameRight->view = View::RIGHT_EYE; + frameRight->view = View::right_eye; ReceiveBuffer& buffer = streamBuffers[uri]; while( buffer.hasCompleteStereoFrame( )) { - frameLeft->segments = buffer.popFrame( View::LEFT_EYE ); - frameRight->segments = buffer.popFrame( View::RIGHT_EYE ); + frameLeft->segments = buffer.popFrame( View::left_eye ); + frameRight->segments = buffer.popFrame( View::right_eye ); } assert( !frameLeft->segments.empty( )); assert( !frameRight->segments.empty( )); // receiver will request a new frame once this frame was consumed - buffer.setAllowedToSend( false, View::LEFT_EYE ); - buffer.setAllowedToSend( false, View::RIGHT_EYE ); + buffer.setAllowedToSend( false, View::left_eye ); + buffer.setAllowedToSend( false, View::right_eye ); return std::make_pair( std::move( frameLeft ), std::move( frameRight )); } @@ -150,11 +150,11 @@ void FrameDispatcher::processFrameFinished( const QString uri, return; } - if( buffer.isAllowedToSend( View::MONO ) && buffer.hasCompleteMonoFrame( )) + if( buffer.isAllowedToSend( View::mono ) && buffer.hasCompleteMonoFrame( )) emit sendFrame( _impl->consumeLatestMonoFrame( uri )); - if( buffer.isAllowedToSend( View::LEFT_EYE ) && - buffer.isAllowedToSend( View::RIGHT_EYE ) && + if( buffer.isAllowedToSend( View::left_eye ) && + buffer.isAllowedToSend( View::right_eye ) && buffer.hasCompleteStereoFrame( )) { const auto frames = _impl->consumeLatestStereoFrame( uri ); @@ -169,9 +169,9 @@ void FrameDispatcher::requestFrame( const QString uri ) return; ReceiveBuffer& buffer = _impl->streamBuffers[uri]; - buffer.setAllowedToSend( true, View::MONO ); - buffer.setAllowedToSend( true, View::LEFT_EYE ); - buffer.setAllowedToSend( true, View::RIGHT_EYE ); + buffer.setAllowedToSend( true, View::mono ); + buffer.setAllowedToSend( true, View::left_eye ); + buffer.setAllowedToSend( true, View::right_eye ); if( buffer.hasCompleteMonoFrame( )) emit sendFrame( _impl->consumeLatestMonoFrame( uri )); diff --git a/deflect/FrameDispatcher.h b/deflect/FrameDispatcher.h index caf1f9c..36c69d6 100644 --- a/deflect/FrameDispatcher.h +++ b/deflect/FrameDispatcher.h @@ -100,12 +100,12 @@ public slots: void processFrameFinished( QString uri, size_t sourceIndex ); /** - * Request the dispatching of a new frame for any stream (MONO/STEREO). + * Request the dispatching of a new frame for any stream (mono/stereo). * * A sendFrame() signal will be emitted for each of the view for which a * frame becomes available. * - * Stereo LEFT/RIGHT frames will only be be dispatched together when both + * Stereo left/right frames will only be be dispatched together when both * are available to ensure that the two eye channels remain synchronized. * * @param uri Identifier for the stream diff --git a/deflect/ImageWrapper.h b/deflect/ImageWrapper.h index 8a172ed..f2a21fd 100644 --- a/deflect/ImageWrapper.h +++ b/deflect/ImageWrapper.h @@ -121,10 +121,10 @@ struct ImageWrapper //@} /** - * The view that this image represents in stereo 3D streams. + * The view that this image represents. * @version 1.6 */ - View view = View::MONO; + View view = View::mono; /** * Get the number of bytes per pixel based on the pixelFormat. diff --git a/deflect/ReceiveBuffer.cpp b/deflect/ReceiveBuffer.cpp index 34261fa..fcb8d62 100644 --- a/deflect/ReceiveBuffer.cpp +++ b/deflect/ReceiveBuffer.cpp @@ -39,16 +39,15 @@ #include "ReceiveBuffer.h" -#include #include #include namespace { const size_t MAX_QUEUE_SIZE = 150; // stream blocked for ~5 seconds at 30Hz -const auto views = std::array{{ deflect::View::MONO, - deflect::View::LEFT_EYE, - deflect::View::RIGHT_EYE }}; +const auto views = std::array{{ deflect::View::mono, + deflect::View::left_eye, + deflect::View::right_eye }}; } namespace deflect @@ -78,7 +77,7 @@ size_t ReceiveBuffer::getSourceCount() const } void ReceiveBuffer::insert( const Segment& segment, const size_t sourceIndex, - const deflect::View view ) + const View view ) { assert( _sourceBuffers.count( sourceIndex )); @@ -108,10 +107,11 @@ bool ReceiveBuffer::hasCompleteMonoFrame() const assert( !_sourceBuffers.empty( )); // Check if all sources for Stream have reached the same index + const auto lastCompleteFrame = _getLastCompleteFrameIndex( View::mono ); for( const auto& kv : _sourceBuffers ) { const auto& buffer = kv.second; - if( buffer.getBackFrameIndex( View::MONO ) <= _lastFrameComplete ) + if( buffer.getBackFrameIndex( View::mono ) <= lastCompleteFrame ) return false; } return true; @@ -122,12 +122,15 @@ bool ReceiveBuffer::hasCompleteStereoFrame() const std::set leftSources; std::set rightSources; + const auto lastFrameLeft = _getLastCompleteFrameIndex( View::left_eye ); + const auto lastFrameRight = _getLastCompleteFrameIndex( View::right_eye ); + for( const auto& kv : _sourceBuffers ) { const auto& buffer = kv.second; - if( buffer.getBackFrameIndex( View::LEFT_EYE ) > _lastFrameCompleteLeft ) + if( buffer.getBackFrameIndex( View::left_eye ) > lastFrameLeft ) leftSources.insert( kv.first ); - if( buffer.getBackFrameIndex( View::RIGHT_EYE ) > _lastFrameCompleteRight ) + if( buffer.getBackFrameIndex( View::right_eye ) > lastFrameRight ) rightSources.insert( kv.first ); } @@ -168,64 +171,22 @@ Segments ReceiveBuffer::popFrame( const View view ) void ReceiveBuffer::setAllowedToSend( const bool enable, const View view ) { - switch( view ) - { - case View::MONO: - _allowedToSend = enable; - break; - case View::LEFT_EYE: - _allowedToSendLeft = enable; - break; - case View::RIGHT_EYE: - _allowedToSendRight = enable; - break; - }; + _allowedToSend[as_underlying_type(view)] = enable; } bool ReceiveBuffer::isAllowedToSend( const View view ) const { - switch( view ) - { - case View::MONO: - return _allowedToSend; - case View::LEFT_EYE: - return _allowedToSendLeft; - case View::RIGHT_EYE: - return _allowedToSendRight; - default: - throw std::invalid_argument( "no such view" ); // keep compiler happy - }; + return _allowedToSend[as_underlying_type(view)]; } FrameIndex ReceiveBuffer::_getLastCompleteFrameIndex( const View view ) const { - switch( view ) - { - case View::MONO: - return _lastFrameComplete; - case View::LEFT_EYE: - return _lastFrameCompleteLeft; - case View::RIGHT_EYE: - return _lastFrameCompleteRight; - default: - throw std::invalid_argument( "no such view" ); // keep compiler happy - }; + return _lastFrameComplete[as_underlying_type(view)]; } void ReceiveBuffer::_incrementLastFrameComplete( const View view ) { - switch( view ) - { - case View::MONO: - ++_lastFrameComplete; - break; - case View::LEFT_EYE: - ++_lastFrameCompleteLeft; - break; - case View::RIGHT_EYE: - ++_lastFrameCompleteRight; - break; - }; + ++_lastFrameComplete[as_underlying_type(view)]; } } diff --git a/deflect/ReceiveBuffer.h b/deflect/ReceiveBuffer.h index efbfad1..e0bee78 100644 --- a/deflect/ReceiveBuffer.h +++ b/deflect/ReceiveBuffer.h @@ -47,6 +47,7 @@ #include +#include #include namespace deflect @@ -85,7 +86,7 @@ class ReceiveBuffer * @param view in which the segment should be inserted */ DEFLECT_API void insert( const Segment& segment, size_t sourceIndex, - deflect::View view = deflect::View::MONO ); + View view = View::mono ); /** * Call when the source has finished sending segments for the current frame. @@ -104,7 +105,7 @@ class ReceiveBuffer * Get the finished frame. * @return A collection of segments that form a frame */ - DEFLECT_API Segments popFrame( View view = deflect::View::MONO ); + DEFLECT_API Segments popFrame( View view = View::mono ); /** Allow this buffer to be used by the next FrameDispatcher::sendLatestFrame */ DEFLECT_API void setAllowedToSend( bool enable, View view ); @@ -115,13 +116,11 @@ class ReceiveBuffer private: std::map _sourceBuffers; - FrameIndex _lastFrameComplete = 0u; - FrameIndex _lastFrameCompleteLeft = 0u; - FrameIndex _lastFrameCompleteRight = 0u; + /** The current indices of the mono/left/right frame for this source. */ + std::array _lastFrameComplete = { { 0u, 0u, 0u } }; - bool _allowedToSend = false; - bool _allowedToSendLeft = false; - bool _allowedToSendRight = false; + /** Is the mono/left/right channel allowed to send. */ + std::array _allowedToSend = { { false, false, false } }; FrameIndex _getLastCompleteFrameIndex( View view ) const; void _incrementLastFrameComplete( View view ); diff --git a/deflect/Server.h b/deflect/Server.h index b7e77e2..2f596ea 100644 --- a/deflect/Server.h +++ b/deflect/Server.h @@ -82,10 +82,10 @@ public slots: * Request the dispatching of the next frame for a given pixel stream. * * A receivedFrame() signal will subsequently be emitted for each of the - * view(s) (MONO or STEREO) for which a frame is or becomes available. + * view(s) (mono or stereo) for which a frame is or becomes available. * * To ensure that the two eye channels remain synchronized, stereo - * LEFT/RIGHT frames are dispatched together only when both are available. + * left/right frames are dispatched together only when both are available. * * @param uri Identifier for the stream */ diff --git a/deflect/ServerWorker.cpp b/deflect/ServerWorker.cpp index 5a09526..a7c86e4 100644 --- a/deflect/ServerWorker.cpp +++ b/deflect/ServerWorker.cpp @@ -61,7 +61,7 @@ ServerWorker::ServerWorker( const int socketDescriptor ) , _sourceId( socketDescriptor ) , _clientProtocolVersion( NETWORK_PROTOCOL_VERSION ) , _registeredToEvents( false ) - , _activeView( View::MONO ) + , _activeView( View::mono ) { if( !_tcpSocket->setSocketDescriptor( socketDescriptor )) { @@ -257,7 +257,7 @@ void ServerWorker::_handleMessage( const MessageHeader& messageHeader, case MESSAGE_TYPE_IMAGE_VIEW: { const auto view = reinterpret_cast( byteArray.data( )); - if( *view >= deflect::View::MONO && *view <= deflect::View::RIGHT_EYE ) + if( *view >= View::mono && *view <= View::right_eye ) _activeView = *view; break; } diff --git a/deflect/SourceBuffer.cpp b/deflect/SourceBuffer.cpp index 7e89e07..99717b8 100644 --- a/deflect/SourceBuffer.cpp +++ b/deflect/SourceBuffer.cpp @@ -46,9 +46,9 @@ namespace deflect SourceBuffer::SourceBuffer() { - _segmentsMono.push( Segments( )); - _segmentsLeft.push( Segments( )); - _segmentsRight.push( Segments( )); + _getQueue( View::mono ).push( Segments( )); + _getQueue( View::left_eye ).push( Segments( )); + _getQueue( View::right_eye ).push( Segments( )); } const Segments& SourceBuffer::getSegments( const View view ) const @@ -58,17 +58,7 @@ const Segments& SourceBuffer::getSegments( const View view ) const FrameIndex SourceBuffer::getBackFrameIndex( const View view ) const { - switch( view ) - { - case View::MONO: - return _backFrameIndexMono; - case View::LEFT_EYE: - return _backFrameIndexLeft; - case View::RIGHT_EYE: - return _backFrameIndexRight; - default: - throw std::invalid_argument( "no such view" ); // keep compiler happy - }; + return _backFrameIndex[as_underlying_type(view)]; } bool SourceBuffer::isBackFrameEmpty( const View view ) const @@ -84,22 +74,10 @@ void SourceBuffer::pop( const View view ) void SourceBuffer::push( const View view ) { _getQueue( view ).push( Segments( )); - - switch( view ) - { - case View::MONO: - ++_backFrameIndexMono; - break; - case View::LEFT_EYE: - ++_backFrameIndexLeft; - break; - case View::RIGHT_EYE: - ++_backFrameIndexRight; - break; - }; + ++_backFrameIndex[as_underlying_type(view)]; } -void SourceBuffer::insert( const Segment& segment, const deflect::View view ) +void SourceBuffer::insert( const Segment& segment, const View view ) { _getQueue( view ).back().push_back( segment ); } @@ -109,35 +87,15 @@ size_t SourceBuffer::getQueueSize( const View view ) const return _getQueue( view ).size(); } -std::queue& SourceBuffer::_getQueue( const deflect::View view ) +std::queue& SourceBuffer::_getQueue( const View view ) { - switch( view ) - { - case View::MONO: - return _segmentsMono; - case View::LEFT_EYE: - return _segmentsLeft; - case View::RIGHT_EYE: - return _segmentsRight; - default: - throw std::invalid_argument( "no such view" ); // keep compiler happy - }; + return _segments[as_underlying_type(view)]; } const std::queue& -SourceBuffer::_getQueue( const deflect::View view ) const +SourceBuffer::_getQueue( const View view ) const { - switch( view ) - { - case View::MONO: - return _segmentsMono; - case View::LEFT_EYE: - return _segmentsLeft; - case View::RIGHT_EYE: - return _segmentsRight; - default: - throw std::invalid_argument( "no such view" ); // keep compiler happy - }; + return _segments[as_underlying_type(view)]; } } diff --git a/deflect/SourceBuffer.h b/deflect/SourceBuffer.h index 65b9cfd..63c1f69 100644 --- a/deflect/SourceBuffer.h +++ b/deflect/SourceBuffer.h @@ -44,6 +44,7 @@ #include #include +#include #include namespace deflect @@ -70,25 +71,23 @@ class SourceBuffer bool isBackFrameEmpty( View view ) const; /** Insert a segment into the back frame of the appropriate queue. */ - void insert( const Segment& segment, const View view ); + void insert( const Segment& segment, View view ); /** Push a new frame to the back of given view. */ - void push( const View view ); + void push( View view ); /** Pop the front frame of the buffer for the given view. */ - void pop( const View view ); + void pop( View view ); /** @return the size of the queue for the given view. */ - size_t getQueueSize( const View view ) const; + size_t getQueueSize( View view ) const; private: - /** The collections of segments for each view. */ - std::queue _segmentsMono, _segmentsLeft, _segmentsRight; + /** The collections of segments for each mono/left/right view. */ + std::queue _segments[3]; - /** The current indices of the frame for this source. */ - FrameIndex _backFrameIndexMono = 0u; - FrameIndex _backFrameIndexLeft = 0u; - FrameIndex _backFrameIndexRight = 0u; + /** The current indices of the mono/left/right frame for this source. */ + std::array _backFrameIndex = { { 0u, 0u, 0u } }; std::queue& _getQueue( View view ); const std::queue& _getQueue( View view ) const; diff --git a/deflect/StreamPrivate.cpp b/deflect/StreamPrivate.cpp index 8e002b0..85472ea 100644 --- a/deflect/StreamPrivate.cpp +++ b/deflect/StreamPrivate.cpp @@ -141,7 +141,9 @@ bool StreamPrivate::send( const ImageWrapper& image ) return false; } - sendImageView( image.view ); + if( !sendImageView( image.view )) + return false; + const auto sendFunc = std::bind( &StreamPrivate::sendPixelStreamSegment, this, std::placeholders::_1 ); return imageSegmenter.generate( image, sendFunc ); diff --git a/deflect/types.h b/deflect/types.h index 6815997..31bd6d6 100644 --- a/deflect/types.h +++ b/deflect/types.h @@ -50,7 +50,14 @@ namespace deflect { /** The different types of view. */ -enum class View : std::int8_t { MONO, LEFT_EYE, RIGHT_EYE }; +enum class View : std::uint8_t { mono, left_eye, right_eye }; + +/** Cast an enum class value to its underlying type. */ +template +constexpr typename std::underlying_type::type as_underlying_type( E e ) +{ + return static_cast::type>( e ); +} class EventReceiver; class Frame; diff --git a/doc/StereoStreaming.md b/doc/StereoStreaming.md index 3f05095..a2240d2 100644 --- a/doc/StereoStreaming.md +++ b/doc/StereoStreaming.md @@ -16,7 +16,7 @@ This document describes the stereo streaming support introduced in Deflect 0.13. New view enum in deflect/types.h: - enum class View : std::int8_t { MONO, LEFT_EYE, RIGHT_EYE }; + enum class View : std::int8_t { mono, left_eye, right_eye }; On the client side, no changes to the Stream API. The ImageWrapper takes an additional View parameter. @@ -42,13 +42,13 @@ Example of a stereo 3D client application using the blocking Stream API: /** ...render left image... */ deflect::ImageWrapper leftImage( data, width, height, deflect::RGBA ); - leftImage.view = deflect::View::LEFT_EYE; + leftImage.view = deflect::View::left_eye; deflectStream->send( leftImage ); /** ...render right image... */ deflect::ImageWrapper rightImage( data, width, height, deflect::RGBA ); - rightImage.view = deflect::View::RIGHT_EYE; + rightImage.view = deflect::View::right_eye; deflectStream->send( rightImage ); deflectStream->finishFrame(); @@ -76,7 +76,7 @@ Example of a stereo 3D client application using the asynchronous Stream API: /** ...render left image... */ deflect::ImageWrapper leftImage( leftData, width, height, deflect::RGBA ); - leftImage.view = deflect::View::LEFT_EYE; + leftImage.view = deflect::View::left_eye; leftFuture = deflectStream->asyncSend( leftImage ); if( !rightFuture.valid() || !rightFuture.get( )) @@ -85,7 +85,7 @@ Example of a stereo 3D client application using the asynchronous Stream API: /** ...render right image... */ deflect::ImageWrapper rightImage( rightData, width, height, deflect::RGBA ); - rightImage.view = deflect::View::RIGHT_EYE; + rightImage.view = deflect::View::right_eye; rightFuture = deflectStream->send( rightImage ); /** ...synchronize with other render clients (network barrier)... */ diff --git a/tests/cpp/ReceiveBufferTests.cpp b/tests/cpp/ReceiveBufferTests.cpp index bf0bf35..59234da 100644 --- a/tests/cpp/ReceiveBufferTests.cpp +++ b/tests/cpp/ReceiveBufferTests.cpp @@ -79,24 +79,24 @@ BOOST_AUTO_TEST_CASE( TestAllowedToSend ) { deflect::ReceiveBuffer buffer; - BOOST_CHECK( !buffer.isAllowedToSend( deflect::View::MONO )); - BOOST_CHECK( !buffer.isAllowedToSend( deflect::View::LEFT_EYE )); - BOOST_CHECK( !buffer.isAllowedToSend( deflect::View::RIGHT_EYE )); - - buffer.setAllowedToSend( true, deflect::View::MONO ); - BOOST_CHECK( buffer.isAllowedToSend( deflect::View::MONO )); - buffer.setAllowedToSend( false, deflect::View::MONO ); - BOOST_CHECK( !buffer.isAllowedToSend( deflect::View::MONO )); - - buffer.setAllowedToSend( true, deflect::View::LEFT_EYE ); - BOOST_CHECK( buffer.isAllowedToSend( deflect::View::LEFT_EYE )); - buffer.setAllowedToSend( false, deflect::View::LEFT_EYE ); - BOOST_CHECK( !buffer.isAllowedToSend( deflect::View::LEFT_EYE )); - - buffer.setAllowedToSend( true, deflect::View::RIGHT_EYE ); - BOOST_CHECK( buffer.isAllowedToSend( deflect::View::RIGHT_EYE )); - buffer.setAllowedToSend( false, deflect::View::RIGHT_EYE ); - BOOST_CHECK( !buffer.isAllowedToSend( deflect::View::RIGHT_EYE )); + BOOST_CHECK( !buffer.isAllowedToSend( deflect::View::mono )); + BOOST_CHECK( !buffer.isAllowedToSend( deflect::View::left_eye )); + BOOST_CHECK( !buffer.isAllowedToSend( deflect::View::right_eye )); + + buffer.setAllowedToSend( true, deflect::View::mono ); + BOOST_CHECK( buffer.isAllowedToSend( deflect::View::mono )); + buffer.setAllowedToSend( false, deflect::View::mono ); + BOOST_CHECK( !buffer.isAllowedToSend( deflect::View::mono )); + + buffer.setAllowedToSend( true, deflect::View::left_eye ); + BOOST_CHECK( buffer.isAllowedToSend( deflect::View::left_eye )); + buffer.setAllowedToSend( false, deflect::View::left_eye ); + BOOST_CHECK( !buffer.isAllowedToSend( deflect::View::left_eye )); + + buffer.setAllowedToSend( true, deflect::View::right_eye ); + BOOST_CHECK( buffer.isAllowedToSend( deflect::View::right_eye )); + buffer.setAllowedToSend( false, deflect::View::right_eye ); + BOOST_CHECK( !buffer.isAllowedToSend( deflect::View::right_eye )); } BOOST_AUTO_TEST_CASE( TestCompleteAFrame ) @@ -114,14 +114,17 @@ BOOST_AUTO_TEST_CASE( TestCompleteAFrame ) buffer.insert( segment, sourceIndex ); BOOST_CHECK( !buffer.hasCompleteMonoFrame( )); + BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); buffer.finishFrameForSource( sourceIndex ); BOOST_CHECK( buffer.hasCompleteMonoFrame( )); + BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); deflect::Segments segments = buffer.popFrame(); BOOST_CHECK_EQUAL( segments.size(), 1 ); BOOST_CHECK( !buffer.hasCompleteMonoFrame( )); + BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); deflect::Frame frame; frame.segments = segments; @@ -327,11 +330,11 @@ void _insert( deflect::ReceiveBuffer& buffer, const size_t sourceIndex, void _testStereoBuffer( deflect::ReceiveBuffer& buffer ) { - const auto leftSegments = buffer.popFrame( deflect::View::LEFT_EYE ); + const auto leftSegments = buffer.popFrame( deflect::View::left_eye ); BOOST_CHECK_EQUAL( leftSegments.size(), 4 ); BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); - const auto rightSegments = buffer.popFrame( deflect::View::RIGHT_EYE ); + const auto rightSegments = buffer.popFrame( deflect::View::right_eye ); BOOST_CHECK_EQUAL( rightSegments.size(), 4 ); BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); @@ -351,10 +354,10 @@ BOOST_AUTO_TEST_CASE( TestStereoOneSource ) deflect::Segments testSegments = generateTestSegments(); - _insert( buffer, sourceIndex, testSegments, deflect::View::LEFT_EYE ); + _insert( buffer, sourceIndex, testSegments, deflect::View::left_eye ); BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); - _insert( buffer, sourceIndex, testSegments, deflect::View::RIGHT_EYE ); + _insert( buffer, sourceIndex, testSegments, deflect::View::right_eye ); BOOST_CHECK( buffer.hasCompleteStereoFrame( )); _testStereoBuffer( buffer ); @@ -370,11 +373,11 @@ BOOST_AUTO_TEST_CASE( TestStereoSingleFinishFrame ) const deflect::Segments testSegments = generateTestSegments(); for( const auto& segment : testSegments ) - buffer.insert( segment, sourceIndex, deflect::View::LEFT_EYE ); + buffer.insert( segment, sourceIndex, deflect::View::left_eye ); BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); for( const auto& segment : testSegments ) - buffer.insert( segment, sourceIndex, deflect::View::RIGHT_EYE ); + buffer.insert( segment, sourceIndex, deflect::View::right_eye ); BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); buffer.finishFrameForSource( sourceIndex ); @@ -398,14 +401,14 @@ BOOST_AUTO_TEST_CASE( TestStereoTwoSourcesScreenSpaceSplit ) const auto segmentsScreen2 = deflect::Segments{ testSegments[2], testSegments[3] }; - _insert( buffer, sourceIndex1, segmentsScreen1, deflect::View::LEFT_EYE ); + _insert( buffer, sourceIndex1, segmentsScreen1, deflect::View::left_eye ); BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); - _insert( buffer, sourceIndex1, segmentsScreen1, deflect::View::RIGHT_EYE ); + _insert( buffer, sourceIndex1, segmentsScreen1, deflect::View::right_eye ); BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); - _insert( buffer, sourceIndex2, segmentsScreen2, deflect::View::LEFT_EYE ); + _insert( buffer, sourceIndex2, segmentsScreen2, deflect::View::left_eye ); BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); - _insert( buffer, sourceIndex2, segmentsScreen2, deflect::View::RIGHT_EYE ); + _insert( buffer, sourceIndex2, segmentsScreen2, deflect::View::right_eye ); BOOST_CHECK( buffer.hasCompleteStereoFrame( )); _testStereoBuffer( buffer ); @@ -422,9 +425,9 @@ BOOST_AUTO_TEST_CASE( TestStereoTwoSourcesStereoSplit ) const auto testSegments = generateTestSegments(); - _insert( buffer, sourceIndex1, testSegments, deflect::View::LEFT_EYE ); + _insert( buffer, sourceIndex1, testSegments, deflect::View::left_eye ); BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); - _insert( buffer, sourceIndex2, testSegments, deflect::View::RIGHT_EYE ); + _insert( buffer, sourceIndex2, testSegments, deflect::View::right_eye ); BOOST_CHECK( buffer.hasCompleteStereoFrame( )); _testStereoBuffer( buffer ); @@ -449,29 +452,29 @@ BOOST_AUTO_TEST_CASE( TestStereoFourSourcesScreenSpaceAndStereoSplit ) const auto segmentsScreen2 = deflect::Segments{ testSegments[2], testSegments[3] }; - _insert( buffer, sourceIndex1, segmentsScreen1, deflect::View::LEFT_EYE ); + _insert( buffer, sourceIndex1, segmentsScreen1, deflect::View::left_eye ); BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); - _insert( buffer, sourceIndex2, segmentsScreen1, deflect::View::RIGHT_EYE ); + _insert( buffer, sourceIndex2, segmentsScreen1, deflect::View::right_eye ); BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); - _insert( buffer, sourceIndex3, segmentsScreen2, deflect::View::LEFT_EYE ); + _insert( buffer, sourceIndex3, segmentsScreen2, deflect::View::left_eye ); BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); - _insert( buffer, sourceIndex4, segmentsScreen2, deflect::View::RIGHT_EYE ); + _insert( buffer, sourceIndex4, segmentsScreen2, deflect::View::right_eye ); BOOST_CHECK( buffer.hasCompleteStereoFrame( )); _testStereoBuffer( buffer ); // Random insertion order - _insert( buffer, sourceIndex1, segmentsScreen1, deflect::View::LEFT_EYE ); + _insert( buffer, sourceIndex1, segmentsScreen1, deflect::View::left_eye ); BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); - _insert( buffer, sourceIndex3, segmentsScreen2, deflect::View::LEFT_EYE ); + _insert( buffer, sourceIndex3, segmentsScreen2, deflect::View::left_eye ); BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); - _insert( buffer, sourceIndex2, segmentsScreen1, deflect::View::RIGHT_EYE ); + _insert( buffer, sourceIndex2, segmentsScreen1, deflect::View::right_eye ); BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); - _insert( buffer, sourceIndex1, segmentsScreen1, deflect::View::LEFT_EYE ); + _insert( buffer, sourceIndex1, segmentsScreen1, deflect::View::left_eye ); BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); - _insert( buffer, sourceIndex2, segmentsScreen1, deflect::View::RIGHT_EYE ); + _insert( buffer, sourceIndex2, segmentsScreen1, deflect::View::right_eye ); BOOST_CHECK( !buffer.hasCompleteStereoFrame( )); - _insert( buffer, sourceIndex4, segmentsScreen2, deflect::View::RIGHT_EYE ); + _insert( buffer, sourceIndex4, segmentsScreen2, deflect::View::right_eye ); BOOST_CHECK( buffer.hasCompleteStereoFrame( )); _testStereoBuffer( buffer );