From 58a5fca6c865b480368d4bde519deb27a820cf39 Mon Sep 17 00:00:00 2001 From: Raphael Dumusc Date: Thu, 2 Mar 2017 11:15:36 +0100 Subject: [PATCH 1/2] Optional JPEG chroma subsampling, segments can be decoded to YUV This feature requires LibJpegTurbo>=1.4, which introduced tjDecompressToYUV2(). The original tjDecompressToYUV() had a fixed padding value of 4, resulting in unexpected decoded image sizes. --- .gitexternals | 2 +- CMakeLists.txt | 6 +- deflect/ImageJpegCompressor.cpp | 30 ++++-- deflect/ImageJpegCompressor.h | 6 +- deflect/ImageJpegDecompressor.cpp | 93 ++++++++++++++---- deflect/ImageJpegDecompressor.h | 42 +++++++- deflect/ImageSegmenter.cpp | 4 +- deflect/ImageWrapper.cpp | 5 +- deflect/ImageWrapper.h | 2 + deflect/Segment.h | 2 +- deflect/SegmentDecoder.cpp | 83 ++++++++++++++-- deflect/SegmentDecoder.h | 29 +++++- deflect/SegmentParameters.h | 17 +++- deflect/types.h | 10 +- doc/Changelog.md | 5 + doc/ChromaSubsampling.md | 43 +++++++++ tests/cpp/SegmentDecoderTests.cpp | 154 ++++++++++++++++++++++++++++-- 17 files changed, 477 insertions(+), 56 deletions(-) create mode 100644 doc/ChromaSubsampling.md diff --git a/.gitexternals b/.gitexternals index 7476bdf..411b9ec 100644 --- a/.gitexternals +++ b/.gitexternals @@ -1,2 +1,2 @@ # -*- mode: cmake -*- -# CMake/common https://github.com/Eyescale/CMake.git 3d5d284 +# CMake/common https://github.com/Eyescale/CMake.git 770b264 diff --git a/CMakeLists.txt b/CMakeLists.txt index 69addca..ca8df8a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,11 @@ include(Common) common_find_package(Boost COMPONENTS program_options unit_test_framework) common_find_package(GLUT) -common_find_package(LibJpegTurbo REQUIRED) +common_find_package(LibJpegTurbo 1.4) +if(NOT LibJpegTurbo_FOUND) + common_find_package(LibJpegTurbo 1.2 REQUIRED) + list(APPEND COMMON_FIND_PACKAGE_DEFINES DEFLECT_USE_LEGACY_LIBJPEGTURBO) +endif() common_find_package(OpenGL) common_find_package(Qt5Concurrent REQUIRED SYSTEM) common_find_package(Qt5Core REQUIRED) diff --git a/deflect/ImageJpegCompressor.cpp b/deflect/ImageJpegCompressor.cpp index a4c438c..4aedec4 100644 --- a/deflect/ImageJpegCompressor.cpp +++ b/deflect/ImageJpegCompressor.cpp @@ -1,6 +1,6 @@ /*********************************************************************/ -/* Copyright (c) 2013, EPFL/Blue Brain Project */ -/* Raphael Dumusc */ +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project */ +/* Raphael Dumusc */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -55,7 +55,7 @@ ImageJpegCompressor::~ImageJpegCompressor() tjDestroy(_tjHandle); } -int getTurboJpegFormat(const PixelFormat pixelFormat) +int _getTurboJpegFormat(const PixelFormat pixelFormat) { switch (pixelFormat) { @@ -77,12 +77,28 @@ int getTurboJpegFormat(const PixelFormat pixelFormat) } } +int _getTurboJpegSubsamp(const ChromaSubsampling subsampling) +{ + switch (subsampling) + { + case ChromaSubsampling::YUV444: + return TJSAMP_444; + case ChromaSubsampling::YUV422: + return TJSAMP_422; + case ChromaSubsampling::YUV420: + return TJSAMP_420; + default: + std::cerr << "unknown subsampling format" << std::endl; + return TJSAMP_444; + } +} + QByteArray ImageJpegCompressor::computeJpeg(const ImageWrapper& sourceImage, const QRect& imageRegion) { // tjCompress API is incorrect and takes a non-const input buffer, even // though it does not modify it. It can "safely" be cast to non-const - // pointer to comply to the incorrect API. + // pointer to comply with the incorrect API. unsigned char* tjSrcBuffer = (unsigned char*)sourceImage.data; tjSrcBuffer += imageRegion.y() * sourceImage.width * sourceImage.getBytesPerPixel(); @@ -92,10 +108,10 @@ QByteArray ImageJpegCompressor::computeJpeg(const ImageWrapper& sourceImage, // assume imageBuffer isn't padded const int tjPitch = sourceImage.width * sourceImage.getBytesPerPixel(); const int tjHeight = imageRegion.height(); - const int tjPixelFormat = getTurboJpegFormat(sourceImage.pixelFormat); - unsigned char* tjJpegBuf = 0; + const int tjPixelFormat = _getTurboJpegFormat(sourceImage.pixelFormat); + unsigned char* tjJpegBuf = nullptr; unsigned long tjJpegSize = 0; - const int tjJpegSubsamp = TJSAMP_444; + const int tjJpegSubsamp = _getTurboJpegSubsamp(sourceImage.subsampling); const int tjJpegQual = sourceImage.compressionQuality; const int tjFlags = 0; // or: TJFLAG_BOTTOMUP diff --git a/deflect/ImageJpegCompressor.h b/deflect/ImageJpegCompressor.h index 9bc3eb8..b254024 100644 --- a/deflect/ImageJpegCompressor.h +++ b/deflect/ImageJpegCompressor.h @@ -1,6 +1,6 @@ /*********************************************************************/ -/* Copyright (c) 2013, EPFL/Blue Brain Project */ -/* Raphael Dumusc */ +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project */ +/* Raphael Dumusc */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -74,4 +74,4 @@ class ImageJpegCompressor }; } -#endif // IMAGEJPEGCOMPRESSOR_H +#endif diff --git a/deflect/ImageJpegDecompressor.cpp b/deflect/ImageJpegDecompressor.cpp index a135aad..5f89a6d 100644 --- a/deflect/ImageJpegDecompressor.cpp +++ b/deflect/ImageJpegDecompressor.cpp @@ -1,6 +1,6 @@ /*********************************************************************/ -/* Copyright (c) 2013, EPFL/Blue Brain Project */ -/* Raphael Dumusc */ +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project */ +/* Raphael Dumusc */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -41,6 +41,24 @@ #include +namespace +{ +deflect::ChromaSubsampling _getSubsamp(const int tjJpegSubsamp) +{ + switch (tjJpegSubsamp) + { + case TJSAMP_444: + return deflect::ChromaSubsampling::YUV444; + case TJSAMP_422: + return deflect::ChromaSubsampling::YUV422; + case TJSAMP_420: + return deflect::ChromaSubsampling::YUV420; + default: + throw std::runtime_error("unsupported subsampling format"); + } +} +} + namespace deflect { ImageJpegDecompressor::ImageJpegDecompressor() @@ -53,29 +71,72 @@ ImageJpegDecompressor::~ImageJpegDecompressor() tjDestroy(_tjHandle); } -QByteArray ImageJpegDecompressor::decompress(const QByteArray& jpegData) +JpegHeader ImageJpegDecompressor::decompressHeader(const QByteArray& jpegData) { - // get information from header - int width, height, jpegSubsamp; + JpegHeader header; + int jpegSubsamp = -1; + +#ifdef TJ_NUMCS // introduced with tjDecompressHeader3() + int jpegColorspace = -1; + int err = + tjDecompressHeader3(_tjHandle, (unsigned char*)jpegData.data(), + (unsigned long)jpegData.size(), &header.width, + &header.height, &jpegSubsamp, &jpegColorspace); + if (err != 0 || jpegColorspace != TJCS_YCbCr) +#else int err = tjDecompressHeader2(_tjHandle, (unsigned char*)jpegData.data(), - (unsigned long)jpegData.size(), &width, - &height, &jpegSubsamp); + (unsigned long)jpegData.size(), &header.width, + &header.height, &jpegSubsamp); if (err != 0) +#endif throw std::runtime_error("libjpeg-turbo header decompression failed"); - // decompress image data - int pixelFormat = TJPF_RGBX; // Format for OpenGL texture (GL_RGBA) - int pitch = width * tjPixelSize[pixelFormat]; - int flags = TJ_FASTUPSAMPLE; - QByteArray decodedData(height * pitch, Qt::Uninitialized); + header.subsampling = _getSubsamp(jpegSubsamp); + return header; +} + +QByteArray ImageJpegDecompressor::decompress(const QByteArray& jpegData) +{ + const auto header = decompressHeader(jpegData); + const int pixelFormat = TJPF_RGBX; // Format for OpenGL texture (GL_RGBA) + const int pitch = header.width * tjPixelSize[pixelFormat]; + const int flags = TJ_FASTUPSAMPLE; + + QByteArray decodedData(header.height * pitch, Qt::Uninitialized); - err = tjDecompress2(_tjHandle, (unsigned char*)jpegData.data(), - (unsigned long)jpegData.size(), - (unsigned char*)decodedData.data(), width, pitch, - height, pixelFormat, flags); + int err = tjDecompress2(_tjHandle, (unsigned char*)jpegData.data(), + (unsigned long)jpegData.size(), + (unsigned char*)decodedData.data(), header.width, + pitch, header.height, pixelFormat, flags); if (err != 0) throw std::runtime_error("libjpeg-turbo image decompression failed"); return decodedData; } + +#ifndef DEFLECT_USE_LEGACY_LIBJPEGTURBO + +ImageJpegDecompressor::YUVData ImageJpegDecompressor::decompressToYUV( + const QByteArray& jpegData) +{ + const auto header = decompressHeader(jpegData); + const int pad = 1; // no padding + const int flags = 0; + const int jpegSubsamp = int(header.subsampling); + const auto decodedSize = + tjBufSizeYUV2(header.width, pad, header.height, jpegSubsamp); + + auto decodedData = QByteArray(decodedSize, Qt::Uninitialized); + + int err = tjDecompressToYUV2(_tjHandle, (unsigned char*)jpegData.data(), + (unsigned long)jpegData.size(), + (unsigned char*)decodedData.data(), + header.width, pad, header.height, flags); + if (err != 0) + throw std::runtime_error("libjpeg-turbo image decompression failed"); + + return std::make_pair(std::move(decodedData), header.subsampling); +} + +#endif } diff --git a/deflect/ImageJpegDecompressor.h b/deflect/ImageJpegDecompressor.h index ab750a5..13db414 100644 --- a/deflect/ImageJpegDecompressor.h +++ b/deflect/ImageJpegDecompressor.h @@ -1,6 +1,6 @@ /*********************************************************************/ -/* Copyright (c) 2013, EPFL/Blue Brain Project */ -/* Raphael Dumusc */ +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project */ +/* Raphael Dumusc */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -41,6 +41,8 @@ #define DEFLECT_IMAGEJPEGDECOMPRESSOR_H #include +#include +#include #include @@ -48,6 +50,16 @@ namespace deflect { +/** + * JPEG header information. + */ +struct JpegHeader +{ + int width = 0; + int height = 0; + ChromaSubsampling subsampling; +}; + /** * Decompress Jpeg compressed data. */ @@ -58,7 +70,16 @@ class ImageJpegDecompressor DEFLECT_API ~ImageJpegDecompressor(); /** - * Decompress a Jpeg image + * Decompress the header of a Jpeg image. + * + * @param jpegData The compressed Jpeg data + * @return The decompressed Jpeg header + * @throw std::runtime_error if a decompression error occured + */ + DEFLECT_API JpegHeader decompressHeader(const QByteArray& jpegData); + + /** + * Decompress a Jpeg image. * * @param jpegData The compressed Jpeg data * @return The decompressed image data in (GL_)RGBA format @@ -66,6 +87,21 @@ class ImageJpegDecompressor */ DEFLECT_API QByteArray decompress(const QByteArray& jpegData); +#ifndef DEFLECT_USE_LEGACY_LIBJPEGTURBO + + using YUVData = std::pair; + + /** + * Decompress a Jpeg image to YUV, skipping the YUV -> RGBA conversion step. + * + * @param jpegData The compressed Jpeg data + * @return The decompressed image data in YUV format + * @throw std::runtime_error if a decompression error occured + */ + DEFLECT_API YUVData decompressToYUV(const QByteArray& jpegData); + +#endif + private: /** libjpeg-turbo handle for decompression */ tjhandle _tjHandle; diff --git a/deflect/ImageSegmenter.cpp b/deflect/ImageSegmenter.cpp index 2123f75..3c6ed54 100644 --- a/deflect/ImageSegmenter.cpp +++ b/deflect/ImageSegmenter.cpp @@ -261,7 +261,9 @@ SegmentParametersList ImageSegmenter::_generateSegmentParameters( : lastSegmentWidth; p.height = (j < numSubdivisionsY - 1) ? _nominalSegmentHeight : lastSegmentHeight; - p.compressed = (image.compressionPolicy == COMPRESSION_ON); + p.dataType = (image.compressionPolicy == COMPRESSION_ON) + ? DataType::jpeg + : DataType::rgba; parameters.push_back(p); } diff --git a/deflect/ImageWrapper.cpp b/deflect/ImageWrapper.cpp index 18ed8b3..8cc6cc5 100644 --- a/deflect/ImageWrapper.cpp +++ b/deflect/ImageWrapper.cpp @@ -1,6 +1,6 @@ /*********************************************************************/ -/* Copyright (c) 2013, EPFL/Blue Brain Project */ -/* Raphael Dumusc */ +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project */ +/* Raphael Dumusc */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -57,6 +57,7 @@ ImageWrapper::ImageWrapper(const void* data_, const unsigned int width_, , y(y_) , compressionPolicy(COMPRESSION_AUTO) , compressionQuality(DEFAULT_COMPRESSION_QUALITY) + , subsampling(ChromaSubsampling::YUV444) { } diff --git a/deflect/ImageWrapper.h b/deflect/ImageWrapper.h index 7d104a2..6f82cbc 100644 --- a/deflect/ImageWrapper.h +++ b/deflect/ImageWrapper.h @@ -127,6 +127,8 @@ struct ImageWrapper unsigned int compressionQuality; /**< Compression quality (0 worst, 100 best, default: 75). @version 1.0 */ + ChromaSubsampling subsampling; /**< Chrominance sub-sampling. + (default: YUV444). @version 1.6 */ //@} /** diff --git a/deflect/Segment.h b/deflect/Segment.h index 624c88a..595a2e4 100644 --- a/deflect/Segment.h +++ b/deflect/Segment.h @@ -60,7 +60,7 @@ struct Segment QByteArray imageData; /** @internal raw, uncompressed source image, used for compression */ - const ImageWrapper* sourceImage; + const ImageWrapper* sourceImage = nullptr; }; } diff --git a/deflect/SegmentDecoder.cpp b/deflect/SegmentDecoder.cpp index 84e63fd..7b5c3fe 100644 --- a/deflect/SegmentDecoder.cpp +++ b/deflect/SegmentDecoder.cpp @@ -69,33 +69,99 @@ SegmentDecoder::~SegmentDecoder() { } -void _decodeSegment(ImageJpegDecompressor* decompressor, Segment* segment) +ChromaSubsampling SegmentDecoder::decodeType(const Segment& segment) { - if (!segment->parameters.compressed) + if (segment.parameters.dataType != DataType::jpeg) + throw std::runtime_error("Segment is not in JPEG format"); + + return _impl->decompressor.decompressHeader(segment.imageData).subsampling; +} + +size_t _getExpectedSize(const DataType dataType, + const SegmentParameters& params) +{ + const size_t imageSize = params.height * params.width; + switch (dataType) + { + case DataType::rgba: + return imageSize * 4; + case DataType::yuv444: + return imageSize * 3; + case DataType::yuv422: + return imageSize * 2; + case DataType::yuv420: + return imageSize + (imageSize >> 1); + default: + return 0; + }; +} + +void _decodeSegment(ImageJpegDecompressor* decompressor, Segment* segment, + const bool skipRgbConversion) +{ + if (segment->parameters.dataType != DataType::jpeg) return; QByteArray decodedData; + DataType dataType; try { - decodedData = decompressor->decompress(segment->imageData); +#ifndef DEFLECT_USE_LEGACY_LIBJPEGTURBO + if (skipRgbConversion) + { + const auto yuv = decompressor->decompressToYUV(segment->imageData); + decodedData = yuv.first; + switch (yuv.second) + { + case ChromaSubsampling::YUV444: + dataType = DataType::yuv444; + break; + case ChromaSubsampling::YUV422: + dataType = DataType::yuv422; + break; + case ChromaSubsampling::YUV420: + dataType = DataType::yuv420; + break; + default: + throw std::runtime_error("unexpected ChromaSubsampling mode"); + }; + } + else +#else + Q_UNUSED(skipRgbConversion); +#endif + { + decodedData = decompressor->decompress(segment->imageData); + dataType = DataType::rgba; + } } catch (const std::runtime_error&) { throw; } - const auto& params = segment->parameters; - if ((size_t)decodedData.size() != params.height * params.width * 4) + + const auto expectedSize = _getExpectedSize(dataType, segment->parameters); + if (size_t(decodedData.size()) != expectedSize) throw std::runtime_error("unexpected segment size"); segment->imageData = decodedData; - segment->parameters.compressed = false; + segment->parameters.dataType = dataType; } void SegmentDecoder::decode(Segment& segment) { - _decodeSegment(&_impl->decompressor, &segment); + _decodeSegment(&_impl->decompressor, &segment, false); } +#ifndef DEFLECT_USE_LEGACY_LIBJPEGTURBO + +void SegmentDecoder::decodeToYUV(Segment& segment) +{ + _decodeSegment(&_impl->decompressor, &segment, true); +} + +#endif + void SegmentDecoder::startDecoding(Segment& segment) { // drop frames if we're currently processing @@ -108,7 +174,8 @@ void SegmentDecoder::startDecoding(Segment& segment) } _impl->decodingFuture = - QtConcurrent::run(_decodeSegment, &_impl->decompressor, &segment); + QtConcurrent::run(_decodeSegment, &_impl->decompressor, &segment, + false); } void SegmentDecoder::waitDecoding() diff --git a/deflect/SegmentDecoder.h b/deflect/SegmentDecoder.h index f2f11fe..afde1ae 100644 --- a/deflect/SegmentDecoder.h +++ b/deflect/SegmentDecoder.h @@ -41,6 +41,7 @@ #define DEFLECT_SEGMENTDECODER_H #include +#include #include namespace deflect @@ -58,15 +59,37 @@ class SegmentDecoder DEFLECT_API ~SegmentDecoder(); /** - * Decode a segment. + * Decode the data type of a JPEG segment. + * + * @param segment The segment to decode. + * @throw std::runtime_error if a decompression error occured + */ + DEFLECT_API ChromaSubsampling decodeType(const Segment& segment); + + /** + * Decode a JPEG segment to RGB. * * @param segment The segment to decode. Upon success, its imageData member - * will hold the decompressed data and its "compressed" flag will be - * set to false. + * will hold the decompressed RGB image and its "dataType" flag will + * be set to DataType::rgba. * @throw std::runtime_error if a decompression error occured */ DEFLECT_API void decode(Segment& segment); +#ifndef DEFLECT_USE_LEGACY_LIBJPEGTURBO + + /** + * Decode a JPEG segment to YUV, skipping the YUV -> RGB step. + * + * @param segment The segment to decode. Upon success, its imageData member + * will hold the decompressed YUV image and its "dataType" flag will + * be set to the matching DataType::yuv4**. + * @throw std::runtime_error if a decompression error occured + */ + DEFLECT_API void decodeToYUV(Segment& segment); + +#endif + /** * Start decoding a segment. * diff --git a/deflect/SegmentParameters.h b/deflect/SegmentParameters.h index a632ae3..a6e2559 100644 --- a/deflect/SegmentParameters.h +++ b/deflect/SegmentParameters.h @@ -50,6 +50,19 @@ typedef unsigned __int32 uint32_t; namespace deflect { +/** + * The possible formats for segment data. + * @version 1.6 + */ +enum class DataType : std::uint8_t +{ + rgba = 0, // equivalent to old compressed=false property + jpeg = 1, // equivalent to old compressed=true property + yuv444, + yuv422, + yuv420 +}; + /** * Parameters for a Frame Segment. */ @@ -67,8 +80,8 @@ struct SegmentParameters uint32_t height = 0u; /**< The height in pixels. */ //@} - /** Is the image raw pixel data or compressed in jpeg format */ - bool compressed = true; + /** Data format of the Segment. @version 1.6 */ + DataType dataType = DataType::jpeg; }; } diff --git a/deflect/types.h b/deflect/types.h index 6715fd8..a7951fe 100644 --- a/deflect/types.h +++ b/deflect/types.h @@ -1,5 +1,5 @@ /*********************************************************************/ -/* Copyright (c) 2014-2015, EPFL/Blue Brain Project */ +/* Copyright (c) 2014-2017, EPFL/Blue Brain Project */ /* Raphael Dumusc */ /* Daniel.Nachbaur@epfl.ch */ /* All rights reserved. */ @@ -56,6 +56,14 @@ enum class View : std::uint8_t right_eye }; +/** Sub-sampling of the image chrominance components in YCbCr color space. */ +enum class ChromaSubsampling +{ + YUV444, /**< No sub-sampling */ + YUV422, /**< 50% vertical sub-sampling */ + YUV420 /**< 50% vertical + horizontal sub-sampling */ +}; + /** Cast an enum class value to its underlying type. */ template constexpr typename std::underlying_type::type as_underlying_type(E e) diff --git a/doc/Changelog.md b/doc/Changelog.md index 294c6de..44c4f3d 100644 --- a/doc/Changelog.md +++ b/doc/Changelog.md @@ -3,6 +3,11 @@ Changelog {#Changelog} ## Deflect 0.13 (git master) +* [154](https://github.com/BlueBrain/Deflect/pull/154): + On the server side, segments can be decoded to YUV images, saving CPU time by + skipping the YUV -> RGB conversion step. + Additionally, Stream images can be further compressed using optional + [chroma subsampling](ChromaSubsampling.md). * [152](https://github.com/BlueBrain/Deflect/pull/152): Changed coding style of the project to conform to new .clang-format rules. * [148](https://github.com/BlueBrain/Deflect/pull/148): diff --git a/doc/ChromaSubsampling.md b/doc/ChromaSubsampling.md new file mode 100644 index 0000000..a01a665 --- /dev/null +++ b/doc/ChromaSubsampling.md @@ -0,0 +1,43 @@ +YUV & Chroma Subsampling +============ + +This document provides general information about the YUV decoding and +chrominance subsampling support introduced in Deflect 0.13. + +## Introduction + +The JPEG algorithm must convert RGB input images to YUV prior to performing the +actual compression. The inverse transform is applied when decoding. + +The rendering process can be made faster by skipping the YUV -> RGB step and +displaying the YUV images directly through an OpenGL shader. + +## Chroma Subsampling + +In addition to JPEG compression, the image size can be reduced by subsampling +the chrominance channels in the YUV color space. This technique is widely used +for movie encoding (most codecs use YUV420 by default). It has limited impact +on visual quality when applied to "natural" images, although it can be more +detrimental to artifical images such as computer interfaces. + +Supported chrominance subsampling modes in Deflect are: + +* YUV444 - No sub-sampling (default, image size: 100%) +* YUV422 - 50% vertical sub-sampling (image size: 75%) +* YUV420 - 50% vertical + horizontal sub-sampling (image size: 50%) + +## Results + +Chroma subsampling test results obtained with a 3840x1200[px] desktop stream. + +| JPEG Quality | YUV444 vs YUV420 | JPEG size reduction (%) | +| ------------- | ---------------- | -----------------------:| +| 20 | 320 / 250 KB | 22 % | +| 50 | 450 / 370 KB | 18 % | +| 75 | 590 / 490 KB | 17 % | +| 90 | 850 / 700 KB | 17 % | +| 100 | 2000 / 1500 KB | 25 % | + +We observe modest gains of network bandwidth using YUV420 subsampling (17% at +quality 75), but further performance is gained on the server side during +the transfer of the decompressed YUV image to GPU memory (50% less data). diff --git a/tests/cpp/SegmentDecoderTests.cpp b/tests/cpp/SegmentDecoderTests.cpp index e6ff004..fb6adbd 100644 --- a/tests/cpp/SegmentDecoderTests.cpp +++ b/tests/cpp/SegmentDecoderTests.cpp @@ -49,9 +49,49 @@ namespace ut = boost::unit_test; #include #include +#include // std::round -void fillTestImage(std::vector& data) +namespace { +int _toY(const int r, const int g, const int b) +{ + return std::round(0.299 * r + 0.587 * g + 0.114 * b); +} +int _toU(const int r, const int g, const int b) +{ + const auto u = 0.492 * (b - _toY(r, g, b)); + const auto uMinMax = 0.436; + return std::round((u + 255.0 * uMinMax) / (2.0 * uMinMax)); +} +int _toV(const int r, const int g, const int b) +{ + const auto v = 0.877 * (r - _toY(r, g, b)); + const auto vMinMax = 0.615; + return std::round((v + 255.0 * vMinMax) / (2.0 * vMinMax)); +} + +const std::vector expectedYData(8 * 8, _toY(92, 28, 0)); +const std::vector expectedUData(8 * 8, _toU(92, 28, 0)); +const std::vector expectedVData(8 * 8, _toV(92, 28, 0)); +} + +namespace deflect +{ +inline std::ostream& operator<<(std::ostream& str, const DataType t) +{ + str << "DataType(" << as_underlying_type(t) << ")"; + return str; +} +inline std::ostream& operator<<(std::ostream& str, const ChromaSubsampling s) +{ + str << "ChromaSubsampling(" << as_underlying_type(s) << ")"; + return str; +} +} + +std::vector makeTestImage() +{ + std::vector data; data.reserve(8 * 8 * 4); for (size_t i = 0; i < 8 * 8; ++i) { @@ -60,13 +100,13 @@ void fillTestImage(std::vector& data) data.push_back(0); // B data.push_back(-1); // A } + return data; } BOOST_AUTO_TEST_CASE(testImageCompressionAndDecompression) { // Vector of RGBA data - std::vector data; - fillTestImage(data); + const auto data = makeTestImage(); deflect::ImageWrapper imageWrapper(data.data(), 8, 8, deflect::RGBA); imageWrapper.compressionQuality = 100; @@ -91,6 +131,107 @@ BOOST_AUTO_TEST_CASE(testImageCompressionAndDecompression) dataOut, dataOut + data.size()); } +#ifndef DEFLECT_USE_LEGACY_LIBJPEGTURBO + +QByteArray decodeToYUVWithDecompressor( + const QByteArray& jpegData, const deflect::ChromaSubsampling expected) +{ + deflect::ImageJpegDecompressor decompressor; + const auto yuvData = decompressor.decompressToYUV(jpegData); + BOOST_CHECK_EQUAL(yuvData.second, expected); + return yuvData.first; +} + +QByteArray decodeToYUVWithSegmentDecoder( + const QByteArray& jpegData, const deflect::ChromaSubsampling expected) +{ + deflect::Segment segment; + segment.parameters.width = 8; + segment.parameters.height = 8; + segment.parameters.dataType = deflect::DataType::jpeg; + segment.imageData = + QByteArray::fromRawData(jpegData.data(), jpegData.size()); + + deflect::SegmentDecoder decoder; + BOOST_CHECK_EQUAL(decoder.decodeType(segment), expected); + + decoder.decodeToYUV(segment); + BOOST_CHECK_NE(segment.parameters.dataType, deflect::DataType::jpeg); + BOOST_CHECK_NE(segment.parameters.dataType, deflect::DataType::rgba); + return segment.imageData; +} + +using DecodeFunc = + std::function; + +void testImageDecompressionToYUV(const deflect::ChromaSubsampling subsamp, + DecodeFunc decode) +{ + // Vector of RGBA data + const auto data = makeTestImage(); + deflect::ImageWrapper imageWrapper(data.data(), 8, 8, deflect::RGBA); + imageWrapper.compressionQuality = 100; + imageWrapper.subsampling = subsamp; + + // Compress image + deflect::ImageJpegCompressor compressor; + const auto jpegData = + compressor.computeJpeg(imageWrapper, QRect(0, 0, 8, 8)); + + BOOST_REQUIRE(jpegData.size() > 0); + BOOST_REQUIRE(jpegData.size() != (int)data.size()); + + const auto yuvImageData = decode(jpegData, subsamp); + + const auto imageSize = imageWrapper.width * imageWrapper.height; + auto uvSize = imageSize; + if (subsamp == deflect::ChromaSubsampling::YUV422) + uvSize >>= 1; + else if (subsamp == deflect::ChromaSubsampling::YUV420) + uvSize >>= 2; + + // Check decoded image in format YUV + BOOST_REQUIRE(!yuvImageData.isEmpty()); + BOOST_REQUIRE_EQUAL(yuvImageData.size(), imageSize + 2 * uvSize); + + // Y component + const char* yData = expectedYData.data(); + const char* yDataOut = yuvImageData.constData(); + BOOST_CHECK_EQUAL_COLLECTIONS(yData, yData + expectedYData.size(), yDataOut, + yDataOut + expectedYData.size()); + + // U component + const char* uData = expectedUData.data(); + const char* uDataOut = yuvImageData.constData() + imageSize; + BOOST_CHECK_EQUAL_COLLECTIONS(uData, uData + uvSize, uDataOut, + uDataOut + uvSize); + + // V component + const char* vData = expectedVData.data(); + const char* vDataOut = yuvImageData.constData() + imageSize + uvSize; + BOOST_CHECK_EQUAL_COLLECTIONS(vData, vData + uvSize, vDataOut, + vDataOut + uvSize); +} + +BOOST_AUTO_TEST_CASE(testImageCompressionAndDecompressionYUV) +{ + testImageDecompressionToYUV(deflect::ChromaSubsampling::YUV444, + &decodeToYUVWithDecompressor); + testImageDecompressionToYUV(deflect::ChromaSubsampling::YUV422, + &decodeToYUVWithDecompressor); + testImageDecompressionToYUV(deflect::ChromaSubsampling::YUV420, + &decodeToYUVWithDecompressor); + + testImageDecompressionToYUV(deflect::ChromaSubsampling::YUV444, + &decodeToYUVWithSegmentDecoder); + testImageDecompressionToYUV(deflect::ChromaSubsampling::YUV422, + &decodeToYUVWithSegmentDecoder); + testImageDecompressionToYUV(deflect::ChromaSubsampling::YUV420, + &decodeToYUVWithSegmentDecoder); +} + +#endif + static bool append(deflect::Segments& segments, const deflect::Segment& segment) { static QMutex lock; @@ -102,8 +243,7 @@ static bool append(deflect::Segments& segments, const deflect::Segment& segment) BOOST_AUTO_TEST_CASE(testImageSegmentationWithCompressionAndDecompression) { // Vector of rgba data - std::vector data; - fillTestImage(data); + const auto data = makeTestImage(); // Compress image deflect::ImageWrapper imageWrapper(data.data(), 8, 8, deflect::RGBA); @@ -118,7 +258,7 @@ BOOST_AUTO_TEST_CASE(testImageSegmentationWithCompressionAndDecompression) BOOST_REQUIRE_EQUAL(segments.size(), 1); deflect::Segment& segment = segments.front(); - BOOST_REQUIRE(segment.parameters.compressed); + BOOST_REQUIRE_EQUAL(segment.parameters.dataType, deflect::DataType::jpeg); BOOST_REQUIRE(segment.imageData.size() != (int)data.size()); // Decompress image @@ -127,7 +267,7 @@ BOOST_AUTO_TEST_CASE(testImageSegmentationWithCompressionAndDecompression) decoder.waitDecoding(); // Check decoded image in format RGBA - BOOST_REQUIRE(!segment.parameters.compressed); + BOOST_REQUIRE_EQUAL(segment.parameters.dataType, deflect::DataType::rgba); BOOST_REQUIRE_EQUAL(segment.imageData.size(), data.size()); const char* dataOut = segment.imageData.constData(); From b88432e80858db767fa296783a047920f7c81272 Mon Sep 17 00:00:00 2001 From: Raphael Dumusc Date: Tue, 14 Mar 2017 13:38:58 +0100 Subject: [PATCH 2/2] DesktopStreamer: added chroma subsampling option --- apps/DesktopStreamer/MainWindow.cpp | 21 ++++++++++++++++++++- apps/DesktopStreamer/MainWindow.h | 3 +++ apps/DesktopStreamer/MainWindow.ui | 26 ++++++++++++++++++++++++++ apps/DesktopStreamer/Stream.cpp | 9 ++++++--- apps/DesktopStreamer/Stream.h | 5 +++-- 5 files changed, 58 insertions(+), 6 deletions(-) diff --git a/apps/DesktopStreamer/MainWindow.cpp b/apps/DesktopStreamer/MainWindow.cpp index 8331d4c..5e9f38f 100644 --- a/apps/DesktopStreamer/MainWindow.cpp +++ b/apps/DesktopStreamer/MainWindow.cpp @@ -250,6 +250,9 @@ void MainWindow::_showAdvancedSettings(const bool visible) _qualitySlider->setVisible(visible); _qualityLabel->setVisible(visible); + + _subsamplingComboBox->setVisible(visible); + _subsamplingLabel->setVisible(visible); } void MainWindow::_updateStreams() @@ -372,7 +375,8 @@ void MainWindow::_shareDesktopUpdate() for (auto i = _streams.begin(); i != _streams.end();) { - const auto error = i->second->update(_qualitySlider->value()); + const auto error = + i->second->update(_qualitySlider->value(), _getSubsampling()); if (error.empty()) ++i; else @@ -436,6 +440,21 @@ std::string MainWindow::_getStreamHost() const return _hostComboBox->currentText().toStdString(); } +deflect::ChromaSubsampling MainWindow::_getSubsampling() const +{ + switch (_subsamplingComboBox->currentIndex()) + { + case 0: + return deflect::ChromaSubsampling::YUV444; + case 1: + return deflect::ChromaSubsampling::YUV422; + case 2: + return deflect::ChromaSubsampling::YUV420; + default: + throw std::runtime_error("unsupported subsampling mode"); + }; +} + void MainWindow::_onStreamEventsBoxClicked(const bool checked) { if (!checked) diff --git a/apps/DesktopStreamer/MainWindow.h b/apps/DesktopStreamer/MainWindow.h index c53444e..d2ec6e0 100644 --- a/apps/DesktopStreamer/MainWindow.h +++ b/apps/DesktopStreamer/MainWindow.h @@ -47,6 +47,8 @@ #include #endif +#include + #include #include #include @@ -109,6 +111,7 @@ private slots: void _shareDesktopUpdate(); void _regulateFrameRate(); std::string _getStreamHost() const; + deflect::ChromaSubsampling _getSubsampling() const; }; #endif diff --git a/apps/DesktopStreamer/MainWindow.ui b/apps/DesktopStreamer/MainWindow.ui index 8acf208..ff37211 100644 --- a/apps/DesktopStreamer/MainWindow.ui +++ b/apps/DesktopStreamer/MainWindow.ui @@ -257,6 +257,32 @@ + + + + Subsampling + + + + + + + + YUV444 + + + + + YUV422 + + + + + YUV420 + + + + diff --git a/apps/DesktopStreamer/Stream.cpp b/apps/DesktopStreamer/Stream.cpp index cf626dc..f33f797 100644 --- a/apps/DesktopStreamer/Stream.cpp +++ b/apps/DesktopStreamer/Stream.cpp @@ -126,7 +126,8 @@ class Stream::Impl return true; } - std::string update(const int quality) + std::string update(const int quality, + const deflect::ChromaSubsampling subsamp) { QPixmap pixmap; @@ -190,6 +191,7 @@ class Stream::Impl deflect::BGRA); deflectImage.compressionPolicy = deflect::COMPRESSION_ON; deflectImage.compressionQuality = std::max(1, std::min(quality, 100)); + deflectImage.subsampling = subsamp; if (!_stream.send(deflectImage) || !_stream.finishFrame()) return "Streaming failure, connection closed"; @@ -306,9 +308,10 @@ Stream::~Stream() { } -std::string Stream::update(const int quality) +std::string Stream::update(const int quality, + const deflect::ChromaSubsampling subsamp) { - return _impl->update(quality); + return _impl->update(quality, subsamp); } bool Stream::processEvents(const bool interact) diff --git a/apps/DesktopStreamer/Stream.h b/apps/DesktopStreamer/Stream.h index c95468f..e1773de 100644 --- a/apps/DesktopStreamer/Stream.h +++ b/apps/DesktopStreamer/Stream.h @@ -57,10 +57,11 @@ class Stream : public deflect::Stream /** * Send an update to the server. - * @param quality the quality setting for compression [1; 100] + * @param quality the quality setting for compression [1; 100]. + * @param subsamp the chrominance subsampling mode. * @return an empty string on success, the error message otherwise. */ - std::string update(int quality); + std::string update(int quality, deflect::ChromaSubsampling subsamp); /** * Process all pending events.