Optional JPEG chroma subsampling, segments can be decoded to YUV

.gitexternals

@@ -1,2 +1,2 @@
 # -*- mode: cmake -*-
-# CMake/common https://github.com/Eyescale/CMake.git 3d5d284
+# CMake/common https://github.com/Eyescale/CMake.git 770b264

CMakeLists.txt

@@ -24,7 +24,11 @@ include(Common)
 
 common_find_package(Boost COMPONENTS program_options unit_test_framework)
 common_find_package(GLUT)
-common_find_package(LibJpegTurbo REQUIRED)
+common_find_package(LibJpegTurbo 1.4)
+if(NOT LibJpegTurbo_FOUND)
+  common_find_package(LibJpegTurbo 1.2 REQUIRED)
+  list(APPEND COMMON_FIND_PACKAGE_DEFINES DEFLECT_USE_LEGACY_LIBJPEGTURBO)
+endif()
 common_find_package(OpenGL)
 common_find_package(Qt5Concurrent REQUIRED SYSTEM)
 common_find_package(Qt5Core REQUIRED)

apps/DesktopStreamer/MainWindow.cpp

@@ -250,6 +250,9 @@ void MainWindow::_showAdvancedSettings(const bool visible)
 
     _qualitySlider->setVisible(visible);
     _qualityLabel->setVisible(visible);
+
+    _subsamplingComboBox->setVisible(visible);
+    _subsamplingLabel->setVisible(visible);
 }
 
 void MainWindow::_updateStreams()
@@ -372,7 +375,8 @@ void MainWindow::_shareDesktopUpdate()
 
     for (auto i = _streams.begin(); i != _streams.end();)
     {
-        const auto error = i->second->update(_qualitySlider->value());
+        const auto error =
+            i->second->update(_qualitySlider->value(), _getSubsampling());
         if (error.empty())
             ++i;
         else
@@ -436,6 +440,21 @@ std::string MainWindow::_getStreamHost() const
     return _hostComboBox->currentText().toStdString();
 }
 
+deflect::ChromaSubsampling MainWindow::_getSubsampling() const
+{
+    switch (_subsamplingComboBox->currentIndex())
+    {
+    case 0:
+        return deflect::ChromaSubsampling::YUV444;
+    case 1:
+        return deflect::ChromaSubsampling::YUV422;
+    case 2:
+        return deflect::ChromaSubsampling::YUV420;
+    default:
+        throw std::runtime_error("unsupported subsampling mode");
+    };
+}
+
 void MainWindow::_onStreamEventsBoxClicked(const bool checked)
 {
     if (!checked)

apps/DesktopStreamer/MainWindow.h

@@ -47,6 +47,8 @@
 #include <deflect/AppNapSuspender.h>
 #endif
 
+#include <deflect/types.h>
+
 #include <QMainWindow>
 #include <QTime>
 #include <QTimer>
@@ -109,6 +111,7 @@ private slots:
     void _shareDesktopUpdate();
     void _regulateFrameRate();
     std::string _getStreamHost() const;
+    deflect::ChromaSubsampling _getSubsampling() const;
 };
 
 #endif

apps/DesktopStreamer/MainWindow.ui

@@ -257,6 +257,32 @@
         </property>
        </widget>
       </item>
+      <item row="7" column="0">
+       <widget class="QLabel" name="_subsamplingLabel">
+        <property name="text">
+         <string>Subsampling</string>
+        </property>
+       </widget>
+      </item>
+      <item row="7" column="1">
+       <widget class="QComboBox" name="_subsamplingComboBox">
+        <item>
+         <property name="text">
+          <string>YUV444</string>
+         </property>
+        </item>
+        <item>
+         <property name="text">
+          <string>YUV422</string>
+         </property>
+        </item>
+        <item>
+         <property name="text">
+          <string>YUV420</string>
+         </property>
+        </item>
+       </widget>
+      </item>
      </layout>
     </item>
    </layout>

apps/DesktopStreamer/Stream.cpp

@@ -126,7 +126,8 @@ class Stream::Impl
         return true;
     }
 
-    std::string update(const int quality)
+    std::string update(const int quality,
+                       const deflect::ChromaSubsampling subsamp)
     {
         QPixmap pixmap;
 
@@ -190,6 +191,7 @@ class Stream::Impl
                                            deflect::BGRA);
         deflectImage.compressionPolicy = deflect::COMPRESSION_ON;
         deflectImage.compressionQuality = std::max(1, std::min(quality, 100));
+        deflectImage.subsampling = subsamp;
 
         if (!_stream.send(deflectImage) || !_stream.finishFrame())
             return "Streaming failure, connection closed";
@@ -306,9 +308,10 @@ Stream::~Stream()
 {
 }
 
-std::string Stream::update(const int quality)
+std::string Stream::update(const int quality,
+                           const deflect::ChromaSubsampling subsamp)
 {
-    return _impl->update(quality);
+    return _impl->update(quality, subsamp);
 }
 
 bool Stream::processEvents(const bool interact)

apps/DesktopStreamer/Stream.h

@@ -57,10 +57,11 @@ class Stream : public deflect::Stream
 
     /**
      * Send an update to the server.
-     * @param quality the quality setting for compression [1; 100]
+     * @param quality the quality setting for compression [1; 100].
+     * @param subsamp the chrominance subsampling mode.
      * @return an empty string on success, the error message otherwise.
      */
-    std::string update(int quality);
+    std::string update(int quality, deflect::ChromaSubsampling subsamp);
 
     /**
      * Process all pending events.

deflect/ImageJpegCompressor.cpp

@@ -1,6 +1,6 @@
 /*********************************************************************/
-/* Copyright (c) 2013, EPFL/Blue Brain Project                       */
-/*                     Raphael Dumusc <raphael.dumusc@epfl.ch>       */
+/* Copyright (c) 2013-2017, EPFL/Blue Brain Project                  */
+/*                          Raphael Dumusc <raphael.dumusc@epfl.ch>  */
 /* All rights reserved.                                              */
 /*                                                                   */
 /* Redistribution and use in source and binary forms, with or        */
@@ -55,7 +55,7 @@ ImageJpegCompressor::~ImageJpegCompressor()
     tjDestroy(_tjHandle);
 }
 
-int getTurboJpegFormat(const PixelFormat pixelFormat)
+int _getTurboJpegFormat(const PixelFormat pixelFormat)
 {
     switch (pixelFormat)
     {
@@ -77,12 +77,28 @@ int getTurboJpegFormat(const PixelFormat pixelFormat)
     }
 }
 
+int _getTurboJpegSubsamp(const ChromaSubsampling subsampling)
+{
+    switch (subsampling)
+    {
+    case ChromaSubsampling::YUV444:
+        return TJSAMP_444;
+    case ChromaSubsampling::YUV422:
+        return TJSAMP_422;
+    case ChromaSubsampling::YUV420:
+        return TJSAMP_420;
+    default:
+        std::cerr << "unknown subsampling format" << std::endl;
+        return TJSAMP_444;
+    }
+}
+
 QByteArray ImageJpegCompressor::computeJpeg(const ImageWrapper& sourceImage,
                                             const QRect& imageRegion)
 {
     // tjCompress API is incorrect and takes a non-const input buffer, even
     // though it does not modify it. It can "safely" be cast to non-const
-    // pointer to comply to the incorrect API.
+    // pointer to comply with the incorrect API.
     unsigned char* tjSrcBuffer = (unsigned char*)sourceImage.data;
     tjSrcBuffer +=
         imageRegion.y() * sourceImage.width * sourceImage.getBytesPerPixel();
@@ -92,10 +108,10 @@ QByteArray ImageJpegCompressor::computeJpeg(const ImageWrapper& sourceImage,
     // assume imageBuffer isn't padded
     const int tjPitch = sourceImage.width * sourceImage.getBytesPerPixel();
     const int tjHeight = imageRegion.height();
-    const int tjPixelFormat = getTurboJpegFormat(sourceImage.pixelFormat);
-    unsigned char* tjJpegBuf = 0;
+    const int tjPixelFormat = _getTurboJpegFormat(sourceImage.pixelFormat);
+    unsigned char* tjJpegBuf = nullptr;
     unsigned long tjJpegSize = 0;
-    const int tjJpegSubsamp = TJSAMP_444;
+    const int tjJpegSubsamp = _getTurboJpegSubsamp(sourceImage.subsampling);
     const int tjJpegQual = sourceImage.compressionQuality;
     const int tjFlags = 0; // or: TJFLAG_BOTTOMUP
 

deflect/ImageJpegCompressor.h

@@ -1,6 +1,6 @@
 /*********************************************************************/
-/* Copyright (c) 2013, EPFL/Blue Brain Project                       */
-/*                     Raphael Dumusc <raphael.dumusc@epfl.ch>       */
+/* Copyright (c) 2013-2017, EPFL/Blue Brain Project                  */
+/*                          Raphael Dumusc <raphael.dumusc@epfl.ch>  */
 /* All rights reserved.                                              */
 /*                                                                   */
 /* Redistribution and use in source and binary forms, with or        */
@@ -74,4 +74,4 @@ class ImageJpegCompressor
 };
 }
 
-#endif // IMAGEJPEGCOMPRESSOR_H
+#endif

deflect/ImageJpegDecompressor.cpp

@@ -1,6 +1,6 @@
 /*********************************************************************/
-/* Copyright (c) 2013, EPFL/Blue Brain Project                       */
-/*                     Raphael Dumusc <raphael.dumusc@epfl.ch>       */
+/* Copyright (c) 2013-2017, EPFL/Blue Brain Project                  */
+/*                          Raphael Dumusc <raphael.dumusc@epfl.ch>  */
 /* All rights reserved.                                              */
 /*                                                                   */
 /* Redistribution and use in source and binary forms, with or        */
@@ -41,6 +41,24 @@
 
 #include <iostream>
 
+namespace
+{
+deflect::ChromaSubsampling _getSubsamp(const int tjJpegSubsamp)
+{
+    switch (tjJpegSubsamp)
+    {
+    case TJSAMP_444:
+        return deflect::ChromaSubsampling::YUV444;
+    case TJSAMP_422:
+        return deflect::ChromaSubsampling::YUV422;
+    case TJSAMP_420:
+        return deflect::ChromaSubsampling::YUV420;
+    default:
+        throw std::runtime_error("unsupported subsampling format");
+    }
+}
+}
+
 namespace deflect
 {
 ImageJpegDecompressor::ImageJpegDecompressor()
@@ -53,29 +71,72 @@ ImageJpegDecompressor::~ImageJpegDecompressor()
     tjDestroy(_tjHandle);
 }
 
-QByteArray ImageJpegDecompressor::decompress(const QByteArray& jpegData)
+JpegHeader ImageJpegDecompressor::decompressHeader(const QByteArray& jpegData)
 {
-    // get information from header
-    int width, height, jpegSubsamp;
+    JpegHeader header;
+    int jpegSubsamp = -1;
+
+#ifdef TJ_NUMCS // introduced with tjDecompressHeader3()
+    int jpegColorspace = -1;
+    int err =
+        tjDecompressHeader3(_tjHandle, (unsigned char*)jpegData.data(),
+                            (unsigned long)jpegData.size(), &header.width,
+                            &header.height, &jpegSubsamp, &jpegColorspace);
+    if (err != 0 || jpegColorspace != TJCS_YCbCr)
+#else
     int err = tjDecompressHeader2(_tjHandle, (unsigned char*)jpegData.data(),
-                                  (unsigned long)jpegData.size(), &width,
-                                  &height, &jpegSubsamp);
+                                  (unsigned long)jpegData.size(), &header.width,
+                                  &header.height, &jpegSubsamp);
     if (err != 0)
+#endif
         throw std::runtime_error("libjpeg-turbo header decompression failed");
 
-    // decompress image data
-    int pixelFormat = TJPF_RGBX; // Format for OpenGL texture (GL_RGBA)
-    int pitch = width * tjPixelSize[pixelFormat];
-    int flags = TJ_FASTUPSAMPLE;
-    QByteArray decodedData(height * pitch, Qt::Uninitialized);
+    header.subsampling = _getSubsamp(jpegSubsamp);
+    return header;
+}
+
+QByteArray ImageJpegDecompressor::decompress(const QByteArray& jpegData)
+{
+    const auto header = decompressHeader(jpegData);
+    const int pixelFormat = TJPF_RGBX; // Format for OpenGL texture (GL_RGBA)
+    const int pitch = header.width * tjPixelSize[pixelFormat];
+    const int flags = TJ_FASTUPSAMPLE;
+
+    QByteArray decodedData(header.height * pitch, Qt::Uninitialized);
 
-    err = tjDecompress2(_tjHandle, (unsigned char*)jpegData.data(),
-                        (unsigned long)jpegData.size(),
-                        (unsigned char*)decodedData.data(), width, pitch,
-                        height, pixelFormat, flags);
+    int err = tjDecompress2(_tjHandle, (unsigned char*)jpegData.data(),
+                            (unsigned long)jpegData.size(),
+                            (unsigned char*)decodedData.data(), header.width,
+                            pitch, header.height, pixelFormat, flags);
     if (err != 0)
         throw std::runtime_error("libjpeg-turbo image decompression failed");
 
     return decodedData;
 }
+
+#ifndef DEFLECT_USE_LEGACY_LIBJPEGTURBO
+
+ImageJpegDecompressor::YUVData ImageJpegDecompressor::decompressToYUV(
+    const QByteArray& jpegData)
+{
+    const auto header = decompressHeader(jpegData);
+    const int pad = 1; // no padding
+    const int flags = 0;
+    const int jpegSubsamp = int(header.subsampling);
+    const auto decodedSize =
+        tjBufSizeYUV2(header.width, pad, header.height, jpegSubsamp);
+
+    auto decodedData = QByteArray(decodedSize, Qt::Uninitialized);
+
+    int err = tjDecompressToYUV2(_tjHandle, (unsigned char*)jpegData.data(),
+                                 (unsigned long)jpegData.size(),
+                                 (unsigned char*)decodedData.data(),
+                                 header.width, pad, header.height, flags);
+    if (err != 0)
+        throw std::runtime_error("libjpeg-turbo image decompression failed");
+
+    return std::make_pair(std::move(decodedData), header.subsampling);
+}
+
+#endif
 }