Fix unexpected format switching in display_vram

src/platform/windows/display.h

@@ -185,6 +185,8 @@ class display_vram_t : public display_base_t, public std::enable_shared_from_thi
   vs_t scene_vs;
 
   gpu_cursor_t cursor;
+
+  texture2d_t last_frame_copy;
 };
 } // namespace platf::dxgi
 

src/platform/windows/display_base.cpp

@@ -292,22 +292,10 @@ int display_base_t::init(int framerate, const std::string &display_name) {
 
   //FIXME: Duplicate output on RX580 in combination with DOOM (2016) --> BSOD
   {
-    dxgi::output1_t output1 {};
-    dxgi::output5_t output5 {};
-
     // IDXGIOutput5 is optional, but can provide improved performance and wide color support
+    dxgi::output5_t output5 {};
     status = output->QueryInterface(IID_IDXGIOutput5, (void **)&output5);
-    if(FAILED(status)) {
-      BOOST_LOG(warning) << "Failed to query IDXGIOutput5 from the output"sv;
-    }
-
-    status = output->QueryInterface(IID_IDXGIOutput1, (void **)&output1);
-    if(FAILED(status)) {
-      BOOST_LOG(error) << "Failed to query IDXGIOutput1 from the output"sv;
-      return -1;
-    }
-
-    if(output5) {
+    if(SUCCEEDED(status)) {
       // Ask the display implementation which formats it supports
       auto supported_formats = get_supported_sdr_capture_formats();
       if(supported_formats.empty()) {
@@ -324,12 +312,24 @@ int display_base_t::init(int framerate, const std::string &display_name) {
         std::this_thread::sleep_for(200ms);
       }
 
+      // We don't retry with DuplicateOutput() because we can hit this codepath when we're racing
+      // with mode changes and we don't want to accidentally fall back to suboptimal capture if
+      // we get unlucky and succeed below.
       if(FAILED(status)) {
         BOOST_LOG(warning) << "DuplicateOutput1 Failed [0x"sv << util::hex(status).to_string_view() << ']';
+        return -1;
       }
     }
+    else {
+      BOOST_LOG(warning) << "IDXGIOutput5 is not supported by your OS. Capture performance may be reduced."sv;
+
+      dxgi::output1_t output1 {};
+      status = output->QueryInterface(IID_IDXGIOutput1, (void **)&output1);
+      if(FAILED(status)) {
+        BOOST_LOG(error) << "Failed to query IDXGIOutput1 from the output"sv;
+        return -1;
+      }
 
-    if(!output5 || FAILED(status)) {
       for(int x = 0; x < 2; ++x) {
         status = output1->DuplicateOutput((IUnknown *)device.get(), &dup.dup);
         if(SUCCEEDED(status)) {

src/platform/windows/display_vram.cpp

@@ -628,7 +628,7 @@ capture_e display_vram_t::snapshot(platf::img_t *img_base, std::chrono::millisec
     cursor.set_pos(frame_info.PointerPosition.Position.x, frame_info.PointerPosition.Position.y, frame_info.PointerPosition.Visible);
   }
 
-  if(capture_format != DXGI_FORMAT_UNKNOWN || frame_update_flag) {
+  if(frame_update_flag) {
     texture2d_t src {};
 
     // Get the texture object from this frame
@@ -641,19 +641,36 @@ capture_e display_vram_t::snapshot(platf::img_t *img_base, std::chrono::millisec
     D3D11_TEXTURE2D_DESC desc;
     src->GetDesc(&desc);
 
-    // If we don't know the capture format yet, grab it from this texture
-    if(capture_format == DXGI_FORMAT_UNKNOWN) {
-      capture_format = desc.Format;
-      BOOST_LOG(info) << "Capture format ["sv << dxgi_format_to_string(capture_format) << ']';
-    }
-
     // It's possible for our display enumeration to race with mode changes and result in
     // mismatched image pool and desktop texture sizes. If this happens, just reinit again.
     if(desc.Width != width || desc.Height != height) {
       BOOST_LOG(info) << "Capture size changed ["sv << width << 'x' << height << " -> "sv << desc.Width << 'x' << desc.Height << ']';
       return capture_e::reinit;
     }
 
+    // If we don't know the capture format yet, grab it from this texture
+    if(capture_format == DXGI_FORMAT_UNKNOWN) {
+      capture_format = desc.Format;
+      BOOST_LOG(info) << "Capture format ["sv << dxgi_format_to_string(capture_format) << ']';
+
+      D3D11_TEXTURE2D_DESC t {};
+      t.Width            = width;
+      t.Height           = height;
+      t.MipLevels        = 1;
+      t.ArraySize        = 1;
+      t.SampleDesc.Count = 1;
+      t.Usage            = D3D11_USAGE_DEFAULT;
+      t.Format           = capture_format;
+      t.BindFlags        = 0;
+
+      // Create a texture to store the most recent copy of the desktop
+      auto status = device->CreateTexture2D(&t, nullptr, &last_frame_copy);
+      if(FAILED(status)) {
+        BOOST_LOG(error) << "Failed to create frame copy texture [0x"sv << util::hex(status).to_string_view() << ']';
+        return capture_e::error;
+      }
+    }
+
     // It's also possible for the capture format to change on the fly. If that happens,
     // reinitialize capture to try format detection again and create new images.
     if(capture_format != desc.Format) {
@@ -666,17 +683,30 @@ capture_e display_vram_t::snapshot(platf::img_t *img_base, std::chrono::millisec
       return capture_e::error;
     }
 
-    // Copy the texture into this image
+    // Copy the texture into this image and the staging texture
     device_ctx->CopyResource(img->texture.get(), src.get());
+    device_ctx->CopyResource(last_frame_copy.get(), src.get());
   }
-  else {
+  else if(capture_format == DXGI_FORMAT_UNKNOWN) {
     // We don't know the final capture format yet, so we will encode a dummy image
     BOOST_LOG(debug) << "Capture format is still unknown. Encoding a blank image"sv;
 
     if(dummy_img(img)) {
       return capture_e::error;
     }
   }
+  else {
+    // We must know the capture format in this path or we would have hit the above unknown format case
+    if(complete_img(img, false)) {
+      return capture_e::error;
+    }
+
+    // We have a previously captured frame to reuse. We can't just grab the src texture from
+    // the call to AcquireNextFrame() because that won't be valid. It seems to return a texture
+    // in the unmodified desktop format (rather than the formats we passed to DuplicateOutput1())
+    // if called in that case.
+    device_ctx->CopyResource(img->texture.get(), last_frame_copy.get());
+  }
 
   if(cursor.visible && cursor_visible) {
     D3D11_VIEWPORT view {