From 8aff38cf7c02c64b6c9470b73b3b586bc8a507d6 Mon Sep 17 00:00:00 2001
From: Yukino Song <nutosservice@gmail.com>
Date: Mon, 12 May 2025 21:31:14 +0800
Subject: [PATCH] Automatically use virtual display if no active display is
 detected

---
 src/process.cpp |  7 ++++++-
 src/video.cpp   | 54 ++++++++++++++++++++++++-------------------------
 src/video.h     |  6 ++++++
 3 files changed, 38 insertions(+), 29 deletions(-)

diff --git a/src/process.cpp b/src/process.cpp
index 32e6337f..e98fcc48 100644
--- a/src/process.cpp
+++ b/src/process.cpp
@@ -223,7 +223,12 @@ namespace proc {
     });
 
 #ifdef _WIN32
-    if (config::video.headless_mode || launch_session->virtual_display || _app.virtual_display) {
+    if (
+      config::video.headless_mode        // Headless mode
+      || launch_session->virtual_display // User requested virtual display
+      || _app.virtual_display            // App is configured to use virtual display
+      || !video::allow_encoder_probing() // No active display presents
+    ) {
       if (vDisplayDriverStatus != VDISPLAY::DRIVER_STATUS::OK) {
         // Try init driver again
         initVDisplayDriver();
diff --git a/src/video.cpp b/src/video.cpp
index 7876bbc7..9ba77949 100644
--- a/src/video.cpp
+++ b/src/video.cpp
@@ -41,36 +41,34 @@ using namespace std::literals;
 
 namespace video {
 
-  namespace {
-    /**
-     * @brief Check if we can allow probing for the encoders.
-     * @return True if there should be no issues with the probing, false if we should prevent it.
-     */
-    bool allow_encoder_probing() {
-      const auto devices {display_device::enumerate_devices()};
+  /**
+   * @brief Check if we can allow probing for the encoders.
+   * @return True if there should be no issues with the probing, false if we should prevent it.
+   */
+  bool allow_encoder_probing() {
+    const auto devices {display_device::enumerate_devices()};
 
-      // If there are no devices, then either the API is not working correctly or OS does not support the lib.
-      // Either way we should not block the probing in this case as we can't tell what's wrong.
-      if (devices.empty()) {
-        return true;
-      }
-
-      // Since Windows 11 24H2, it is possible that there will be no active devices present
-      // for some reason (probably a bug). Trying to probe encoders in such a state locks/breaks the DXGI
-      // and also the display device for Windows. So we must have at least 1 active device.
-      const bool at_least_one_device_is_active = std::any_of(std::begin(devices), std::end(devices), [](const auto &device) {
-        // If device has additional info, it is active.
-        return static_cast<bool>(device.m_info);
-      });
-
-      if (at_least_one_device_is_active) {
-        return true;
-      }
-
-      BOOST_LOG(error) << "No display devices are active at the moment! Cannot probe the encoders.";
-      return false;
+    // If there are no devices, then either the API is not working correctly or OS does not support the lib.
+    // Either way we should not block the probing in this case as we can't tell what's wrong.
+    if (devices.empty()) {
+      return true;
     }
-  }  // namespace
+
+    // Since Windows 11 24H2, it is possible that there will be no active devices present
+    // for some reason (probably a bug). Trying to probe encoders in such a state locks/breaks the DXGI
+    // and also the display device for Windows. So we must have at least 1 active device.
+    const bool at_least_one_device_is_active = std::any_of(std::begin(devices), std::end(devices), [](const auto &device) {
+      // If device has additional info, it is active.
+      return static_cast<bool>(device.m_info);
+    });
+
+    if (at_least_one_device_is_active) {
+      return true;
+    }
+
+    BOOST_LOG(error) << "No display devices are active at the moment! Cannot probe the encoders.";
+    return false;
+  }
 
   void free_ctx(AVCodecContext *ctx) {
     avcodec_free_context(&ctx);
diff --git a/src/video.h b/src/video.h
index 621bc51f..a35dc190 100644
--- a/src/video.h
+++ b/src/video.h
@@ -349,6 +349,12 @@ namespace video {
 
   bool validate_encoder(encoder_t &encoder, bool expect_failure);
 
+  /**
+   * @brief Check if we can allow probing for the encoders.
+   * @return True if there should be no issues with the probing, false if we should prevent it.
+   */
+  bool allow_encoder_probing();
+
   /**
    * @brief Probe encoders and select the preferred encoder.
    * This is called once at startup and each time a stream is launched to