fix(video): prevent encoder probing with no active displays (#3592)

src/display_device.cpp

@@ -810,6 +810,22 @@ namespace display_device {
     });
   }
 
+  bool is_any_device_active() {
+    std::lock_guard lock {DD_DATA.mutex};
+    if (!DD_DATA.sm_instance) {
+      // Platform is not supported, assume success.
+      return true;
+    }
+
+    return DD_DATA.sm_instance->execute([](auto &settings_iface) {
+      const auto devices {settings_iface.enumAvailableDevices()};
+      // If at least one device has additional info, it is active.
+      return std::any_of(std::begin(devices), std::end(devices), [](const auto &device) {
+        return static_cast<bool>(device.m_info);
+      });
+    });
+  }
+
   std::variant<failed_to_parse_tag_t, configuration_disabled_tag_t, SingleDisplayConfiguration> parse_configuration(const config::video_t &video_config, const rtsp_stream::launch_session_t &session) {
     const auto device_prep {parse_device_prep_option(video_config)};
     if (!device_prep) {

src/display_device.h

@@ -110,8 +110,8 @@ namespace display_device {
    * The user then accepts that Sunshine is not able to restore the state and "agrees" to
    * do it manually.
    *
-   * @return
-   * @note Whether the function succeeds or fails, the any of the scheduled "retries" from
+   * @return True if persistence was reset, false otherwise.
+   * @note Whether the function succeeds or fails, any of the scheduled "retries" from
    *       other methods will be stopped to not interfere with the user actions.
    *
    * @examples
@@ -120,6 +120,16 @@ namespace display_device {
    */
   [[nodiscard]] bool reset_persistence();
 
+  /**
+   * @brief Check if any of the display devices is currently active.
+   * @return True if at least one device is active.
+   *
+   * @examples
+   * const auto result = is_any_device_active();
+   * @examples_end
+   */
+  [[nodiscard]] bool is_any_device_active();
+
   /**
    * @brief A tag structure indicating that configuration parsing has failed.
    */

src/nvhttp.cpp

@@ -852,14 +852,14 @@ namespace nvhttp {
     auto launch_session = make_launch_session(host_audio, args);
 
     if (rtsp_stream::session_count() == 0) {
+      // The display should be restored in case something fails as there are no other sessions.
+      revert_display_configuration = true;
+
       // We want to prepare display only if there are no active sessions at
       // the moment. This should be done before probing encoders as it could
       // change the active displays.
       display_device::configure_display(config::video, *launch_session);
 
-      // The display should be restored in case something fails as there are no other sessions.
-      revert_display_configuration = true;
-
       // Probe encoders again before streaming to ensure our chosen
       // encoder matches the active GPU (which could have changed
       // due to hotplugging, driver crash, primary monitor change,

src/video.cpp

@@ -2550,6 +2550,11 @@ namespace video {
   }
 
   int probe_encoders() {
+    if (!display_device::is_any_device_active()) {
+      BOOST_LOG(error) << "No display devices are active at the moment! Cannot probe encoders as this could break Sunshine.";
+      return -1;
+    }
+
     auto encoder_list = encoders;
 
     // If we already have a good encoder, check to see if another probe is required