Allow the display to reject unsupported codecs

src/platform/common.h

@@ -480,6 +480,17 @@ namespace platf {
       return false;
     }
 
+    /**
+     * @brief Checks that a given codec is supported by the display device.
+     * @param name The FFmpeg codec name (or similar for non-FFmpeg codecs).
+     * @param config The codec configuration.
+     * @return true if supported, false otherwise.
+     */
+    virtual bool
+    is_codec_supported(std::string_view name, const ::video::config_t &config) {
+      return true;
+    }
+
     virtual ~display_t() = default;
 
     // Offsets for when streaming a specific monitor. By default, they are 0.

src/platform/windows/display.h

@@ -219,6 +219,9 @@ namespace platf::dxgi {
     int
     init(const ::video::config_t &config, const std::string &display_name);
 
+    bool
+    is_codec_supported(std::string_view name, const ::video::config_t &config) override;
+
     std::unique_ptr<avcodec_encode_device_t>
     make_avcodec_encode_device(pix_fmt_e pix_fmt) override;
 

src/platform/windows/display_vram.cpp

@@ -25,6 +25,8 @@ extern "C" {
 
 #include <AMF/core/Factory.h>
 
+#include <boost/algorithm/string/predicate.hpp>
+
 #define SUNSHINE_SHADERS_DIR SUNSHINE_ASSETS_DIR "/shaders/directx"
 namespace platf {
   using namespace std::literals;
@@ -1326,60 +1328,6 @@ namespace platf::dxgi {
       return -1;
     }
 
-    DXGI_ADAPTER_DESC adapter_desc;
-    adapter->GetDesc(&adapter_desc);
-
-    // Perform AMF version checks if we're using an AMD GPU. This check is placed in display_vram_t
-    // to avoid hitting the display_ram_t path which uses software encoding and doesn't touch AMF.
-    if ((config.dynamicRange || config.videoFormat == 2) && adapter_desc.VendorId == 0x1002) {
-      HMODULE amfrt = LoadLibraryW(AMF_DLL_NAME);
-      if (amfrt) {
-        auto unload_amfrt = util::fail_guard([amfrt]() {
-          FreeLibrary(amfrt);
-        });
-
-        auto fnAMFQueryVersion = (AMFQueryVersion_Fn) GetProcAddress(amfrt, AMF_QUERY_VERSION_FUNCTION_NAME);
-        if (fnAMFQueryVersion) {
-          amf_uint64 version;
-          auto result = fnAMFQueryVersion(&version);
-          if (result == AMF_OK) {
-            if (config.videoFormat == 2 && version < AMF_MAKE_FULL_VERSION(1, 4, 30, 0)) {
-              // AMF 1.4.30 adds ultra low latency mode for AV1. Don't use AV1 on earlier versions.
-              // This corresponds to driver version 23.5.2 (23.10.01.45) or newer.
-              BOOST_LOG(warning) << "AV1 encoding is disabled on AMF version "sv
-                                 << AMF_GET_MAJOR_VERSION(version) << '.'
-                                 << AMF_GET_MINOR_VERSION(version) << '.'
-                                 << AMF_GET_SUBMINOR_VERSION(version) << '.'
-                                 << AMF_GET_BUILD_VERSION(version);
-              BOOST_LOG(warning) << "If your AMD GPU supports AV1 encoding, update your graphics drivers!"sv;
-              return -1;
-            }
-            else if (config.dynamicRange && version < AMF_MAKE_FULL_VERSION(1, 4, 23, 0)) {
-              // Older versions of the AMD AMF runtime can crash when fed P010 surfaces.
-              // Fail if AMF version is below 1.4.23 where HEVC Main10 encoding was introduced.
-              // AMF 1.4.23 corresponds to driver version 21.12.1 (21.40.11.03) or newer.
-              BOOST_LOG(warning) << "HDR encoding is disabled on AMF version "sv
-                                 << AMF_GET_MAJOR_VERSION(version) << '.'
-                                 << AMF_GET_MINOR_VERSION(version) << '.'
-                                 << AMF_GET_SUBMINOR_VERSION(version) << '.'
-                                 << AMF_GET_BUILD_VERSION(version);
-              BOOST_LOG(warning) << "If your AMD GPU supports HEVC Main10 encoding, update your graphics drivers!"sv;
-              return -1;
-            }
-          }
-          else {
-            BOOST_LOG(warning) << "AMFQueryVersion() failed: "sv << result;
-          }
-        }
-        else {
-          BOOST_LOG(warning) << "AMF DLL missing export: "sv << AMF_QUERY_VERSION_FUNCTION_NAME;
-        }
-      }
-      else {
-        BOOST_LOG(warning) << "Detected AMD GPU but AMF failed to load"sv;
-      }
-    }
-
     D3D11_SAMPLER_DESC sampler_desc {};
     sampler_desc.Filter = D3D11_FILTER_MIN_MAG_MIP_LINEAR;
     sampler_desc.AddressU = D3D11_TEXTURE_ADDRESS_CLAMP;
@@ -1581,6 +1529,91 @@ namespace platf::dxgi {
     };
   }
 
+  /**
+   * @brief Checks that a given codec is supported by the display device.
+   * @param name The FFmpeg codec name (or similar for non-FFmpeg codecs).
+   * @param config The codec configuration.
+   * @return true if supported, false otherwise.
+   */
+  bool
+  display_vram_t::is_codec_supported(std::string_view name, const ::video::config_t &config) {
+    DXGI_ADAPTER_DESC adapter_desc;
+    adapter->GetDesc(&adapter_desc);
+
+    if (adapter_desc.VendorId == 0x1002) {  // AMD
+      // If it's not an AMF encoder, it's not compatible with an AMD GPU
+      if (!boost::algorithm::ends_with(name, "_amf")) {
+        return false;
+      }
+
+      // Perform AMF version checks if we're using an AMD GPU. This check is placed in display_vram_t
+      // to avoid hitting the display_ram_t path which uses software encoding and doesn't touch AMF.
+      HMODULE amfrt = LoadLibraryW(AMF_DLL_NAME);
+      if (amfrt) {
+        auto unload_amfrt = util::fail_guard([amfrt]() {
+          FreeLibrary(amfrt);
+        });
+
+        auto fnAMFQueryVersion = (AMFQueryVersion_Fn) GetProcAddress(amfrt, AMF_QUERY_VERSION_FUNCTION_NAME);
+        if (fnAMFQueryVersion) {
+          amf_uint64 version;
+          auto result = fnAMFQueryVersion(&version);
+          if (result == AMF_OK) {
+            if (config.videoFormat == 2 && version < AMF_MAKE_FULL_VERSION(1, 4, 30, 0)) {
+              // AMF 1.4.30 adds ultra low latency mode for AV1. Don't use AV1 on earlier versions.
+              // This corresponds to driver version 23.5.2 (23.10.01.45) or newer.
+              BOOST_LOG(warning) << "AV1 encoding is disabled on AMF version "sv
+                                 << AMF_GET_MAJOR_VERSION(version) << '.'
+                                 << AMF_GET_MINOR_VERSION(version) << '.'
+                                 << AMF_GET_SUBMINOR_VERSION(version) << '.'
+                                 << AMF_GET_BUILD_VERSION(version);
+              BOOST_LOG(warning) << "If your AMD GPU supports AV1 encoding, update your graphics drivers!"sv;
+              return false;
+            }
+            else if (config.dynamicRange && version < AMF_MAKE_FULL_VERSION(1, 4, 23, 0)) {
+              // Older versions of the AMD AMF runtime can crash when fed P010 surfaces.
+              // Fail if AMF version is below 1.4.23 where HEVC Main10 encoding was introduced.
+              // AMF 1.4.23 corresponds to driver version 21.12.1 (21.40.11.03) or newer.
+              BOOST_LOG(warning) << "HDR encoding is disabled on AMF version "sv
+                                 << AMF_GET_MAJOR_VERSION(version) << '.'
+                                 << AMF_GET_MINOR_VERSION(version) << '.'
+                                 << AMF_GET_SUBMINOR_VERSION(version) << '.'
+                                 << AMF_GET_BUILD_VERSION(version);
+              BOOST_LOG(warning) << "If your AMD GPU supports HEVC Main10 encoding, update your graphics drivers!"sv;
+              return false;
+            }
+          }
+          else {
+            BOOST_LOG(warning) << "AMFQueryVersion() failed: "sv << result;
+          }
+        }
+        else {
+          BOOST_LOG(warning) << "AMF DLL missing export: "sv << AMF_QUERY_VERSION_FUNCTION_NAME;
+        }
+      }
+      else {
+        BOOST_LOG(warning) << "Detected AMD GPU but AMF failed to load"sv;
+      }
+    }
+    else if (adapter_desc.VendorId == 0x8086) {  // Intel
+      // If it's not a QSV encoder, it's not compatible with an Intel GPU
+      if (!boost::algorithm::ends_with(name, "_qsv")) {
+        return false;
+      }
+    }
+    else if (adapter_desc.VendorId == 0x10de) {  // Nvidia
+      // If it's not an NVENC encoder, it's not compatible with an Nvidia GPU
+      if (!boost::algorithm::ends_with(name, "_nvenc")) {
+        return false;
+      }
+    }
+    else {
+      BOOST_LOG(warning) << "Unknown GPU vendor ID: " << util::hex(adapter_desc.VendorId).to_string_view();
+    }
+
+    return true;
+  }
+
   std::unique_ptr<avcodec_encode_device_t>
   display_vram_t::make_avcodec_encode_device(pix_fmt_e pix_fmt) {
     if (pix_fmt != platf::pix_fmt_e::nv12 && pix_fmt != platf::pix_fmt_e::p010) {

src/video.cpp

@@ -1377,7 +1377,7 @@ namespace video {
     auto &video_format = config.videoFormat == 0 ? encoder.h264 :
                          config.videoFormat == 1 ? encoder.hevc :
                                                    encoder.av1;
-    if (!video_format[encoder_t::PASSED]) {
+    if (!video_format[encoder_t::PASSED] || !disp->is_codec_supported(video_format.name, config)) {
       BOOST_LOG(error) << encoder.name << ": "sv << video_format.name << " mode not supported"sv;
       return nullptr;
     }
@@ -2255,6 +2255,17 @@ namespace video {
     config_t config_max_ref_frames { 1920, 1080, 60, 1000, 1, 1, 1, 0, 0 };
     config_t config_autoselect { 1920, 1080, 60, 1000, 1, 0, 1, 0, 0 };
 
+    // If the encoder isn't supported at all (not even H.264), bail early
+    reset_display(disp, encoder.platform_formats->dev_type, config::video.output_name, config_autoselect);
+    if (!disp) {
+      return false;
+    }
+    if (!disp->is_codec_supported(encoder.h264.name, config_autoselect)) {
+      fg.disable();
+      BOOST_LOG(info) << "Encoder ["sv << encoder.name << "] is not supported on this GPU"sv;
+      return false;
+    }
+
   retry:
     // If we're expecting failure, use the autoselect ref config first since that will always succeed
     // if the encoder is available.