Add a fallback to retry codec init with more lenient config options

This allows use of low_power=1 for VAAPI to allow more performant
encoding on capable Intel hardware (like we do for QSV).

This also provides a low_power=0 fallback for QSV to allow use
on old/low-end Intel GPUs that don't support low power encoding.

Finally, this also implements a fallback to deal with the AMD
driver regression on pre-RDNA cards that causes H.264 encoding
to fail with AMF_VIDEO_ENCODER_USAGE_ULTRA_LOW_LATENCY.

src/video.cpp

@@ -364,6 +364,7 @@ namespace video {
       std::vector<option_t> common_options;
       std::vector<option_t> sdr_options;
       std::vector<option_t> hdr_options;
+      std::vector<option_t> fallback_options;
       std::optional<option_t> qp;
 
       std::string name;
@@ -578,6 +579,8 @@ namespace video {
       {},
       // HDR-specific options
       {},
+      // Fallback options
+      {},
       std::nullopt,  // QP
       "av1_nvenc"s,
     },
@@ -588,6 +591,8 @@ namespace video {
       {},
       // HDR-specific options
       {},
+      // Fallback options
+      {},
       std::nullopt,  // QP
       "hevc_nvenc"s,
     },
@@ -598,6 +603,8 @@ namespace video {
       {},
       // HDR-specific options
       {},
+      // Fallback options
+      {},
       std::nullopt,  // QP
       "h264_nvenc"s,
     },
@@ -636,6 +643,8 @@ namespace video {
       {},
       // HDR-specific options
       {},
+      // Fallback options
+      {},
       std::nullopt,
       "av1_nvenc"s,
     },
@@ -658,6 +667,7 @@ namespace video {
       {
         { "profile"s, (int) nv::profile_hevc_e::main_10 },
       },
+      {},  // Fallback options
       std::nullopt,
       "hevc_nvenc"s,
     },
@@ -677,6 +687,7 @@ namespace video {
         { "profile"s, (int) nv::profile_h264_e::high },
       },
       {},  // HDR-specific options
+      {},  // Fallback options
       std::make_optional<encoder_t::option_t>({ "qp"s, &config::video.qp }),
       "h264_nvenc"s,
     },
@@ -705,6 +716,8 @@ namespace video {
       {},
       // HDR-specific options
       {},
+      // Fallback options
+      {},
       std::make_optional<encoder_t::option_t>({ "qp"s, &config::video.qp }),
       "av1_qsv"s,
     },
@@ -727,6 +740,8 @@ namespace video {
       {
         { "profile"s, (int) qsv::profile_hevc_e::main_10 },
       },
+      // Fallback options
+      {},
       std::make_optional<encoder_t::option_t>({ "qp"s, &config::video.qp }),
       "hevc_qsv"s,
     },
@@ -748,7 +763,12 @@ namespace video {
       {
         { "profile"s, (int) qsv::profile_h264_e::high },
       },
-      {},  // HDR-specific options
+      // HDR-specific options
+      {},
+      // Fallback options
+      {
+        { "low_power"s, 0 },  // Some old/low-end Intel GPUs don't support low power encoding
+      },
       std::make_optional<encoder_t::option_t>({ "qp"s, &config::video.qp }),
       "h264_qsv"s,
     },
@@ -774,6 +794,7 @@ namespace video {
       },
       {},  // SDR-specific options
       {},  // HDR-specific options
+      {},  // Fallback options
       std::make_optional<encoder_t::option_t>({ "qp_p"s, &config::video.qp }),
       "av1_amf"s,
     },
@@ -794,6 +815,7 @@ namespace video {
       },
       {},  // SDR-specific options
       {},  // HDR-specific options
+      {},  // Fallback options
       std::make_optional<encoder_t::option_t>({ "qp_p"s, &config::video.qp }),
       "hevc_amf"s,
     },
@@ -810,8 +832,14 @@ namespace video {
         { "usage"s, &config::video.amd.amd_usage_h264 },
         { "vbaq"s, &config::video.amd.amd_vbaq },
       },
-      {},  // SDR-specific options
+      // SDR-specific options
-      {},  // HDR-specific options
+      {},
+      // HDR-specific options
+      {},
+      // Fallback options
+      {
+        { "usage"s, 2 /* AMF_VIDEO_ENCODER_USAGE_LOW_LATENCY */ },  // Workaround for https://github.com/GPUOpen-LibrariesAndSDKs/AMF/issues/410
+      },
       std::make_optional<encoder_t::option_t>({ "qp_p"s, &config::video.qp }),
       "h264_amf"s,
     },
@@ -837,6 +865,7 @@ namespace video {
       },
       {},  // SDR-specific options
       {},  // HDR-specific options
+      {},  // Fallback options
       std::make_optional<encoder_t::option_t>("qp"s, &config::video.qp),
 
 #ifdef ENABLE_BROKEN_AV1_ENCODER
@@ -861,6 +890,7 @@ namespace video {
       },
       {},  // SDR-specific options
       {},  // HDR-specific options
+      {},  // Fallback options
       std::make_optional<encoder_t::option_t>("qp"s, &config::video.qp),
       "libx265"s,
     },
@@ -872,6 +902,7 @@ namespace video {
       },
       {},  // SDR-specific options
       {},  // HDR-specific options
+      {},  // Fallback options
       std::make_optional<encoder_t::option_t>("qp"s, &config::video.qp),
       "libx264"s,
     },
@@ -889,35 +920,56 @@ namespace video {
     {
       // Common options
       {
+        { "low_power"s, 1 },
         { "async_depth"s, 1 },
         { "idr_interval"s, std::numeric_limits<int>::max() },
       },
-      {},  // SDR-specific options
+      // SDR-specific options
-      {},  // HDR-specific options
+      {},
+      // HDR-specific options
+      {},
+      // Fallback options
+      {
+        { "low_power"s, 0 },  // Not all VAAPI drivers expose LP entrypoints
+      },
       std::make_optional<encoder_t::option_t>("qp"s, &config::video.qp),
       "av1_vaapi"s,
     },
     {
       // Common options
       {
+        { "low_power"s, 1 },
         { "async_depth"s, 1 },
         { "sei"s, 0 },
         { "idr_interval"s, std::numeric_limits<int>::max() },
       },
-      {},  // SDR-specific options
+      // SDR-specific options
-      {},  // HDR-specific options
+      {},
+      // HDR-specific options
+      {},
+      // Fallback options
+      {
+        { "low_power"s, 0 },  // Not all VAAPI drivers expose LP entrypoints
+      },
       std::make_optional<encoder_t::option_t>("qp"s, &config::video.qp),
       "hevc_vaapi"s,
     },
     {
       // Common options
       {
+        { "low_power"s, 1 },
         { "async_depth"s, 1 },
         { "sei"s, 0 },
         { "idr_interval"s, std::numeric_limits<int>::max() },
       },
-      {},  // SDR-specific options
+      // SDR-specific options
-      {},  // HDR-specific options
+      {},
+      // HDR-specific options
+      {},
+      // Fallback options
+      {
+        { "low_power"s, 0 },  // Not all VAAPI drivers expose LP entrypoints
+      },
       std::make_optional<encoder_t::option_t>("qp"s, &config::video.qp),
       "h264_vaapi"s,
     },
@@ -943,6 +995,7 @@ namespace video {
       },
       {},  // SDR-specific options
       {},  // HDR-specific options
+      {},  // Fallback options
       std::nullopt,
       "av1_videotoolbox"s,
     },
@@ -956,6 +1009,7 @@ namespace video {
       },
       {},  // SDR-specific options
       {},  // HDR-specific options
+      {},  // Fallback options
       std::nullopt,
       "hevc_videotoolbox"s,
     },
@@ -969,6 +1023,7 @@ namespace video {
       },
       {},  // SDR-specific options
       {},  // HDR-specific options
+      {},  // Fallback options
       std::nullopt,
       "h264_videotoolbox"s,
     },
@@ -1402,7 +1457,17 @@ namespace video {
       return nullptr;
     }
 
-    avcodec_ctx_t ctx { avcodec_alloc_context3(codec) };
+    auto colorspace = encode_device->colorspace;
+    auto sw_fmt = (colorspace.bit_depth == 10) ? platform_formats->avcodec_pix_fmt_10bit : platform_formats->avcodec_pix_fmt_8bit;
+
+    // Allow up to 1 retry to apply the set of fallback options.
+    //
+    // Note: If we later end up needing multiple sets of
+    // fallback options, we may need to allow more retries
+    // to try applying each set.
+    avcodec_ctx_t ctx;
+    for (int retries = 0; retries < 2; retries++) {
+      ctx.reset(avcodec_alloc_context3(codec));
       ctx->width = config.width;
       ctx->height = config.height;
       ctx->time_base = AVRational { 1, config.framerate };
@@ -1446,7 +1511,6 @@ namespace video {
       ctx->flags |= (AV_CODEC_FLAG_CLOSED_GOP | AV_CODEC_FLAG_LOW_DELAY);
       ctx->flags2 |= AV_CODEC_FLAG2_FAST;
 
-    auto colorspace = encode_device->colorspace;
       auto avcodec_colorspace = avcodec_colorspace_from_sunshine_colorspace(colorspace);
 
       ctx->color_range = avcodec_colorspace.range;
@@ -1454,8 +1518,6 @@ namespace video {
       ctx->color_trc = avcodec_colorspace.transfer_function;
       ctx->colorspace = avcodec_colorspace.matrix;
 
-    auto sw_fmt = (colorspace.bit_depth == 10) ? platform_formats->avcodec_pix_fmt_10bit : platform_formats->avcodec_pix_fmt_8bit;
-
       // Used by cbs::make_sps_hevc
       ctx->sw_pix_fmt = sw_fmt;
 
@@ -1550,6 +1612,11 @@ namespace video {
       for (auto &option : (config.dynamicRange ? video_format.hdr_options : video_format.sdr_options)) {
         handle_option(option);
       }
+      if (retries > 0) {
+        for (auto &option : video_format.fallback_options) {
+          handle_option(option);
+        }
+      }
 
       if (video_format[encoder_t::CBR]) {
         auto bitrate = config.bitrate * 1000;
@@ -1591,6 +1658,15 @@ namespace video {
 
       if (auto status = avcodec_open2(ctx.get(), codec, &options)) {
         char err_str[AV_ERROR_MAX_STRING_SIZE] { 0 };
+
+        if (!video_format.fallback_options.empty() && retries == 0) {
+          BOOST_LOG(info)
+            << "Retrying with fallback configuration options for ["sv << video_format.name << "] after error: "sv
+            << av_make_error_string(err_str, AV_ERROR_MAX_STRING_SIZE, status);
+
+          continue;
+        }
+        else {
           BOOST_LOG(error)
             << "Could not open codec ["sv
             << video_format.name << "]: "sv
@@ -1598,6 +1674,11 @@ namespace video {
 
           return nullptr;
         }
+      }
+
+      // Successfully opened the codec
+      break;
+    }
 
     avcodec_frame_t frame { av_frame_alloc() };
     frame->format = ctx->pix_fmt;