From 971c784f1496eed57c46afb7f017ed4b986971d8 Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Fri, 9 Jul 2021 08:13:05 -0500 Subject: [PATCH 1/4] Add basic NVENC support on Linux We're not offloading scaling and YUV conversion from the CPU yet, so the gains aren't as high as one of the fully accelerated backends like Windows NVENC or Linux VAAPI. Still, offloading the H.264/HEVC encoding itself is an improvement over doing everything on the CPU. --- sunshine/platform/common.h | 1 + sunshine/platform/linux/display.cpp | 2 +- sunshine/video.cpp | 32 ++++++++++++++++++++++++++--- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/sunshine/platform/common.h b/sunshine/platform/common.h index e866ce4b..0699d886 100644 --- a/sunshine/platform/common.h +++ b/sunshine/platform/common.h @@ -75,6 +75,7 @@ enum class mem_type_e { system, vaapi, dxgi, + cuda, unknown }; diff --git a/sunshine/platform/linux/display.cpp b/sunshine/platform/linux/display.cpp index 867220d3..188974b0 100644 --- a/sunshine/platform/linux/display.cpp +++ b/sunshine/platform/linux/display.cpp @@ -372,7 +372,7 @@ struct shm_attr_t : public x11_attr_t { }; std::shared_ptr display(platf::mem_type_e hwdevice_type) { - if(hwdevice_type != platf::mem_type_e::system && hwdevice_type != platf::mem_type_e::vaapi) { + if(hwdevice_type != platf::mem_type_e::system && hwdevice_type != platf::mem_type_e::vaapi && hwdevice_type != platf::mem_type_e::cuda) { BOOST_LOG(error) << "Could not initialize display with the given hw device type."sv; return nullptr; } diff --git a/sunshine/video.cpp b/sunshine/video.cpp index b7c2fd2b..3652cc30 100644 --- a/sunshine/video.cpp +++ b/sunshine/video.cpp @@ -71,6 +71,7 @@ platf::pix_fmt_e map_pix_fmt(AVPixelFormat fmt); util::Either dxgi_make_hwdevice_ctx(platf::hwdevice_t *hwdevice_ctx); util::Either vaapi_make_hwdevice_ctx(platf::hwdevice_t *hwdevice_ctx); +util::Either cuda_make_hwdevice_ctx(platf::hwdevice_t *hwdevice_ctx); int hwframe_ctx(ctx_t &ctx, buffer_t &hwdevice, AVPixelFormat format); @@ -402,12 +403,16 @@ void end_capture_async(capture_thread_async_ctx_t &ctx); auto capture_thread_async = safe::make_shared(start_capture_async, end_capture_async); auto capture_thread_sync = safe::make_shared(start_capture_sync, end_capture_sync); -#ifdef _WIN32 static encoder_t nvenc { "nvenc"sv, { (int)nv::profile_h264_e::high, (int)nv::profile_hevc_e::main, (int)nv::profile_hevc_e::main_10 }, +#ifdef _WIN32 AV_HWDEVICE_TYPE_D3D11VA, AV_PIX_FMT_D3D11, +#else + AV_HWDEVICE_TYPE_CUDA, + AV_PIX_FMT_CUDA, +#endif AV_PIX_FMT_NV12, AV_PIX_FMT_P010, { { @@ -432,10 +437,16 @@ static encoder_t nvenc { std::make_optional({ "qp"s, &config::video.qp }), "h264_nvenc"s, }, +#ifdef _WIN32 DEFAULT, dxgi_make_hwdevice_ctx +#else + SYSTEM_MEMORY, + cuda_make_hwdevice_ctx +#endif }; +#ifdef _WIN32 static encoder_t amdvce { "amdvce"sv, { FF_PROFILE_H264_HIGH, FF_PROFILE_HEVC_MAIN }, @@ -537,8 +548,8 @@ static encoder_t vaapi { #endif static std::vector encoders { -#ifdef _WIN32 nvenc, +#ifdef _WIN32 amdvce, #endif #ifdef __linux__ @@ -1648,6 +1659,19 @@ util::Either vaapi_make_hwdevice_ctx(platf::hwdevice_t *base) { return hw_device_buf; } +util::Either cuda_make_hwdevice_ctx(platf::hwdevice_t *base) { + buffer_t hw_device_buf; + + auto status = av_hwdevice_ctx_create(&hw_device_buf, AV_HWDEVICE_TYPE_CUDA, nullptr, nullptr, 0); + if(status < 0) { + char string[AV_ERROR_MAX_STRING_SIZE]; + BOOST_LOG(error) << "Failed to create a CUDA device: "sv << av_make_error_string(string, AV_ERROR_MAX_STRING_SIZE, status); + return -1; + } + + return hw_device_buf; +} + #ifdef _WIN32 } @@ -1715,7 +1739,9 @@ platf::mem_type_e map_dev_type(AVHWDeviceType type) { return platf::mem_type_e::dxgi; case AV_HWDEVICE_TYPE_VAAPI: return platf::mem_type_e::vaapi; - case AV_PICTURE_TYPE_NONE: + case AV_HWDEVICE_TYPE_CUDA: + return platf::mem_type_e::cuda; + case AV_HWDEVICE_TYPE_NONE: return platf::mem_type_e::system; default: return platf::mem_type_e::unknown; From 42472bec85a148c59ec1a1ab0cdad0f0aa5aa854 Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Fri, 9 Jul 2021 17:21:37 -0500 Subject: [PATCH 2/4] Enable NVENC configuration page on Linux --- assets/web/config.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assets/web/config.html b/assets/web/config.html index a4a39c58..74a95140 100644 --- a/assets/web/config.html +++ b/assets/web/config.html @@ -528,7 +528,7 @@ } if (this.platform == "linux") { this.tabs = this.tabs.filter(el => { - return el.id !== "nv" && el.id !== "amd"; + return el.id !== "amd"; }); } From e8feb00b339fa66e8e196ab5d6d26ebcb902cac0 Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Fri, 9 Jul 2021 19:47:04 -0500 Subject: [PATCH 3/4] Use a valid RTP version to fix Wireshark parsing --- sunshine/stream.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sunshine/stream.cpp b/sunshine/stream.cpp index ac434352..aae647fe 100644 --- a/sunshine/stream.cpp +++ b/sunshine/stream.cpp @@ -674,7 +674,7 @@ void videoBroadcastThread(udp::socket &sock) { video_packet->packet.flags |= FLAG_EOF; } - video_packet->rtp.header = FLAG_EXTENSION; + video_packet->rtp.header = 0x80 | FLAG_EXTENSION; video_packet->rtp.sequenceNumber = util::endian::big(lowseq + fecIndex); }); @@ -691,7 +691,7 @@ void videoBroadcastThread(udp::socket &sock) { inspect->packet.frameIndex = packet->pts; - inspect->rtp.header = FLAG_EXTENSION; + inspect->rtp.header = 0x80 | FLAG_EXTENSION; inspect->rtp.sequenceNumber = util::endian::big(lowseq + x); } From 43dc7cf7c0a2a8acc177dde9691d2c6c937fc5a3 Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Fri, 9 Jul 2021 19:53:52 -0500 Subject: [PATCH 4/4] Fix video bitstream corruption when matching replacement isn't found NVENC doesn't always include the HEVC bitstream prefix that Sunshine is looking to patch. When this happens replace() appends a spurious 00 00 00 01 28 NALU prefix to the end of the bitstream rather than simply doing nothing. This causes varying degrees of malfunctioning depending on the client, with the worst being complete video corruption on iOS. --- sunshine/stream.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sunshine/stream.cpp b/sunshine/stream.cpp index aae647fe..27725272 100644 --- a/sunshine/stream.cpp +++ b/sunshine/stream.cpp @@ -410,11 +410,14 @@ std::vector replace(const std::string_view &original, const std::string std::vector replaced; auto begin = std::begin(original); - auto next = std::search(begin, std::end(original), std::begin(old), std::end(old)); + auto end = std::end(original); + auto next = std::search(begin, end, std::begin(old), std::end(old)); std::copy(begin, next, std::back_inserter(replaced)); - std::copy(std::begin(_new), std::end(_new), std::back_inserter(replaced)); - std::copy(next + old.size(), std::end(original), std::back_inserter(replaced)); + if(next != end) { + std::copy(std::begin(_new), std::end(_new), std::back_inserter(replaced)); + std::copy(next + old.size(), end, std::back_inserter(replaced)); + } return replaced; }