Terminate early when hang in encoder is detected

src/entry_handler.cpp

@@ -101,6 +101,8 @@ namespace lifetime {
 #else
     std::raise(SIGTRAP);
 #endif
+    // If debug trap still doesn't work, abort
+    abort();
   }
 
   char **

src/platform/windows/virtual_display.cpp

@@ -213,9 +213,20 @@ void closeVDisplayDevice() {
 }
 
 DRIVER_STATUS openVDisplayDevice() {
-	SUDOVDA_DRIVER_HANDLE = OpenDevice(&SUVDA_INTERFACE_GUID);
-	if (SUDOVDA_DRIVER_HANDLE == INVALID_HANDLE_VALUE) {
-		return DRIVER_STATUS::FAILED;
+	uint32_t retryInterval = 20;
+	while (true) {
+		SUDOVDA_DRIVER_HANDLE = OpenDevice(&SUVDA_INTERFACE_GUID);
+		if (SUDOVDA_DRIVER_HANDLE == INVALID_HANDLE_VALUE) {
+			if (retryInterval > 320) {
+				printf("[SUDOVDA] Open device failed!\n");
+				return DRIVER_STATUS::FAILED;
+			}
+			retryInterval *= 2;
+			Sleep(retryInterval);
+			continue;
+		}
+
+		break;
 	}
 
 	if (!CheckProtocolCompatible(SUDOVDA_DRIVER_HANDLE)) {
@@ -317,7 +328,7 @@ std::wstring createVirtualDisplay(
 	wchar_t deviceName[CCHDEVICENAME]{};
 	while (!GetAddedDisplayName(output, deviceName)) {
 		Sleep(retryInterval);
-		if (retryInterval > 160) {
+		if (retryInterval > 320) {
 			printf("[SUDOVDA] Cannot get name for newly added virtual display!\n");
 			return std::wstring();
 		}

src/process.cpp

@@ -510,10 +510,14 @@ namespace proc {
   }
 
   void
-  proc_t::terminate() {
+  proc_t::terminate(bool immediate) {
     std::error_code ec;
     placebo = false;
-    terminate_process_group(_process, _process_group, _app.exit_timeout);
+
+    if (!immediate) {
+      terminate_process_group(_process, _process_group, _app.exit_timeout);
+    }
+
     _process = boost::process::v1::child();
     _process_group = boost::process::v1::group();
 

src/process.h

@@ -118,7 +118,7 @@ namespace proc {
     boost::process::environment
     get_env();
     void
-    terminate();
+    terminate(bool immediate = false);
 
   private:
     int _app_id;

src/rtsp.cpp

@@ -1050,6 +1050,12 @@ namespace rtsp_stream {
       config.monitor.chromaSamplingType = util::from_view(args.at("x-ss-video[0].chromaSamplingType"sv));
       config.monitor.enableIntraRefresh = util::from_view(args.at("x-ss-video[0].intraRefresh"sv));
 
+      if (config::video.limit_framerate) {
+        config.monitor.encodingFramerate = session.fps;
+      } else {
+        config.monitor.encodingFramerate = config.monitor.framerate;
+      }
+
       configuredBitrateKbps = util::from_view(args.at("x-ml-video.configuredBitrateKbps"sv));
     }
     catch (std::out_of_range &) {

src/video.cpp

@@ -1823,7 +1823,13 @@ namespace video {
 
     // set minimum frame time, avoiding violation of client-requested target framerate
     auto minimum_frame_time = std::chrono::milliseconds(1000 / std::min(config.framerate, (config::video.min_fps_factor * 10)));
+    auto frame_threshold = std::chrono::milliseconds(1000 / config.encodingFramerate);
+    // Leave 1ms headroom for slight variations
+    if (frame_threshold >= 2ms) {
+      frame_threshold -= 1ms;
+    }
     BOOST_LOG(debug) << "Minimum frame time set to "sv << minimum_frame_time.count() << "ms, based on min fps factor of "sv << config::video.min_fps_factor << "."sv;
+    BOOST_LOG(info) << "Frame threshold: "sv << frame_threshold;
 
     auto shutdown_event = mail->event<bool>(mail::shutdown);
     auto packets = mail::man->queue<packet_t>(mail::video_packets);
@@ -1841,6 +1847,32 @@ namespace video {
       }
     }
 
+    std::chrono::steady_clock::time_point last_frame_timestamp;
+    std::chrono::steady_clock::time_point last_encoded_timestamp = std::chrono::steady_clock::now();
+    bool stop_encoding = false;
+
+    std::thread alive_check_thread([&last_encoded_timestamp, &stop_encoding]{
+      uint8_t fail_count = 0;
+      std::chrono::steady_clock::time_point _last_timestamp = last_encoded_timestamp;
+      for (;;) {
+        if (stop_encoding) return;
+        std::this_thread::sleep_for(1s);
+        if (last_encoded_timestamp == _last_timestamp) {
+          fail_count += 1;
+          if (fail_count > 3) {
+            BOOST_LOG(error) << "Hang detected!!! Aborting..."sv;
+            proc::proc.terminate(true);
+            std::this_thread::sleep_for(1s);
+            abort();
+            return;
+          }
+        } else {
+          fail_count = 0;
+          _last_timestamp = last_encoded_timestamp;
+        };
+      }
+    });
+
     while (true) {
       // Break out of the encoding loop if any of the following are true:
       // a) The stream is ending
@@ -1876,9 +1908,13 @@ namespace video {
       if (!requested_idr_frame || images->peek()) {
         if (auto img = images->pop(minimum_frame_time)) {
           frame_timestamp = img->frame_timestamp;
+          // If new frame comes in way too fast, just drop
+          if (*frame_timestamp - last_frame_timestamp < frame_threshold) {
+            continue;
+          }
           if (session->convert(*img)) {
             BOOST_LOG(error) << "Could not convert image"sv;
-            return;
+            break;
           }
         }
         else if (!images->running()) {
@@ -1888,11 +1924,17 @@ namespace video {
 
       if (encode(frame_nr++, *session, packets, channel_data, frame_timestamp)) {
         BOOST_LOG(error) << "Could not encode video packet"sv;
-        return;
+        break;
       }
 
+      last_frame_timestamp = *frame_timestamp;
+      last_encoded_timestamp = std::chrono::steady_clock::now();
+
       session->request_normal_frame();
     }
+
+    stop_encoding = true;
+    alive_check_thread.join();
   }
 
   input::touch_port_t

src/video.h

@@ -22,6 +22,7 @@ namespace video {
     int width;  // Video width in pixels
     int height;  // Video height in pixels
     int framerate;  // Requested framerate, used in individual frame bitrate budget calculation
+    int encodingFramerate; // Requested display framerate
     int bitrate;  // Video bitrate in kilobits (1000 bits) for requested framerate
     int slicesPerFrame;  // Number of slices per frame
     int numRefFrames;  // Max number of reference frames