fix(macos): fix broken streaming on MacOS (#2485)

src/platform/macos/av_img_t.h

@@ -11,42 +11,56 @@
 
 namespace platf {
   struct av_sample_buf_t {
+    CMSampleBufferRef buf;
+
     explicit av_sample_buf_t(CMSampleBufferRef buf):
         buf((CMSampleBufferRef) CFRetain(buf)) {}
 
     ~av_sample_buf_t() {
-      CFRelease(buf);
+      if (buf != nullptr) {
+        CFRelease(buf);
+      }
     }
-
-    CMSampleBufferRef buf;
   };
 
   struct av_pixel_buf_t {
-    explicit av_pixel_buf_t(CVPixelBufferRef buf):
+    CVPixelBufferRef buf;
-        buf((CVPixelBufferRef) CFRetain(buf)),
+
-        locked(false) {}
+    // Constructor
+    explicit av_pixel_buf_t(CMSampleBufferRef sb):
+        buf(
+          CMSampleBufferGetImageBuffer(sb)) {
+      CVPixelBufferLockBaseAddress(buf, kCVPixelBufferLock_ReadOnly);
+    }
 
     [[nodiscard]] uint8_t *
-    lock() const {
+    data() const {
-      if (!locked) {
+      return static_cast<uint8_t *>(CVPixelBufferGetBaseAddress(buf));
-        CVPixelBufferLockBaseAddress(buf, kCVPixelBufferLock_ReadOnly);
-      }
-      return (uint8_t *) CVPixelBufferGetBaseAddress(buf);
     }
 
+    // Destructor
     ~av_pixel_buf_t() {
-      if (locked) {
+      if (buf != nullptr) {
         CVPixelBufferUnlockBaseAddress(buf, kCVPixelBufferLock_ReadOnly);
       }
-      CFRelease(buf);
     }
-
-    CVPixelBufferRef buf;
-    bool locked;
   };
 
-  struct av_img_t: public img_t {
+  struct av_img_t: img_t {
     std::shared_ptr<av_sample_buf_t> sample_buffer;
     std::shared_ptr<av_pixel_buf_t> pixel_buffer;
   };
+
+  struct temp_retain_av_img_t {
+    std::shared_ptr<av_sample_buf_t> sample_buffer;
+    std::shared_ptr<av_pixel_buf_t> pixel_buffer;
+    uint8_t *data;
+
+    temp_retain_av_img_t(
+      std::shared_ptr<av_sample_buf_t> sb,
+      std::shared_ptr<av_pixel_buf_t> pb,
+      uint8_t *dt):
+        sample_buffer(std::move(sb)),
+        pixel_buffer(std::move(pb)), data(dt) {}
+  };
 }  // namespace platf

src/platform/macos/display.mm

@@ -31,6 +31,9 @@ namespace platf {
     capture_e
     capture(const push_captured_image_cb_t &push_captured_image_cb, const pull_free_image_cb_t &pull_free_image_cb, bool *cursor) override {
       auto signal = [av_capture capture:^(CMSampleBufferRef sampleBuffer) {
+        auto new_sample_buffer = std::make_shared<av_sample_buf_t>(sampleBuffer);
+        auto new_pixel_buffer = std::make_shared<av_pixel_buf_t>(new_sample_buffer->buf);
+
         std::shared_ptr<img_t> img_out;
         if (!pull_free_image_cb(img_out)) {
           // got interrupt signal
@@ -39,17 +42,22 @@ namespace platf {
         }
         auto av_img = std::static_pointer_cast<av_img_t>(img_out);
 
-        CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
+        auto old_data_retainer = std::make_shared<temp_retain_av_img_t>(
+          av_img->sample_buffer,
+          av_img->pixel_buffer,
+          img_out->data);
 
-        av_img->sample_buffer = std::make_shared<av_sample_buf_t>(sampleBuffer);
+        av_img->sample_buffer = new_sample_buffer;
-        av_img->pixel_buffer = std::make_shared<av_pixel_buf_t>(pixelBuffer);
+        av_img->pixel_buffer = new_pixel_buffer;
-        img_out->data = av_img->pixel_buffer->lock();
+        img_out->data = new_pixel_buffer->data();
 
-        img_out->width = (int) CVPixelBufferGetWidth(pixelBuffer);
+        img_out->width = (int) CVPixelBufferGetWidth(new_pixel_buffer->buf);
-        img_out->height = (int) CVPixelBufferGetHeight(pixelBuffer);
+        img_out->height = (int) CVPixelBufferGetHeight(new_pixel_buffer->buf);
-        img_out->row_pitch = (int) CVPixelBufferGetBytesPerRow(pixelBuffer);
+        img_out->row_pitch = (int) CVPixelBufferGetBytesPerRow(new_pixel_buffer->buf);
         img_out->pixel_pitch = img_out->row_pitch / img_out->width;
 
+        old_data_retainer = nullptr;
+
         if (!push_captured_image_cb(std::move(img_out), true)) {
           // got interrupt signal
           // returning false here stops capture backend
@@ -93,19 +101,27 @@ namespace platf {
     int
     dummy_img(img_t *img) override {
       auto signal = [av_capture capture:^(CMSampleBufferRef sampleBuffer) {
+        auto new_sample_buffer = std::make_shared<av_sample_buf_t>(sampleBuffer);
+        auto new_pixel_buffer = std::make_shared<av_pixel_buf_t>(new_sample_buffer->buf);
+
         auto av_img = (av_img_t *) img;
 
-        CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
+        auto old_data_retainer = std::make_shared<temp_retain_av_img_t>(
+          av_img->sample_buffer,
+          av_img->pixel_buffer,
+          img->data);
 
-        av_img->sample_buffer = std::make_shared<av_sample_buf_t>(sampleBuffer);
+        av_img->sample_buffer = new_sample_buffer;
-        av_img->pixel_buffer = std::make_shared<av_pixel_buf_t>(pixelBuffer);
+        av_img->pixel_buffer = new_pixel_buffer;
-        img->data = av_img->pixel_buffer->lock();
+        img->data = new_pixel_buffer->data();
 
-        img->width = (int) CVPixelBufferGetWidth(pixelBuffer);
+        img->width = (int) CVPixelBufferGetWidth(new_pixel_buffer->buf);
-        img->height = (int) CVPixelBufferGetHeight(pixelBuffer);
+        img->height = (int) CVPixelBufferGetHeight(new_pixel_buffer->buf);
-        img->row_pitch = (int) CVPixelBufferGetBytesPerRow(pixelBuffer);
+        img->row_pitch = (int) CVPixelBufferGetBytesPerRow(new_pixel_buffer->buf);
         img->pixel_pitch = img->row_pitch / img->width;
 
+        old_data_retainer = nullptr;
+
         // returning false here stops capture backend
         return false;
       }];

src/platform/macos/nv12_zero_device.cpp

@@ -4,6 +4,7 @@
  */
 #include <utility>
 
+#include "src/platform/macos/av_img_t.h"
 #include "src/platform/macos/nv12_zero_device.h"
 
 #include "src/video.h"
@@ -24,6 +25,8 @@ namespace platf {
     CVPixelBufferRelease((CVPixelBufferRef) data);
   }
 
+  util::safe_ptr<AVFrame, free_frame> av_frame;
+
   int
   nv12_zero_device::convert(platf::img_t &img) {
     auto *av_img = (av_img_t *) &img;

src/platform/macos/nv12_zero_device.h

@@ -5,7 +5,6 @@
 #pragma once
 
 #include "src/platform/common.h"
-#include "src/platform/macos/av_img_t.h"
 
 struct AVFrame;
 

src/video.cpp

@@ -909,7 +909,9 @@ namespace video {
       },
       {},  // SDR-specific options
       {},  // HDR-specific options
-      {},  // Fallback options
+      {
+        { "flags"s, "-low_delay" },
+      },  // Fallback options
       std::nullopt,
       "h264_videotoolbox"s,
     },
@@ -1451,7 +1453,10 @@ namespace video {
         }
       }
 
-      ctx->flags |= (AV_CODEC_FLAG_CLOSED_GOP | AV_CODEC_FLAG_LOW_DELAY);
+      // We forcefully reset the flags to avoid clash on reuse of AVCodecContext
+      ctx->flags = 0;
+      ctx->flags |= AV_CODEC_FLAG_CLOSED_GOP | AV_CODEC_FLAG_LOW_DELAY;
+
       ctx->flags2 |= AV_CODEC_FLAG2_FAST;
 
       auto avcodec_colorspace = avcodec_colorspace_from_sunshine_colorspace(colorspace);