kmsgrab: Support multi-plane formats

sunshine/platform/linux/graphics.cpp

@@ -9,11 +9,13 @@
 // They aren't likely to change any time soon.
 #define fourcc_code(a, b, c, d) ((std::uint32_t)(a) | ((std::uint32_t)(b) << 8) | \
                                  ((std::uint32_t)(c) << 16) | ((std::uint32_t)(d) << 24))
+#define fourcc_mod_code(vendor, val) ((((uint64_t)vendor) << 56) | ((val) & 0x00ffffffffffffffULL))
 #define DRM_FORMAT_R8 fourcc_code('R', '8', ' ', ' ')       /* [7:0] R */
 #define DRM_FORMAT_GR88 fourcc_code('G', 'R', '8', '8')     /* [15:0] G:R 8:8 little endian */
 #define DRM_FORMAT_ARGB8888 fourcc_code('A', 'R', '2', '4') /* [31:0] A:R:G:B 8:8:8:8 little endian */
 #define DRM_FORMAT_XRGB8888 fourcc_code('X', 'R', '2', '4') /* [31:0] x:R:G:B 8:8:8:8 little endian */
 #define DRM_FORMAT_XBGR8888 fourcc_code('X', 'B', '2', '4') /* [31:0] x:B:G:R 8:8:8:8 little endian */
+#define DRM_FORMAT_MOD_INVALID fourcc_mod_code(0, ((1ULL << 56) - 1))
 
 
 #define SUNSHINE_SHADERS_DIR SUNSHINE_ASSETS_DIR "/shaders/opengl"
@@ -283,6 +285,23 @@ constexpr auto EGL_LINUX_DRM_FOURCC_EXT      = 0x3271;
 constexpr auto EGL_DMA_BUF_PLANE0_FD_EXT     = 0x3272;
 constexpr auto EGL_DMA_BUF_PLANE0_OFFSET_EXT = 0x3273;
 constexpr auto EGL_DMA_BUF_PLANE0_PITCH_EXT  = 0x3274;
+constexpr auto EGL_DMA_BUF_PLANE1_FD_EXT     = 0x3275;
+constexpr auto EGL_DMA_BUF_PLANE1_OFFSET_EXT = 0x3276;
+constexpr auto EGL_DMA_BUF_PLANE1_PITCH_EXT  = 0x3277;
+constexpr auto EGL_DMA_BUF_PLANE2_FD_EXT     = 0x3278;
+constexpr auto EGL_DMA_BUF_PLANE2_OFFSET_EXT = 0x3279;
+constexpr auto EGL_DMA_BUF_PLANE2_PITCH_EXT  = 0x327A;
+constexpr auto EGL_DMA_BUF_PLANE3_FD_EXT     = 0x3440;
+constexpr auto EGL_DMA_BUF_PLANE3_OFFSET_EXT = 0x3441;
+constexpr auto EGL_DMA_BUF_PLANE3_PITCH_EXT  = 0x3442;
+constexpr auto EGL_DMA_BUF_PLANE0_MODIFIER_LO_EXT = 0x3443;
+constexpr auto EGL_DMA_BUF_PLANE0_MODIFIER_HI_EXT = 0x3444;
+constexpr auto EGL_DMA_BUF_PLANE1_MODIFIER_LO_EXT = 0x3445;
+constexpr auto EGL_DMA_BUF_PLANE1_MODIFIER_HI_EXT = 0x3446;
+constexpr auto EGL_DMA_BUF_PLANE2_MODIFIER_LO_EXT = 0x3447;
+constexpr auto EGL_DMA_BUF_PLANE2_MODIFIER_HI_EXT = 0x3448;
+constexpr auto EGL_DMA_BUF_PLANE3_MODIFIER_LO_EXT = 0x3449;
+constexpr auto EGL_DMA_BUF_PLANE3_MODIFIER_HI_EXT = 0x344A;
 
 bool fail() {
   return eglGetError() != EGL_SUCCESS;
@@ -377,19 +396,75 @@ std::optional<ctx_t> make_ctx(display_t::pointer display) {
   return ctx;
 }
 std::optional<rgb_t> import_source(display_t::pointer egl_display, const surface_descriptor_t &xrgb) {
-  EGLAttrib img_attr_planes[13] {
+  EGLAttrib attribs[47];
-    EGL_LINUX_DRM_FOURCC_EXT, DRM_FORMAT_XRGB8888,
+  int atti = 0;
-    EGL_WIDTH, xrgb.width,
+  attribs[atti++] = EGL_WIDTH;
-    EGL_HEIGHT, xrgb.height,
+  attribs[atti++] = xrgb.width;
-    EGL_DMA_BUF_PLANE0_FD_EXT, xrgb.fd,
+  attribs[atti++] = EGL_HEIGHT;
-    EGL_DMA_BUF_PLANE0_OFFSET_EXT, xrgb.offset,
+  attribs[atti++] = xrgb.height;
-    EGL_DMA_BUF_PLANE0_PITCH_EXT, xrgb.pitch,
+  attribs[atti++] = EGL_LINUX_DRM_FOURCC_EXT;
-    EGL_NONE
+  attribs[atti++] = xrgb.fourcc;
-  };
+  if(xrgb.fds[0] >= 0) {
+    attribs[atti++] = EGL_DMA_BUF_PLANE0_FD_EXT;
+    attribs[atti++] = xrgb.fds[0];
+    attribs[atti++] = EGL_DMA_BUF_PLANE0_OFFSET_EXT;
+    attribs[atti++] = xrgb.offsets[0];
+    attribs[atti++] = EGL_DMA_BUF_PLANE0_PITCH_EXT;
+    attribs[atti++] = xrgb.pitches[0];
+    if(xrgb.modifier != DRM_FORMAT_MOD_INVALID) {
+      attribs[atti++] = EGL_DMA_BUF_PLANE0_MODIFIER_LO_EXT;
+      attribs[atti++] = xrgb.modifier & 0xFFFFFFFF;
+      attribs[atti++] = EGL_DMA_BUF_PLANE0_MODIFIER_HI_EXT;
+      attribs[atti++] = xrgb.modifier >> 32;
+    }
+  }
+  if(xrgb.fds[1] >= 0) {
+    attribs[atti++] = EGL_DMA_BUF_PLANE1_FD_EXT;
+    attribs[atti++] = xrgb.fds[1];
+    attribs[atti++] = EGL_DMA_BUF_PLANE1_OFFSET_EXT;
+    attribs[atti++] = xrgb.offsets[1];
+    attribs[atti++] = EGL_DMA_BUF_PLANE1_PITCH_EXT;
+    attribs[atti++] = xrgb.pitches[1];
+    if(xrgb.modifier != DRM_FORMAT_MOD_INVALID) {
+      attribs[atti++] = EGL_DMA_BUF_PLANE1_MODIFIER_LO_EXT;
+      attribs[atti++] = xrgb.modifier & 0xFFFFFFFF;
+      attribs[atti++] = EGL_DMA_BUF_PLANE1_MODIFIER_HI_EXT;
+      attribs[atti++] = xrgb.modifier >> 32;
+    }
+  }
+  if(xrgb.fds[2] >= 0) {
+    attribs[atti++] = EGL_DMA_BUF_PLANE2_FD_EXT;
+    attribs[atti++] = xrgb.fds[2];
+    attribs[atti++] = EGL_DMA_BUF_PLANE2_OFFSET_EXT;
+    attribs[atti++] = xrgb.offsets[2];
+    attribs[atti++] = EGL_DMA_BUF_PLANE2_PITCH_EXT;
+    attribs[atti++] = xrgb.pitches[2];
+    if(xrgb.modifier != DRM_FORMAT_MOD_INVALID) {
+        attribs[atti++] = EGL_DMA_BUF_PLANE2_MODIFIER_LO_EXT;
+        attribs[atti++] = xrgb.modifier & 0xFFFFFFFF;
+        attribs[atti++] = EGL_DMA_BUF_PLANE2_MODIFIER_HI_EXT;
+        attribs[atti++] = xrgb.modifier >> 32;
+    }
+  }
+  if(xrgb.fds[3] >= 0) {
+    attribs[atti++] = EGL_DMA_BUF_PLANE3_FD_EXT;
+    attribs[atti++] = xrgb.fds[3];
+    attribs[atti++] = EGL_DMA_BUF_PLANE3_OFFSET_EXT;
+    attribs[atti++] = xrgb.offsets[3];
+    attribs[atti++] = EGL_DMA_BUF_PLANE3_PITCH_EXT;
+    attribs[atti++] = xrgb.pitches[3];
+    if(xrgb.modifier != DRM_FORMAT_MOD_INVALID) {
+      attribs[atti++] = EGL_DMA_BUF_PLANE3_MODIFIER_LO_EXT;
+      attribs[atti++] = xrgb.modifier & 0xFFFFFFFF;
+      attribs[atti++] = EGL_DMA_BUF_PLANE3_MODIFIER_HI_EXT;
+      attribs[atti++] = xrgb.modifier >> 32;
+    }
+  }
+  attribs[atti++] = EGL_NONE;
 
   rgb_t rgb {
     egl_display,
-    eglCreateImage(egl_display, EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, nullptr, img_attr_planes),
+    eglCreateImage(egl_display, EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, nullptr, attribs),
     gl::tex_t::make(1)
   };
 
@@ -414,17 +489,17 @@ std::optional<nv12_t> import_target(display_t::pointer egl_display, std::array<f
     { EGL_LINUX_DRM_FOURCC_EXT, DRM_FORMAT_R8,
       EGL_WIDTH, r8.width,
       EGL_HEIGHT, r8.height,
-      EGL_DMA_BUF_PLANE0_FD_EXT, r8.fd,
+      EGL_DMA_BUF_PLANE0_FD_EXT, r8.fds[0],
-      EGL_DMA_BUF_PLANE0_OFFSET_EXT, r8.offset,
+      EGL_DMA_BUF_PLANE0_OFFSET_EXT, r8.offsets[0],
-      EGL_DMA_BUF_PLANE0_PITCH_EXT, r8.pitch,
+      EGL_DMA_BUF_PLANE0_PITCH_EXT, r8.pitches[0],
       EGL_NONE },
 
     { EGL_LINUX_DRM_FOURCC_EXT, DRM_FORMAT_GR88,
       EGL_WIDTH, gr88.width,
       EGL_HEIGHT, gr88.height,
-      EGL_DMA_BUF_PLANE0_FD_EXT, r8.fd,
+      EGL_DMA_BUF_PLANE0_FD_EXT, r8.fds[0],
-      EGL_DMA_BUF_PLANE0_OFFSET_EXT, gr88.offset,
+      EGL_DMA_BUF_PLANE0_OFFSET_EXT, gr88.offsets[0],
-      EGL_DMA_BUF_PLANE0_PITCH_EXT, gr88.pitch,
+      EGL_DMA_BUF_PLANE0_PITCH_EXT, gr88.pitches[0],
       EGL_NONE },
   };
 

sunshine/platform/linux/graphics.h

@@ -201,12 +201,13 @@ KITTY_USING_MOVE_T(ctx_t, (std::tuple<display_t::pointer, EGLContext>), , {
 });
 
 struct surface_descriptor_t {
-  int fd;
-
   int width;
   int height;
-  int offset;
+  int fds[4];
-  int pitch;
+  uint32_t fourcc;
+  uint64_t modifier;
+  uint32_t pitches[4];
+  uint32_t offsets[4];
 };
 
 display_t make_display(gbm::gbm_t::pointer gbm);

sunshine/platform/linux/kmsgrab.cpp

@@ -23,9 +23,63 @@ namespace fs = std::filesystem;
 namespace platf {
 
 namespace kms {
+
+class wrapper_fb {
+public:
+  wrapper_fb(drmModeFB *fb)
+    : fb { fb }, fb_id { fb->fb_id }, width { fb->width }, height { fb->height } {
+      pixel_format = DRM_FORMAT_XRGB8888;
+      modifier = DRM_FORMAT_MOD_INVALID;
+      std::fill_n(handles, 4, 0);
+      std::fill_n(pitches, 4, 0);
+      std::fill_n(offsets, 4, 0);
+      handles[0] = fb->handle;
+      pitches[0] = fb->pitch;
+    }
+
+  wrapper_fb(drmModeFB2 *fb2)
+    : fb2 { fb2 }, fb_id { fb2->fb_id }, width { fb2->width }, height { fb2->height } {
+      pixel_format = fb2->pixel_format;
+      modifier = fb2->modifier;
+      memcpy(handles, fb2->handles, sizeof(handles));
+      memcpy(pitches, fb2->pitches, sizeof(pitches));
+      memcpy(offsets, fb2->offsets, sizeof(offsets));
+    }
+
+  ~wrapper_fb() {
+    if(fb) {
+      drmModeFreeFB(fb);
+    } else if(fb2) {
+      drmModeFreeFB2(fb2);
+    }
+  }
+
+  egl::surface_descriptor_t to_sd() const {
+    egl::surface_descriptor_t sd = {};
+    sd.width = width;
+    sd.height = height;
+    sd.fourcc = pixel_format;
+    sd.modifier = modifier;
+    memcpy(sd.pitches, pitches, sizeof(pitches));
+    memcpy(sd.offsets, offsets, sizeof(offsets));
+    return sd;
+  }
+
+  drmModeFB *fb = nullptr;
+  drmModeFB2 *fb2 = nullptr;
+  uint32_t fb_id;
+  uint32_t width;
+  uint32_t height;
+  uint32_t pixel_format;
+  uint64_t modifier;
+  uint32_t handles[4];
+  uint32_t pitches[4];
+  uint32_t offsets[4];
+};
+
 using plane_res_t = util::safe_ptr<drmModePlaneRes, drmModeFreePlaneResources>;
 using plane_t     = util::safe_ptr<drmModePlane, drmModeFreePlane>;
-using fb_t        = util::safe_ptr<drmModeFB, drmModeFreeFB>;
+using fb_t        = std::unique_ptr<wrapper_fb>;
 using crtc_t      = util::safe_ptr<drmModeCrtc, drmModeFreeCrtc>;
 using obj_prop_t  = util::safe_ptr<drmModeObjectProperties, drmModeFreeObjectProperties>;
 using prop_t      = util::safe_ptr<drmModePropertyRes, drmModeFreeProperty>;
@@ -122,7 +176,11 @@ public:
   }
 
   fb_t fb(plane_t::pointer plane) {
-    return drmModeGetFB(fd.el, plane->fb_id);
+    auto fb = drmModeGetFB2(fd.el, plane->fb_id);
+    if(fb) {
+      return std::make_unique<wrapper_fb>(fb);
+    }
+    return std::make_unique<wrapper_fb>(drmModeGetFB(fd.el, plane->fb_id));
   }
 
   crtc_t crtc(std::uint32_t id) {
@@ -212,9 +270,7 @@ void print(plane_t::pointer plane, fb_t::pointer fb, crtc_t::pointer crtc) {
 
   BOOST_LOG(debug)
     << "Resolution: "sv << fb->width << 'x' << fb->height
-    << ": Pitch: "sv << fb->pitch
+    << ": Pitch: "sv << fb->pitches[0];
-    << ": bpp: "sv << fb->bpp
-    << ": depth: "sv << fb->depth;
 
   std::stringstream ss;
 
@@ -262,9 +318,11 @@ public:
       auto crct = card.crtc(plane->crtc_id);
 
       bool different =
-        fb->width != img_width ||
+        fb->width != surf_sd.width ||
-        fb->height != img_height ||
+        fb->height != surf_sd.height ||
-        fb->pitch != pitch ||
+        fb->pixel_format != surf_sd.fourcc ||
+        fb->modifier != surf_sd.modifier ||
+        fb->pitches[0] != surf_sd.pitches[0] ||
         crct->x != offset_x ||
         crct->y != offset_y;
 
@@ -335,16 +393,21 @@ public:
           return -1;
         }
 
-        if(!fb->handle) {
+        if(!fb->handles[0]) {
           BOOST_LOG(error)
             << "Couldn't get handle for DRM Framebuffer ["sv << plane->fb_id << "]: Possibly not permitted: do [sudo setcap cap_sys_admin+ep sunshine]"sv;
           return -1;
         }
 
-        fb_fd = card.handleFD(fb->handle);
+        for(int i = 0; i < 4; ++i) {
-        if(fb_fd.el < 0) {
+          if(!fb->handles[i]) {
-          BOOST_LOG(error) << "Couldn't get primary file descriptor for Framebuffer ["sv << fb->fb_id << "]: "sv << strerror(errno);
+            break;
-          continue;
+          }
+          fb_fd[i] = card.handleFD(fb->handles[i]);
+          if(fb_fd[i].el < 0) {
+            BOOST_LOG(error) << "Couldn't get primary file descriptor for Framebuffer ["sv << fb->fb_id << "]: "sv << strerror(errno);
+            continue;
+          }
         }
 
         BOOST_LOG(info) << "Found monitor for DRM screencasting"sv;
@@ -352,14 +415,11 @@ public:
         auto crct = card.crtc(plane->crtc_id);
         kms::print(plane.get(), fb.get(), crct.get());
 
-        img_width  = fb->width;
+        surf_sd = fb->to_sd();
-        img_height = fb->height;
 
         width  = crct->width;
         height = crct->height;
 
-        pitch = fb->pitch;
-
         this->env_width  = ::platf::kms::env_width;
         this->env_height = ::platf::kms::env_height;
 
@@ -397,11 +457,9 @@ public:
 
   capture_e status;
 
-  int img_width, img_height;
-  int pitch;
-
   card_t card;
-  file_t fb_fd;
+  file_t fb_fd[4];
+  egl::surface_descriptor_t surf_sd;
 
   std::optional<x11::cursor_t> cursor_opt;
 
@@ -441,14 +499,11 @@ public:
 
     ctx = std::move(*ctx_opt);
 
-    auto rgb_opt = egl::import_source(display.get(),
+    auto sd = surf_sd;
-      {
+    for(auto i = 0; i < 4; ++i) {
-        fb_fd.el,
+      sd.fds[i] = fb_fd[i].el;
-        img_width,
+    }
-        img_height,
+    auto rgb_opt = egl::import_source(display.get(), sd);
-        0,
-        pitch,
-      });
 
     if(!rgb_opt) {
       return -1;
@@ -541,14 +596,11 @@ public:
 
   std::shared_ptr<hwdevice_t> make_hwdevice(pix_fmt_e pix_fmt) override {
     if(mem_type == mem_type_e::vaapi) {
-      return va::make_hwdevice(width, height, dup(card.fd.el), offset_x, offset_y,
+      auto sd = surf_sd;
-        {
+      for(auto i = 0; i < 4; ++i) {
-          fb_fd.el,
+        sd.fds[i] = fb_fd[i].el;
-          img_width,
+      }
-          img_height,
+      return va::make_hwdevice(width, height, dup(card.fd.el), offset_x, offset_y, sd);
-          0,
-          pitch,
-        });
     }
 
     BOOST_LOG(error) << "Unsupported pixel format for egl::display_vram_t: "sv << platf::from_pix_fmt(pix_fmt);
@@ -689,7 +741,7 @@ std::vector<std::string> kms_display_names() {
         continue;
       }
 
-      if(!fb->handle) {
+      if(!fb->handles[0]) {
         BOOST_LOG(error)
           << "Couldn't get handle for DRM Framebuffer ["sv << plane->fb_id << "]: Possibly not permitted: do [sudo setcap cap_sys_admin+ep sunshine]"sv;
         break;

sunshine/platform/linux/vaapi.cpp

@@ -351,18 +351,22 @@ public:
       display.get(),
       std::move(fds),
       {
-        prime.objects[prime.layers[0].object_index[0]].fd,
         (int)prime.width,
         (int)prime.height,
-        (int)prime.layers[0].offset[0],
+        {prime.objects[prime.layers[0].object_index[0]].fd, -1, -1, -1},
-        (int)prime.layers[0].pitch[0],
+        0,
+        0,
+        {prime.layers[0].pitch[0], 0, 0, 0},
+        {prime.layers[0].offset[0], 0, 0, 0}
       },
       {
-        prime.objects[prime.layers[0].object_index[1]].fd,
         (int)prime.width / 2,
         (int)prime.height / 2,
-        (int)prime.layers[0].offset[1],
+        {prime.objects[prime.layers[0].object_index[1]].fd, -1, -1, -1},
-        (int)prime.layers[0].pitch[1],
+        0,
+        0,
+        {prime.layers[0].pitch[1], 0, 0, 0},
+        {prime.layers[0].offset[1], 0, 0, 0}
       });
 
     if(!nv12_opt) {