izackp/Apollo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use AVVideoScalingModeResizeAspect instead of abusing extended pixels

Browse Source 
This not only eliminates the hand-rolled aspect ratio correction (which didn't actually handle scaling),
but it also avoids us having to write to the GPU frames to add padding which improves performance.
This commit is contained in:
Cameron Gutman
2023-10-03 18:44:17 -05:00
parent a29d2e11ea
commit ca041f2934
4 changed files with 12 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
src/platform/macos/av_video.m
View File

@@ -39,11 +39,6 @@
self.pixelFormat = kCVPixelFormatType_32BGRA;
self.frameWidth = CGDisplayModeGetPixelWidth(mode);
self.frameHeight = CGDisplayModeGetPixelHeight(mode);
self.scaling = CGDisplayPixelsWide(displayID) / CGDisplayModeGetPixelWidth(mode);
self.paddingLeft = 0;
self.paddingRight = 0;
self.paddingTop = 0;
self.paddingBottom = 0;
self.minFrameDuration = CMTimeMake(1, frameRate);
self.session = [[AVCaptureSession alloc] init];
self.videoOutputs = [[NSMapTable alloc] init];
@@ -77,48 +72,8 @@
}
- (void)setFrameWidth:(int)frameWidth frameHeight:(int)frameHeight {
CGImageRef screenshot = CGDisplayCreateImage(self.displayID);
self.frameWidth = frameWidth;
self.frameHeight = frameHeight;
double screenRatio = (double) CGImageGetWidth(screenshot) / (double) CGImageGetHeight(screenshot);
double streamRatio = (double) frameWidth / (double) frameHeight;
if (screenRatio < streamRatio) {
int padding = frameWidth - (frameHeight * screenRatio);
self.paddingLeft = padding / 2;
self.paddingRight = padding - self.paddingLeft;
self.paddingTop = 0;
self.paddingBottom = 0;
}
else {
int padding = frameHeight - (frameWidth / screenRatio);
self.paddingLeft = 0;
self.paddingRight = 0;
self.paddingTop = padding / 2;
self.paddingBottom = padding - self.paddingTop;
}
// XXX: if the streamed image is larger than the native resolution, we add a black box around
// the frame. Instead the frame should be resized entirely.
int delta_width = frameWidth - (CGImageGetWidth(screenshot) + self.paddingLeft + self.paddingRight);
if (delta_width > 0) {
int adjust_left = delta_width / 2;
int adjust_right = delta_width - adjust_left;
self.paddingLeft += adjust_left;
self.paddingRight += adjust_right;
}
int delta_height = frameHeight - (CGImageGetHeight(screenshot) + self.paddingTop + self.paddingBottom);
if (delta_height > 0) {
int adjust_top = delta_height / 2;
int adjust_bottom = delta_height - adjust_top;
self.paddingTop += adjust_top;
self.paddingBottom += adjust_bottom;
}
CFRelease(screenshot);
}
- (dispatch_semaphore_t)capture:(FrameCallbackBlock)frameCallback {
@@ -128,11 +83,8 @@
[videoOutput setVideoSettings:@{
(NSString *) kCVPixelBufferPixelFormatTypeKey: [NSNumber numberWithUnsignedInt:self.pixelFormat],
(NSString *) kCVPixelBufferWidthKey: [NSNumber numberWithInt:self.frameWidth],
(NSString *) kCVPixelBufferExtendedPixelsRightKey: [NSNumber numberWithInt:self.paddingRight],
(NSString *) kCVPixelBufferExtendedPixelsLeftKey: [NSNumber numberWithInt:self.paddingLeft],
(NSString *) kCVPixelBufferExtendedPixelsTopKey: [NSNumber numberWithInt:self.paddingTop],
(NSString *) kCVPixelBufferExtendedPixelsBottomKey: [NSNumber numberWithInt:self.paddingBottom],
(NSString *) kCVPixelBufferHeightKey: [NSNumber numberWithInt:self.frameHeight]
(NSString *) kCVPixelBufferHeightKey: [NSNumber numberWithInt:self.frameHeight],
(NSString *) AVVideoScalingModeKey: AVVideoScalingModeResizeAspect,
}];
dispatch_queue_attr_t qos = dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL,