Metal rendering should account for cropping.

Also:
- added a rotation override to allow ignoring frame rotation
- fixed a couple of minor issues
- made it possible to run the MTKView without the DisplayLink

Bug: webrtc:9301
Change-Id: Ia83c152d9b6d45d56ceb80d287b5d3eacfaebddd
Reviewed-on: https://webrtc-review.googlesource.com/78282
Reviewed-by: Kári Helgason <kthelgason@webrtc.org>
Reviewed-by: Anders Carlsson <andersc@webrtc.org>
Commit-Queue: Peter Hanspers <peterhanspers@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#23452}

sdk/objc/Framework/Classes/Metal/RTCMTLVideoView.m

@@ -29,17 +29,16 @@
 #define RTCMTLRGBRendererClass NSClassFromString(@"RTCMTLRGBRenderer")
 
 @interface RTCMTLVideoView () <MTKViewDelegate>
-@property(nonatomic, strong) RTCMTLI420Renderer *rendererI420;
-@property(nonatomic, strong) RTCMTLNV12Renderer *rendererNV12;
-@property(nonatomic, strong) RTCMTLRGBRenderer *rendererRGB;
-@property(nonatomic, strong) MTKView *metalView;
-@property(atomic, strong) RTCVideoFrame *videoFrame;
+@property(nonatomic) RTCMTLI420Renderer *rendererI420;
+@property(nonatomic) RTCMTLNV12Renderer *rendererNV12;
+@property(nonatomic) RTCMTLRGBRenderer *rendererRGB;
+@property(nonatomic) MTKView *metalView;
+@property(atomic) RTCVideoFrame *videoFrame;
+@property(nonatomic) CGSize videoFrameSize;
+@property(nonatomic) int64_t lastFrameTimeNs;
 @end
 
-@implementation RTCMTLVideoView {
-  int64_t _lastFrameTimeNs;
-  CGSize _videoFrameSize;
-}
+@implementation RTCMTLVideoView
 
 @synthesize delegate = _delegate;
 @synthesize rendererI420 = _rendererI420;
@@ -47,6 +46,11 @@
 @synthesize rendererRGB = _rendererRGB;
 @synthesize metalView = _metalView;
 @synthesize videoFrame = _videoFrame;
+@synthesize useDisplayLink = _useDisplayLink;
+@synthesize videoFrameSize = _videoFrameSize;
+@synthesize lastFrameTimeNs = _lastFrameTimeNs;
+@synthesize enabled = _enabled;
+@synthesize rotationOverride = _rotationOverride;
 
 - (instancetype)initWithFrame:(CGRect)frameRect {
   self = [super initWithFrame:frameRect];
@@ -64,8 +68,36 @@
   return self;
 }
 
+- (void)setUseDisplayLink:(BOOL)useDisplayLink {
+  _useDisplayLink = useDisplayLink;
+  [self updateRunningState];
+}
+
+- (void)setEnabled:(BOOL)enabled {
+  _enabled = enabled;
+  [self updateRunningState];
+}
+
+- (UIViewContentMode)videoContentMode {
+  return self.metalView.contentMode;
+}
+
+- (void)setVideoContentMode:(UIViewContentMode)mode {
+  self.metalView.contentMode = mode;
+}
+
 #pragma mark - Private
 
+- (void)updateRunningState {
+  if (self.useDisplayLink) {
+    self.metalView.paused = !self.enabled;
+    self.metalView.enableSetNeedsDisplay = YES;
+  } else {
+    self.metalView.paused = YES;
+    self.metalView.enableSetNeedsDisplay = NO;
+  }
+}
+
 + (BOOL)isMetalAvailable {
 #if defined(RTC_SUPPORTS_METAL)
   return MTLCreateSystemDefaultDevice() != nil;
@@ -74,11 +106,6 @@
 #endif
 }
 
-+ (MTKView *)createMetalView:(CGRect)frame {
-  MTKView *view = [[MTKViewClass alloc] initWithFrame:frame];
-  return view;
-}
-
 + (RTCMTLNV12Renderer *)createNV12Renderer {
   return [[RTCMTLNV12RendererClass alloc] init];
 }
@@ -94,33 +121,28 @@
 - (void)configure {
   NSAssert([RTCMTLVideoView isMetalAvailable], @"Metal not availiable on this device");
 
-  _metalView = [RTCMTLVideoView createMetalView:self.bounds];
-  [self configureMetalView];
-}
+  _enabled = YES;
+  _useDisplayLink = YES;
+  [self updateRunningState];
 
-- (void)configureMetalView {
-  if (_metalView) {
-    _metalView.delegate = self;
-    [self addSubview:_metalView];
-    _metalView.contentMode = UIViewContentModeScaleAspectFit;
-    _videoFrameSize = CGSizeZero;
-  }
+  self.metalView = [[MTKViewClass alloc] initWithFrame:self.bounds];
+  self.metalView.delegate = self;
+  self.metalView.paused = YES;
+  self.metalView.enableSetNeedsDisplay = NO;
+  self.metalView.contentMode = UIViewContentModeScaleAspectFill;
+  [self addSubview:self.metalView];
+  self.videoFrameSize = CGSizeZero;
 }
 
-- (void)setVideoContentMode:(UIViewContentMode)mode {
-  _metalView.contentMode = mode;
-}
-
-#pragma mark - Private
-
 - (void)layoutSubviews {
   [super layoutSubviews];
+
   CGRect bounds = self.bounds;
-  _metalView.frame = bounds;
-  if (!CGSizeEqualToSize(_videoFrameSize, CGSizeZero)) {
-    _metalView.drawableSize = _videoFrameSize;
+  self.metalView.frame = bounds;
+  if (!CGSizeEqualToSize(self.videoFrameSize, CGSizeZero)) {
+    self.metalView.drawableSize = [self drawableSize];
   } else {
-    _metalView.drawableSize = bounds.size;
+    self.metalView.drawableSize = bounds.size;
   }
 }
 
@@ -130,10 +152,11 @@
   NSAssert(view == self.metalView, @"Receiving draw callbacks from foreign instance.");
   RTCVideoFrame *videoFrame = self.videoFrame;
   // Skip rendering if we've already rendered this frame.
-  if (!videoFrame || videoFrame.timeStampNs == _lastFrameTimeNs) {
+  if (!videoFrame || videoFrame.timeStampNs == self.lastFrameTimeNs) {
     return;
   }
 
+  RTCMTLRenderer *renderer;
   if ([videoFrame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) {
     RTCCVPixelBuffer *buffer = (RTCCVPixelBuffer*)videoFrame.buffer;
     const OSType pixelFormat = CVPixelBufferGetPixelFormatType(buffer.pixelBuffer);
@@ -146,7 +169,7 @@
           return;
         }
       }
-      [self.rendererRGB drawFrame:videoFrame];
+      renderer = self.rendererRGB;
     } else {
       if (!self.rendererNV12) {
         self.rendererNV12 = [RTCMTLVideoView createNV12Renderer];
@@ -156,7 +179,7 @@
           return;
         }
       }
-      [self.rendererNV12 drawFrame:videoFrame];
+      renderer = self.rendererNV12;
     }
   } else {
     if (!self.rendererI420) {
@@ -167,30 +190,82 @@
         return;
       }
     }
-    [self.rendererI420 drawFrame:videoFrame];
+    renderer = self.rendererI420;
   }
-  _lastFrameTimeNs = videoFrame.timeStampNs;
+
+  renderer.rotationOverride = self.rotationOverride;
+
+  [renderer drawFrame:videoFrame];
+  self.lastFrameTimeNs = videoFrame.timeStampNs;
 }
 
 - (void)mtkView:(MTKView *)view drawableSizeWillChange:(CGSize)size {
 }
 
+- (RTCVideoRotation)frameRotation {
+  if (self.rotationOverride) {
+    RTCVideoRotation rotation;
+#if defined(__IPHONE_11_0) && (__IPHONE_OS_VERSION_MAX_ALLOWED >= __IPHONE_11_0)
+    if (@available(iOS 11, *)) {
+      [self.rotationOverride getValue:&rotation size:sizeof(rotation)];
+    } else
+#endif
+    {
+      [self.rotationOverride getValue:&rotation];
+    }
+    return rotation;
+  }
+
+  return self.videoFrame.rotation;
+}
+
+- (CGSize)drawableSize {
+  // Flip width/height if the rotations are not the same.
+  CGSize videoFrameSize = self.videoFrameSize;
+
+  BOOL useLandscape = ([self frameRotation] == RTCVideoRotation_0) ||
+      ([self frameRotation] == RTCVideoRotation_180);
+  BOOL sizeIsLandscape = (self.videoFrame.rotation == RTCVideoRotation_0) ||
+      (self.videoFrame.rotation == RTCVideoRotation_180);
+
+  if (useLandscape == sizeIsLandscape) {
+    return videoFrameSize;
+  } else {
+    return CGSizeMake(videoFrameSize.height, videoFrameSize.width);
+  }
+}
+
 #pragma mark - RTCVideoRenderer
 
 - (void)setSize:(CGSize)size {
-  self.metalView.drawableSize = size;
+  __weak RTCMTLVideoView *weakSelf = self;
   dispatch_async(dispatch_get_main_queue(), ^{
-    _videoFrameSize = size;
-    [self.delegate videoView:self didChangeVideoSize:size];
+    RTCMTLVideoView *strongSelf = weakSelf;
+
+    strongSelf.videoFrameSize = size;
+    CGSize drawableSize = [strongSelf drawableSize];
+
+    strongSelf.metalView.drawableSize = drawableSize;
+    [strongSelf setNeedsLayout];
+    [strongSelf.delegate videoView:self didChangeVideoSize:size];
   });
 }
 
 - (void)renderFrame:(nullable RTCVideoFrame *)frame {
+  if (!self.isEnabled) {
+    return;
+  }
+
   if (frame == nil) {
     RTCLogInfo(@"Incoming frame is nil. Exiting render callback.");
     return;
   }
+
   self.videoFrame = frame;
+
+  if (!self.useDisplayLink) {
+    [self.metalView draw];
+  }
 }
 
 @end