Adding a Metal RGB renderer.

The new RTCMTLRGBRenderer dynamically handles both the kCVPixelFormatType_32BGRA
and the kCVPixelFormatType_32ARGB pixel formats.

Change-Id: I935532f762eff74c4b84fea9b855191f4c321fb7
Bug: webrtc:9200
Reviewed-on: https://webrtc-review.googlesource.com/72482
Reviewed-by: Kári Helgason <kthelgason@webrtc.org>
Reviewed-by: Anders Carlsson <andersc@webrtc.org>
Commit-Queue: Peter Hanspers <peterhanspers@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#23100}

sdk/BUILD.gn

@@ -274,6 +274,8 @@ if (is_ios || is_mac) {
           sources += [
             "objc/Framework/Classes/Metal/RTCMTLNV12Renderer.h",
             "objc/Framework/Classes/Metal/RTCMTLNV12Renderer.mm",
+            "objc/Framework/Classes/Metal/RTCMTLRGBRenderer.h",
+            "objc/Framework/Classes/Metal/RTCMTLRGBRenderer.mm",
             "objc/Framework/Classes/Metal/RTCMTLVideoView.m",
             "objc/Framework/Headers/WebRTC/RTCMTLVideoView.h",
           ]

sdk/objc/Framework/Classes/Metal/RTCMTLI420Renderer.mm

@@ -19,10 +19,10 @@
 
 #import "RTCMTLRenderer+Private.h"
 
-#define MTL_STRINGIFY(s) @ #s
-
 static NSString *const shaderSource = MTL_STRINGIFY(
-    using namespace metal; typedef struct {
+    using namespace metal;
+
+    typedef struct {
       packed_float2 position;
       packed_float2 texcoord;
     } Vertex;

sdk/objc/Framework/Classes/Metal/RTCMTLNV12Renderer.mm

@@ -20,10 +20,10 @@
 #import "RTCMTLRenderer+Private.h"
 #include "rtc_base/checks.h"
 
-#define MTL_STRINGIFY(s) @ #s
-
 static NSString *const shaderSource = MTL_STRINGIFY(
-    using namespace metal; typedef struct {
+    using namespace metal;
+
+    typedef struct {
       packed_float2 position;
       packed_float2 texcoord;
     } Vertex;
@@ -67,18 +67,20 @@ static NSString *const shaderSource = MTL_STRINGIFY(
 
 - (BOOL)addRenderingDestination:(__kindof MTKView *)view {
   if ([super addRenderingDestination:view]) {
-    [self initializeTextureCache];
+    return [self initializeTextureCache];
-    return YES;
   }
   return NO;
 }
 
-- (void)initializeTextureCache {
+- (BOOL)initializeTextureCache {
   CVReturn status = CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, [self currentMetalDevice],
                                               nil, &_textureCache);
   if (status != kCVReturnSuccess) {
     RTCLogError(@"Metal: Failed to initialize metal texture cache. Return status is %d", status);
+    return NO;
   }
+
+  return YES;
 }
 
 - (NSString *)shaderSource {

sdk/objc/Framework/Classes/Metal/RTCMTLRGBRenderer.h

@@ -0,0 +1,21 @@
+/*
+ *  Copyright 2018 The WebRTC Project Authors. All rights reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#import <Foundation/Foundation.h>
+
+#import "RTCMTLRenderer.h"
+
+/** @abstract RGB/BGR renderer.
+ *  @discussion This renderer handles both kCVPixelFormatType_32BGRA and kCVPixelFormatType_32ARGB.
+ */
+NS_AVAILABLE(10_11, 9_0)
+@interface RTCMTLRGBRenderer : RTCMTLRenderer
+
+@end

sdk/objc/Framework/Classes/Metal/RTCMTLRGBRenderer.mm

@@ -0,0 +1,145 @@
+/*
+ *  Copyright 2018 The WebRTC Project Authors. All rights reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#import "RTCMTLRGBRenderer.h"
+
+#import <Metal/Metal.h>
+#import <MetalKit/MetalKit.h>
+
+#import "WebRTC/RTCLogging.h"
+#import "WebRTC/RTCVideoFrame.h"
+#import "WebRTC/RTCVideoFrameBuffer.h"
+
+#import "RTCMTLRenderer+Private.h"
+#include "rtc_base/checks.h"
+
+static NSString *const shaderSource = MTL_STRINGIFY(
+    using namespace metal;
+
+    typedef struct {
+      packed_float2 position;
+      packed_float2 texcoord;
+    } Vertex;
+
+    typedef struct {
+      float4 position[[position]];
+      float2 texcoord;
+    } VertexIO;
+
+    vertex VertexIO vertexPassthrough(device Vertex * verticies[[buffer(0)]],
+                                      uint vid[[vertex_id]]) {
+      VertexIO out;
+      device Vertex &v = verticies[vid];
+      out.position = float4(float2(v.position), 0.0, 1.0);
+      out.texcoord = v.texcoord;
+      return out;
+    }
+
+    fragment half4 fragmentColorConversion(
+        VertexIO in[[stage_in]], texture2d<half, access::sample> texture[[texture(0)]],
+                                           constant bool &isARGB[[buffer(0)]]) {
+      constexpr sampler s(address::clamp_to_edge, filter::linear);
+
+      half4 out = texture.sample(s, in.texcoord);
+      if (isARGB) {
+        out = half4(out.g, out.b, out.a, out.r);
+      }
+
+      return out;
+    });
+
+@implementation RTCMTLRGBRenderer {
+  // Textures.
+  CVMetalTextureCacheRef _textureCache;
+  id<MTLTexture> _texture;
+
+  // Uniforms.
+  id<MTLBuffer> _uniformsBuffer;
+}
+
+- (BOOL)addRenderingDestination:(__kindof MTKView *)view {
+  if ([super addRenderingDestination:view]) {
+    return [self initializeTextureCache];
+  }
+  return NO;
+}
+
+- (BOOL)initializeTextureCache {
+  CVReturn status = CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, [self currentMetalDevice],
+                                              nil, &_textureCache);
+  if (status != kCVReturnSuccess) {
+    RTCLogError(@"Metal: Failed to initialize metal texture cache. Return status is %d", status);
+    return NO;
+  }
+
+  return YES;
+}
+
+- (NSString *)shaderSource {
+  return shaderSource;
+}
+
+- (BOOL)setupTexturesForFrame:(nonnull RTCVideoFrame *)frame {
+  RTC_DCHECK([frame.buffer isKindOfClass:[RTCCVPixelBuffer class]]);
+  [super setupTexturesForFrame:frame];
+  CVPixelBufferRef pixelBuffer = ((RTCCVPixelBuffer *)frame.buffer).pixelBuffer;
+
+  id<MTLTexture> gpuTexture = nil;
+  CVMetalTextureRef textureOut = nullptr;
+  bool isARGB;
+
+  int width = CVPixelBufferGetWidth(pixelBuffer);
+  int height = CVPixelBufferGetHeight(pixelBuffer);
+  OSType pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer);
+
+  MTLPixelFormat mtlPixelFormat;
+  if (pixelFormat == kCVPixelFormatType_32BGRA) {
+    mtlPixelFormat = MTLPixelFormatBGRA8Unorm;
+    isARGB = false;
+  } else if (pixelFormat == kCVPixelFormatType_32ARGB) {
+    mtlPixelFormat = MTLPixelFormatRGBA8Unorm;
+    isARGB = true;
+  } else {
+    RTC_NOTREACHED();
+    return NO;
+  }
+
+  CVReturn result = CVMetalTextureCacheCreateTextureFromImage(
+                kCFAllocatorDefault, _textureCache, pixelBuffer, nil, mtlPixelFormat,
+                width, height, 0, &textureOut);
+  if (result == kCVReturnSuccess) {
+    gpuTexture = CVMetalTextureGetTexture(textureOut);
+  }
+  CVBufferRelease(textureOut);
+
+  if (gpuTexture != nil) {
+    _texture = gpuTexture;
+    _uniformsBuffer =
+        [[self currentMetalDevice] newBufferWithBytes:&isARGB
+                                               length:sizeof(isARGB)
+                                              options:MTLResourceCPUCacheModeDefaultCache];
+    return YES;
+  }
+
+  return NO;
+}
+
+- (void)uploadTexturesToRenderEncoder:(id<MTLRenderCommandEncoder>)renderEncoder {
+  [renderEncoder setFragmentTexture:_texture atIndex:0];
+  [renderEncoder setFragmentBuffer:_uniformsBuffer offset:0 atIndex:0];
+}
+
+- (void)dealloc {
+  if (_textureCache) {
+    CFRelease(_textureCache);
+  }
+}
+
+@end

sdk/objc/Framework/Classes/Metal/RTCMTLRenderer+Private.h

@@ -11,6 +11,8 @@
 #import <Metal/Metal.h>
 #import "RTCMTLRenderer.h"
 
+#define MTL_STRINGIFY(s) @ #s
+
 NS_ASSUME_NONNULL_BEGIN
 @interface RTCMTLRenderer (Private)
 - (nullable id<MTLDevice>)currentMetalDevice;

sdk/objc/Framework/Classes/Metal/RTCMTLRenderer.mm

@@ -139,9 +139,10 @@ static const NSInteger kMaxInflightBuffers = 1;
 
   // Load metal library from source.
   NSError *libraryError = nil;
+  NSString *shaderSource = [self shaderSource];
 
   id<MTLLibrary> sourceLibrary =
-      [_device newLibraryWithSource:[self shaderSource] options:NULL error:&libraryError];
+      [_device newLibraryWithSource:shaderSource options:NULL error:&libraryError];
 
   if (libraryError) {
     RTCLogError(@"Metal: Library with source failed\n%@", libraryError);

sdk/objc/Framework/Classes/Metal/RTCMTLVideoView.m

@@ -19,16 +19,19 @@
 
 #import "RTCMTLI420Renderer.h"
 #import "RTCMTLNV12Renderer.h"
+#import "RTCMTLRGBRenderer.h"
 
 // To avoid unreconized symbol linker errors, we're taking advantage of the objc runtime.
 // Linking errors occur when compiling for architectures that don't support Metal.
 #define MTKViewClass NSClassFromString(@"MTKView")
 #define RTCMTLNV12RendererClass NSClassFromString(@"RTCMTLNV12Renderer")
 #define RTCMTLI420RendererClass NSClassFromString(@"RTCMTLI420Renderer")
+#define RTCMTLRGBRendererClass NSClassFromString(@"RTCMTLRGBRenderer")
 
 @interface RTCMTLVideoView () <MTKViewDelegate>
 @property(nonatomic, strong) RTCMTLI420Renderer *rendererI420;
 @property(nonatomic, strong) RTCMTLNV12Renderer *rendererNV12;
+@property(nonatomic, strong) RTCMTLRGBRenderer *rendererRGB;
 @property(nonatomic, strong) MTKView *metalView;
 @property(atomic, strong) RTCVideoFrame *videoFrame;
 @end
@@ -41,6 +44,7 @@
 @synthesize delegate = _delegate;
 @synthesize rendererI420 = _rendererI420;
 @synthesize rendererNV12 = _rendererNV12;
+@synthesize rendererRGB = _rendererRGB;
 @synthesize metalView = _metalView;
 @synthesize videoFrame = _videoFrame;
 
@@ -83,6 +87,10 @@
   return [[RTCMTLI420RendererClass alloc] init];
 }
 
++ (RTCMTLRGBRenderer *)createRGBRenderer {
+  return [[RTCMTLRGBRenderer alloc] init];
+}
+
 - (void)configure {
   NSAssert([RTCMTLVideoView isMetalAvailable], @"Metal not availiable on this device");
 
@@ -127,15 +135,29 @@
   }
 
   if ([videoFrame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) {
-    if (!self.rendererNV12) {
+    RTCCVPixelBuffer *buffer = (RTCCVPixelBuffer*)videoFrame.buffer;
-      self.rendererNV12 = [RTCMTLVideoView createNV12Renderer];
+    const OSType pixelFormat = CVPixelBufferGetPixelFormatType(buffer.pixelBuffer);
-      if (![self.rendererNV12 addRenderingDestination:self.metalView]) {
+    if (pixelFormat == kCVPixelFormatType_32BGRA || pixelFormat == kCVPixelFormatType_32ARGB) {
-        self.rendererNV12 = nil;
+      if (!self.rendererRGB) {
-        RTCLogError(@"Failed to create NV12 renderer");
+        self.rendererRGB = [RTCMTLVideoView createRGBRenderer];
-        return;
+        if (![self.rendererRGB addRenderingDestination:self.metalView]) {
+          self.rendererRGB = nil;
+          RTCLogError(@"Failed to create RGB renderer");
+          return;
+        }
       }
+      [self.rendererRGB drawFrame:videoFrame];
+    } else {
+      if (!self.rendererNV12) {
+        self.rendererNV12 = [RTCMTLVideoView createNV12Renderer];
+        if (![self.rendererNV12 addRenderingDestination:self.metalView]) {
+          self.rendererNV12 = nil;
+          RTCLogError(@"Failed to create NV12 renderer");
+          return;
+        }
+      }
+      [self.rendererNV12 drawFrame:videoFrame];
     }
-    [self.rendererNV12 drawFrame:videoFrame];
   } else {
     if (!self.rendererI420) {
       self.rendererI420 = [RTCMTLVideoView createI420Renderer];