zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

iOS/MacOS:Refactor metal rendering by extracting common implementation

Browse Source 
in separate super class.

Included changes:
- Fix rendering on iOS to support NV12 and i420 frames
- Improve code style
- Update build targets
- Update tests

BUG=webrtc:7079

Review-Url: https://codereview.webrtc.org/2784243003
Cr-Commit-Position: refs/heads/master@{#17923}
This commit is contained in:
denicija
2017-04-28 02:14:54 -07:00
committed by Commit bot
parent 7a38fd2628
commit d208815685
14 changed files with 534 additions and 584 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLI420Renderer.h
View File

@ -9,34 +9,9 @@
*/
#import <Foundation/Foundation.h>
#import <MetalKit/MTKView.h>
#import "WebRTC/RTCVideoFrame.h"
#import "RTCMTLRenderer.h"
/**
* Protocol defining ability to render RTCVideoFrame in Metal enabled views.
*/
@protocol RTCMTLRenderer <NSObject>
/**
* Method to be implemented to perform actual rendering of the provided frame.
*
* @param frame The frame to be rendered.
*/
- (void)drawFrame:(RTCVideoFrame *)frame;
@end
NS_AVAILABLE_MAC(10.11)
/**
* Implementation of RTCMTLRenderer protocol for rendering native nv12 video frames.
*/
@interface RTCMTLI420Renderer : NSObject <RTCMTLRenderer>
/**
* Sets the provided view as rendering destination if possible.
*
* If not possible method returns NO and callers of the method are responisble for performing
* cleanups.
*/
- (BOOL)addRenderingDestination:(__kindof MTKView *)view;
NS_AVAILABLE(10_11, 9_0)
@interface RTCMTLI420Renderer : RTCMTLRenderer
@end

227
webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLI420Renderer.mm
View File

@ -16,46 +16,10 @@
#import "WebRTC/RTCLogging.h"
#import "WebRTC/RTCVideoFrame.h"
#include "webrtc/api/video/video_rotation.h"
#import "RTCMTLRenderer+Private.h"
#define MTL_STRINGIFY(s) @ #s
// As defined in shaderSource.
static NSString *const vertexFunctionName = @"vertexPassthrough";
static NSString *const fragmentFunctionName = @"fragmentColorConversion";
static NSString *const pipelineDescriptorLabel = @"RTCPipeline";
static NSString *const commandBufferLabel = @"RTCCommandBuffer";
static NSString *const renderEncoderLabel = @"RTCEncoder";
static NSString *const renderEncoderDebugGroup = @"RTCDrawFrame";
static const float cubeVertexData[64] = {
-1.0, -1.0, 0.0, 1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 0.0,
// rotation = 90, offset = 16.
-1.0, -1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 0.0, -1.0, 1.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0,
// rotation = 180, offset = 32.
-1.0, -1.0, 1.0, 0.0, 1.0, -1.0, 0.0, 0.0, -1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 1.0,
// rotation = 270, offset = 48.
-1.0, -1.0, 0.0, 0.0, 1.0, -1.0, 0.0, 1.0, -1.0, 1.0, 1.0, 0.0, 1.0, 1.0, 1.0, 1.0,
};
static inline int offsetForRotation(webrtc::VideoRotation rotation) {
switch (rotation) {
case webrtc::kVideoRotation_0:
return 0;
case webrtc::kVideoRotation_90:
return 16;
case webrtc::kVideoRotation_180:
return 32;
case webrtc::kVideoRotation_270:
return 48;
}
return 0;
}
static NSString *const shaderSource = MTL_STRINGIFY(
using namespace metal; typedef struct {
packed_float2 position;
@ -103,24 +67,7 @@ static NSString *const shaderSource = MTL_STRINGIFY(
return half4(out);
});
// The max number of command buffers in flight.
// For now setting it up to 1.
// In future we might use triple buffering method if it improves performance.
static const NSInteger kMaxInflightBuffers = 1;
@implementation RTCMTLI420Renderer {
__kindof MTKView *_view;
// Controller.
dispatch_semaphore_t _inflight_semaphore;
// Renderer.
id<MTLDevice> _device;
id<MTLCommandQueue> _commandQueue;
id<MTLLibrary> _defaultLibrary;
id<MTLRenderPipelineState> _pipelineState;
// Textures.
id<MTLTexture> _yTexture;
id<MTLTexture> _uTexture;
@ -133,161 +80,23 @@ static const NSInteger kMaxInflightBuffers = 1;
int _height;
int _chromaWidth;
int _chromaHeight;
// Buffers.
id<MTLBuffer> _vertexBuffer;
// RTC Frame parameters.
int _offset;
}
- (instancetype)init {
if (self = [super init]) {
// Offset of 0 is equal to rotation of 0.
_offset = 0;
_inflight_semaphore = dispatch_semaphore_create(kMaxInflightBuffers);
}
#pragma mark - Virtual
return self;
}
- (BOOL)addRenderingDestination:(__kindof MTKView *)view {
return [self setupWithView:view];
}
#pragma mark - Private
- (BOOL)setupWithView:(__kindof MTKView *)view {
BOOL success = NO;
if ([self setupMetal]) {
[self setupView:view];
[self loadAssets];
[self setupBuffers];
success = YES;
}
return success;
}
#pragma mark - GPU methods
- (BOOL)setupMetal {
// Set the view to use the default device.
_device = MTLCreateSystemDefaultDevice();
if (!_device) {
return NO;
}
// Create a new command queue.
_commandQueue = [_device newCommandQueue];
// Load metal library from source.
NSError *libraryError = nil;
id<MTLLibrary> sourceLibrary =
[_device newLibraryWithSource:shaderSource options:NULL error:&libraryError];
if (libraryError) {
RTCLogError(@"Metal: Library with source failed\n%@", libraryError);
return NO;
}
if (!sourceLibrary) {
RTCLogError(@"Metal: Failed to load library. %@", libraryError);
return NO;
}
_defaultLibrary = sourceLibrary;
return YES;
}
- (void)setupView:(__kindof MTKView *)view {
view.device = _device;
view.preferredFramesPerSecond = 30;
view.autoResizeDrawable = NO;
// We need to keep reference to the view as it's needed down the rendering pipeline.
_view = view;
}
- (void)loadAssets {
id<MTLFunction> vertexFunction = [_defaultLibrary newFunctionWithName:vertexFunctionName];
id<MTLFunction> fragmentFunction = [_defaultLibrary newFunctionWithName:fragmentFunctionName];
MTLRenderPipelineDescriptor *pipelineDescriptor = [[MTLRenderPipelineDescriptor alloc] init];
pipelineDescriptor.label = pipelineDescriptorLabel;
pipelineDescriptor.vertexFunction = vertexFunction;
pipelineDescriptor.fragmentFunction = fragmentFunction;
pipelineDescriptor.colorAttachments[0].pixelFormat = _view.colorPixelFormat;
pipelineDescriptor.depthAttachmentPixelFormat = MTLPixelFormatInvalid;
NSError *error = nil;
_pipelineState = [_device newRenderPipelineStateWithDescriptor:pipelineDescriptor error:&error];
if (!_pipelineState) {
RTCLogError(@"Metal: Failed to create pipeline state. %@", error);
}
}
- (void)setupBuffers {
_vertexBuffer = [_device newBufferWithBytes:cubeVertexData
length:sizeof(cubeVertexData)
options:MTLStorageModeShared];
}
- (void)render {
dispatch_semaphore_wait(_inflight_semaphore, DISPATCH_TIME_FOREVER);
id<MTLCommandBuffer> commandBuffer = [_commandQueue commandBuffer];
commandBuffer.label = commandBufferLabel;
__block dispatch_semaphore_t block_semaphore = _inflight_semaphore;
[commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> _Nonnull) {
dispatch_semaphore_signal(block_semaphore);
}];
MTLRenderPassDescriptor *_renderPassDescriptor = _view.currentRenderPassDescriptor;
if (_renderPassDescriptor) { // Valid drawable.
id<MTLRenderCommandEncoder> renderEncoder =
[commandBuffer renderCommandEncoderWithDescriptor:_renderPassDescriptor];
renderEncoder.label = renderEncoderLabel;
// Set context state.
[renderEncoder pushDebugGroup:renderEncoderDebugGroup];
[renderEncoder setRenderPipelineState:_pipelineState];
[renderEncoder setVertexBuffer:_vertexBuffer offset:_offset * sizeof(float) atIndex:0];
[renderEncoder setFragmentTexture:_yTexture atIndex:0];
[renderEncoder setFragmentTexture:_uTexture atIndex:1];
[renderEncoder setFragmentTexture:_vTexture atIndex:2];
[renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip
vertexStart:0
vertexCount:4
instanceCount:1];
[renderEncoder popDebugGroup];
[renderEncoder endEncoding];
[commandBuffer presentDrawable:_view.currentDrawable];
}
[commandBuffer commit];
}
#pragma mark - RTCMTLRenderer
- (void)drawFrame:(RTCVideoFrame *)frame {
if (!frame) {
return;
}
if ([self setupTexturesForFrame:frame]) {
@autoreleasepool {
[self render];
}
}
- (NSString *)shaderSource {
return shaderSource;
}
- (BOOL)setupTexturesForFrame:(nonnull RTCVideoFrame *)frame {
// Luma (y) texture.
[super setupTexturesForFrame:frame];
id<MTLDevice> device = [self currentMetalDevice];
if (!device) {
return NO;
}
// Luma (y) texture.
if (!_descriptor || (_width != frame.width && _height != frame.height)) {
_width = frame.width;
_height = frame.height;
@ -296,7 +105,7 @@ static const NSInteger kMaxInflightBuffers = 1;
height:_height
mipmapped:NO];
_descriptor.usage = MTLTextureUsageShaderRead;
_yTexture = [_device newTextureWithDescriptor:_descriptor];
_yTexture = [device newTextureWithDescriptor:_descriptor];
}
// Chroma (u,v) textures
@ -315,8 +124,8 @@ static const NSInteger kMaxInflightBuffers = 1;
height:_chromaHeight
mipmapped:NO];
_chromaDescriptor.usage = MTLTextureUsageShaderRead;
_uTexture = [_device newTextureWithDescriptor:_chromaDescriptor];
_vTexture = [_device newTextureWithDescriptor:_chromaDescriptor];
_uTexture = [device newTextureWithDescriptor:_chromaDescriptor];
_vTexture = [device newTextureWithDescriptor:_chromaDescriptor];
}
[_uTexture replaceRegion:MTLRegionMake2D(0, 0, _chromaWidth, _chromaHeight)
@ -328,9 +137,13 @@ static const NSInteger kMaxInflightBuffers = 1;
withBytes:frame.dataV
bytesPerRow:frame.strideV];
_offset = offsetForRotation((webrtc::VideoRotation)frame.rotation);
return (_uTexture != nil) && (_yTexture != nil) && (_vTexture != nil);
}
- (void)uploadTexturesToRenderEncoder:(id<MTLRenderCommandEncoder>)renderEncoder {
[renderEncoder setFragmentTexture:_yTexture atIndex:0];
[renderEncoder setFragmentTexture:_uTexture atIndex:1];
[renderEncoder setFragmentTexture:_vTexture atIndex:2];
}
@end

5
webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLNSVideoView.m
View File

@ -17,7 +17,7 @@
#import "RTCMTLI420Renderer.h"
@interface RTCMTLNSVideoView () <MTKViewDelegate>
@interface RTCMTLNSVideoView ()<MTKViewDelegate>
@property(nonatomic) id<RTCMTLRenderer> renderer;
@property(nonatomic, strong) MTKView *metalView;
@property(atomic, strong) RTCVideoFrame *videoFrame;
@ -90,6 +90,9 @@
#pragma mark - MTKViewDelegate methods
- (void)drawInMTKView:(nonnull MTKView *)view {
if (self.videoFrame == nil) {
return;
}
if (view == self.metalView) {
[_renderer drawFrame:self.videoFrame];
}

33
webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLNV12Renderer.h
View File

@ -1,5 +1,5 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
* Copyright 2017 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
@ -9,35 +9,10 @@
*/
#import <Foundation/Foundation.h>
#import <MetalKit/MTKView.h>
#import "WebRTC/RTCVideoFrame.h"
#import "RTCMTLRenderer.h"
/**
* Protocol defining ability to render RTCVideoFrame in Metal enabled views.
*/
@protocol RTCMTLRenderer <NSObject>
/**
* Method to be implemented to perform actual rendering of the provided frame.
*
* @param frame The frame to be rendered.
*/
- (void)drawFrame:(RTCVideoFrame *)frame;
/**
* Sets the provided view as rendering destination if possible.
*
* If not possible method returns NO and callers of the method are responisble for performing
* cleanups.
*/
- (BOOL)addRenderingDestination:(__kindof UIView *)view;
@end
/**
* Implementation of RTCMTLRenderer protocol for rendering native nv12 video frames.
*/
@interface RTCMTLNV12Renderer : NSObject <RTCMTLRenderer>
NS_AVAILABLE(10_11, 9_0)
@interface RTCMTLNV12Renderer : RTCMTLRenderer
@end

217
webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLNV12Renderer.mm
View File

@ -1,5 +1,5 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
* Copyright 2017 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
@ -16,46 +16,10 @@
#import "WebRTC/RTCLogging.h"
#import "WebRTC/RTCVideoFrame.h"
#include "webrtc/api/video/video_rotation.h"
#import "RTCMTLRenderer+Private.h"
#define MTL_STRINGIFY(s) @ #s
// As defined in shaderSource.
static NSString *const vertexFunctionName = @"vertexPassthrough";
static NSString *const fragmentFunctionName = @"fragmentColorConversion";
static NSString *const pipelineDescriptorLabel = @"RTCPipeline";
static NSString *const commandBufferLabel = @"RTCCommandBuffer";
static NSString *const renderEncoderLabel = @"RTCEncoder";
static NSString *const renderEncoderDebugGroup = @"RTCDrawFrame";
static const float cubeVertexData[64] = {
-1.0, -1.0, 0.0, 1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 0.0,
// rotation = 90, offset = 16.
-1.0, -1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 0.0, -1.0, 1.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0,
// rotation = 180, offset = 32.
-1.0, -1.0, 1.0, 0.0, 1.0, -1.0, 0.0, 0.0, -1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 1.0,
// rotation = 270, offset = 48.
-1.0, -1.0, 0.0, 0.0, 1.0, -1.0, 0.0, 1.0, -1.0, 1.0, 1.0, 0.0, 1.0, 1.0, 1.0, 1.0,
};
static inline int offsetForRotation(webrtc::VideoRotation rotation) {
switch (rotation) {
case webrtc::kVideoRotation_0:
return 0;
case webrtc::kVideoRotation_90:
return 16;
case webrtc::kVideoRotation_180:
return 32;
case webrtc::kVideoRotation_270:
return 48;
}
return 0;
}
static NSString *const shaderSource = MTL_STRINGIFY(
using namespace metal; typedef struct {
packed_float2 position;
@ -73,7 +37,6 @@ static NSString *const shaderSource = MTL_STRINGIFY(
device Vertex &v = verticies[vid];
out.position = float4(float2(v.position), 0.0, 1.0);
out.texcoord = v.texcoord;
return out;
}
@ -93,189 +56,35 @@ static NSString *const shaderSource = MTL_STRINGIFY(
return half4(out);
});
// The max number of command buffers in flight (submitted to GPU).
// For now setting it up to 1.
// In future we might use triple buffering method if it improves performance.
static const NSInteger kMaxInflightBuffers = 1;
@implementation RTCMTLNV12Renderer {
__kindof MTKView *_view;
// Controller.
dispatch_semaphore_t _inflight_semaphore;
// Renderer.
id<MTLDevice> _device;
id<MTLCommandQueue> _commandQueue;
id<MTLLibrary> _defaultLibrary;
id<MTLRenderPipelineState> _pipelineState;
// Textures.
CVMetalTextureCacheRef _textureCache;
id<MTLTexture> _yTexture;
id<MTLTexture> _CrCbTexture;
// Buffers.
id<MTLBuffer> _vertexBuffer;
// RTC Frame parameters.
int _offset;
}
- (instancetype)init {
if (self = [super init]) {
// _offset of 0 is equal to rotation of 0.
_offset = 0;
_inflight_semaphore = dispatch_semaphore_create(kMaxInflightBuffers);
}
return self;
}
- (BOOL)addRenderingDestination:(__kindof MTKView *)view {
return [self setupWithView:view];
}
#pragma mark - Private
- (BOOL)setupWithView:(__kindof MTKView *)view {
BOOL success = NO;
if ([self setupMetal]) {
[self setupView:view];
[self loadAssets];
[self setupBuffers];
if ([super addRenderingDestination:view]) {
[self initializeTextureCache];
success = YES;
return YES;
}
return success;
}
#pragma mark - GPU methods
- (BOOL)setupMetal {
// Set the view to use the default device.
_device = MTLCreateSystemDefaultDevice();
if (!_device) {
return NO;
}
// Create a new command queue.
_commandQueue = [_device newCommandQueue];
// Load metal library from source.
NSError *libraryError = nil;
id<MTLLibrary> sourceLibrary =
[_device newLibraryWithSource:shaderSource options:NULL error:&libraryError];
if (libraryError) {
RTCLogError(@"Metal: Library with source failed\n%@", libraryError);
return NO;
}
if (!sourceLibrary) {
RTCLogError(@"Metal: Failed to load library. %@", libraryError);
return NO;
}
_defaultLibrary = sourceLibrary;
return YES;
}
- (void)setupView:(__kindof MTKView *)view {
view.device = _device;
view.preferredFramesPerSecond = 30;
view.autoResizeDrawable = NO;
// We need to keep reference to the view as it's needed down the rendering pipeline.
_view = view;
}
- (void)loadAssets {
id<MTLFunction> vertexFunction = [_defaultLibrary newFunctionWithName:vertexFunctionName];
id<MTLFunction> fragmentFunction =
[_defaultLibrary newFunctionWithName:fragmentFunctionName];
MTLRenderPipelineDescriptor *pipelineDescriptor = [[MTLRenderPipelineDescriptor alloc] init];
pipelineDescriptor.label = pipelineDescriptorLabel;
pipelineDescriptor.vertexFunction = vertexFunction;
pipelineDescriptor.fragmentFunction = fragmentFunction;
pipelineDescriptor.colorAttachments[0].pixelFormat = _view.colorPixelFormat;
pipelineDescriptor.depthAttachmentPixelFormat = MTLPixelFormatInvalid;
NSError *error = nil;
_pipelineState = [_device newRenderPipelineStateWithDescriptor:pipelineDescriptor error:&error];
if (!_pipelineState) {
RTCLogError(@"Metal: Failed to create pipeline state. %@", error);
}
}
- (void)setupBuffers {
_vertexBuffer = [_device newBufferWithBytes:cubeVertexData
length:sizeof(cubeVertexData)
options:MTLResourceOptionCPUCacheModeDefault];
return NO;
}
- (void)initializeTextureCache {
CVReturn status =
CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, _device, nil, &_textureCache);
CVReturn status = CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, [self currentMetalDevice],
nil, &_textureCache);
if (status != kCVReturnSuccess) {
RTCLogError(@"Metal: Failed to initialize metal texture cache. Return status is %d", status);
}
}
- (void)render {
// Wait until the inflight (curently sent to GPU) command buffer
// has completed the GPU work.
dispatch_semaphore_wait(_inflight_semaphore, DISPATCH_TIME_FOREVER);
id<MTLCommandBuffer> commandBuffer = [_commandQueue commandBuffer];
commandBuffer.label = commandBufferLabel;
__block dispatch_semaphore_t block_semaphore = _inflight_semaphore;
[commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> _Nonnull) {
// GPU work completed.
dispatch_semaphore_signal(block_semaphore);
}];
MTLRenderPassDescriptor *renderPassDescriptor = _view.currentRenderPassDescriptor;
if (renderPassDescriptor) { // Valid drawable.
id<MTLRenderCommandEncoder> renderEncoder =
[commandBuffer renderCommandEncoderWithDescriptor:renderPassDescriptor];
renderEncoder.label = renderEncoderLabel;
// Set context state.
[renderEncoder pushDebugGroup:renderEncoderDebugGroup];
[renderEncoder setRenderPipelineState:_pipelineState];
[renderEncoder setVertexBuffer:_vertexBuffer offset:_offset * sizeof(float) atIndex:0];
[renderEncoder setFragmentTexture:_yTexture atIndex:0];
[renderEncoder setFragmentTexture:_CrCbTexture atIndex:1];
[renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip
vertexStart:0
vertexCount:4
instanceCount:1];
[renderEncoder popDebugGroup];
[renderEncoder endEncoding];
[commandBuffer presentDrawable:_view.currentDrawable];
}
// CPU work is completed, GPU work can be started.
[commandBuffer commit];
}
#pragma mark - RTCMTLRenderer
- (void)drawFrame:(RTCVideoFrame *)frame {
@autoreleasepool {
if ([self setupTexturesForFrame:frame])
[self render];
}
- (NSString *)shaderSource {
return shaderSource;
}
- (BOOL)setupTexturesForFrame:(nonnull RTCVideoFrame *)frame {
[super setupTexturesForFrame:frame];
CVPixelBufferRef pixelBuffer = frame.nativeHandle;
id<MTLTexture> lumaTexture = nil;
@ -312,10 +121,14 @@ static const NSInteger kMaxInflightBuffers = 1;
if (lumaTexture != nil && chromaTexture != nil) {
_yTexture = lumaTexture;
_CrCbTexture = chromaTexture;
_offset = offsetForRotation((webrtc::VideoRotation)frame.rotation);
return YES;
}
return NO;
}
- (void)uploadTexturesToRenderEncoder:(id<MTLRenderCommandEncoder>)renderEncoder {
[renderEncoder setFragmentTexture:_yTexture atIndex:0];
[renderEncoder setFragmentTexture:_CrCbTexture atIndex:1];
}
@end

21
webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLRenderer+Private.h Normal file
View File

@ -0,0 +1,21 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#import <Metal/Metal.h>
#import "RTCMTLRenderer.h"
NS_ASSUME_NONNULL_BEGIN
@interface RTCMTLRenderer (Private)
- (nullable id<MTLDevice>)currentMetalDevice;
- (NSString *)shaderSource;
- (BOOL)setupTexturesForFrame:(nonnull RTCVideoFrame *)frame;
- (void)uploadTexturesToRenderEncoder:(id<MTLRenderCommandEncoder>)renderEncoder;
@end
NS_ASSUME_NONNULL_END

55
webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLRenderer.h Normal file
View File

@ -0,0 +1,55 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#import <Foundation/Foundation.h>
#if TARGET_OS_IPHONE
#import <UIKit/UIKit.h>
#else
#import <AppKit/AppKit.h>
#endif
#import "WebRTC/RTCVideoFrame.h"
NS_ASSUME_NONNULL_BEGIN
/**
* Protocol defining ability to render RTCVideoFrame in Metal enabled views.
*/
@protocol RTCMTLRenderer<NSObject>
/**
* Method to be implemented to perform actual rendering of the provided frame.
*
* @param frame The frame to be rendered.
*/
- (void)drawFrame:(RTCVideoFrame *)frame;
/**
* Sets the provided view as rendering destination if possible.
*
* If not possible method returns NO and callers of the method are responisble for performing
* cleanups.
*/
#if TARGET_OS_IOS
- (BOOL)addRenderingDestination:(__kindof UIView *)view;
#else
- (BOOL)addRenderingDestination:(__kindof NSView *)view;
#endif
@end
/**
* Implementation of RTCMTLRenderer protocol for rendering native nv12 video frames.
*/
NS_AVAILABLE(10_11, 9_0)
@interface RTCMTLRenderer : NSObject<RTCMTLRenderer>
@end
NS_ASSUME_NONNULL_END

244
webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLRenderer.mm Normal file
View File

@ -0,0 +1,244 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#import "RTCMTLRenderer+Private.h"
#import <Metal/Metal.h>
#import <MetalKit/MetalKit.h>
#import "WebRTC/RTCLogging.h"
#import "WebRTC/RTCVideoFrame.h"
#include "webrtc/api/video/video_rotation.h"
#include "webrtc/base/checks.h"
// As defined in shaderSource.
static NSString *const vertexFunctionName = @"vertexPassthrough";
static NSString *const fragmentFunctionName = @"fragmentColorConversion";
static NSString *const pipelineDescriptorLabel = @"RTCPipeline";
static NSString *const commandBufferLabel = @"RTCCommandBuffer";
static NSString *const renderEncoderLabel = @"RTCEncoder";
static NSString *const renderEncoderDebugGroup = @"RTCDrawFrame";
static const float cubeVertexData[64] = {
-1.0, -1.0, 0.0, 1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 0.0,
// rotation = 90, offset = 16.
-1.0, -1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 0.0, -1.0, 1.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0,
// rotation = 180, offset = 32.
-1.0, -1.0, 1.0, 0.0, 1.0, -1.0, 0.0, 0.0, -1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 1.0,
// rotation = 270, offset = 48.
-1.0, -1.0, 0.0, 0.0, 1.0, -1.0, 0.0, 1.0, -1.0, 1.0, 1.0, 0.0, 1.0, 1.0, 1.0, 1.0,
};
static inline int offsetForRotation(RTCVideoRotation rotation) {
switch (rotation) {
case RTCVideoRotation_0:
return 0;
case RTCVideoRotation_90:
return 16;
case RTCVideoRotation_180:
return 32;
case RTCVideoRotation_270:
return 48;
}
return 0;
}
// The max number of command buffers in flight (submitted to GPU).
// For now setting it up to 1.
// In future we might use triple buffering method if it improves performance.
static const NSInteger kMaxInflightBuffers = 1;
@implementation RTCMTLRenderer {
__kindof MTKView *_view;
// Controller.
dispatch_semaphore_t _inflight_semaphore;
// Renderer.
id<MTLDevice> _device;
id<MTLCommandQueue> _commandQueue;
id<MTLLibrary> _defaultLibrary;
id<MTLRenderPipelineState> _pipelineState;
// Buffers.
id<MTLBuffer> _vertexBuffer;
// RTC Frame parameters.
int _offset;
}
- (instancetype)init {
if (self = [super init]) {
// _offset of 0 is equal to rotation of 0.
_offset = 0;
_inflight_semaphore = dispatch_semaphore_create(kMaxInflightBuffers);
}
return self;
}
- (BOOL)addRenderingDestination:(__kindof MTKView *)view {
return [self setupWithView:view];
}
#pragma mark - Private
- (BOOL)setupWithView:(__kindof MTKView *)view {
BOOL success = NO;
if ([self setupMetal]) {
[self setupView:view];
[self loadAssets];
[self setupBuffers];
success = YES;
}
return success;
}
#pragma mark - Inheritance
- (id<MTLDevice>)currentMetalDevice {
return _device;
}
- (NSString *)shaderSource {
RTC_NOTREACHED() << "Virtual method not implemented in subclass.";
return nil;
}
- (void)uploadTexturesToRenderEncoder:(id<MTLRenderCommandEncoder>)renderEncoder {
RTC_NOTREACHED() << "Virtual method not implemented in subclass.";
}
- (BOOL)setupTexturesForFrame:(nonnull RTCVideoFrame *)frame {
_offset = offsetForRotation(frame.rotation);
return YES;
}
#pragma mark - GPU methods
- (BOOL)setupMetal {
// Set the view to use the default device.
_device = MTLCreateSystemDefaultDevice();
if (!_device) {
return NO;
}
// Create a new command queue.
_commandQueue = [_device newCommandQueue];
// Load metal library from source.
NSError *libraryError = nil;
id<MTLLibrary> sourceLibrary =
[_device newLibraryWithSource:[self shaderSource] options:NULL error:&libraryError];
if (libraryError) {
RTCLogError(@"Metal: Library with source failed\n%@", libraryError);
return NO;
}
if (!sourceLibrary) {
RTCLogError(@"Metal: Failed to load library. %@", libraryError);
return NO;
}
_defaultLibrary = sourceLibrary;
return YES;
}
- (void)setupView:(__kindof MTKView *)view {
view.device = _device;
view.preferredFramesPerSecond = 30;
view.autoResizeDrawable = NO;
// We need to keep reference to the view as it's needed down the rendering pipeline.
_view = view;
}
- (void)loadAssets {
id<MTLFunction> vertexFunction = [_defaultLibrary newFunctionWithName:vertexFunctionName];
id<MTLFunction> fragmentFunction = [_defaultLibrary newFunctionWithName:fragmentFunctionName];
MTLRenderPipelineDescriptor *pipelineDescriptor = [[MTLRenderPipelineDescriptor alloc] init];
pipelineDescriptor.label = pipelineDescriptorLabel;
pipelineDescriptor.vertexFunction = vertexFunction;
pipelineDescriptor.fragmentFunction = fragmentFunction;
pipelineDescriptor.colorAttachments[0].pixelFormat = _view.colorPixelFormat;
pipelineDescriptor.depthAttachmentPixelFormat = MTLPixelFormatInvalid;
NSError *error = nil;
_pipelineState = [_device newRenderPipelineStateWithDescriptor:pipelineDescriptor error:&error];
if (!_pipelineState) {
RTCLogError(@"Metal: Failed to create pipeline state. %@", error);
}
}
- (void)setupBuffers {
_vertexBuffer = [_device newBufferWithBytes:cubeVertexData
length:sizeof(cubeVertexData)
options:MTLResourceOptionCPUCacheModeDefault];
}
- (void)render {
// Wait until the inflight (curently sent to GPU) command buffer
// has completed the GPU work.
dispatch_semaphore_wait(_inflight_semaphore, DISPATCH_TIME_FOREVER);
id<MTLCommandBuffer> commandBuffer = [_commandQueue commandBuffer];
commandBuffer.label = commandBufferLabel;
__block dispatch_semaphore_t block_semaphore = _inflight_semaphore;
[commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> _Nonnull) {
// GPU work completed.
dispatch_semaphore_signal(block_semaphore);
}];
MTLRenderPassDescriptor *renderPassDescriptor = _view.currentRenderPassDescriptor;
if (renderPassDescriptor) { // Valid drawable.
id<MTLRenderCommandEncoder> renderEncoder =
[commandBuffer renderCommandEncoderWithDescriptor:renderPassDescriptor];
renderEncoder.label = renderEncoderLabel;
// Set context state.
[renderEncoder pushDebugGroup:renderEncoderDebugGroup];
[renderEncoder setRenderPipelineState:_pipelineState];
[renderEncoder setVertexBuffer:_vertexBuffer offset:_offset * sizeof(float) atIndex:0];
[self uploadTexturesToRenderEncoder:renderEncoder];
[renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip
vertexStart:0
vertexCount:4
instanceCount:1];
[renderEncoder popDebugGroup];
[renderEncoder endEncoding];
[commandBuffer presentDrawable:_view.currentDrawable];
}
// CPU work is completed, GPU work can be started.
[commandBuffer commit];
}
#pragma mark - RTCMTLRenderer
- (void)drawFrame:(RTCVideoFrame *)frame {
@autoreleasepool {
if ([self setupTexturesForFrame:frame]) {
[self render];
}
}
}
@end

62
webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLVideoView.m
View File

@ -16,22 +16,26 @@
#import "WebRTC/RTCLogging.h"
#import "WebRTC/RTCVideoFrame.h"
#import "RTCMTLI420Renderer.h"
#import "RTCMTLNV12Renderer.h"
// To avoid unreconized symbol linker errors, we're taking advantage of the objc runtime.
// Linking errors occur when compiling for architectures that don't support Metal.
#define MTKViewClass NSClassFromString(@"MTKView")
#define RTCMTLNV12RendererClass NSClassFromString(@"RTCMTLNV12Renderer")
#define RTCMTLI420RendererClass NSClassFromString(@"RTCMTLI420Renderer")
@interface RTCMTLVideoView () <MTKViewDelegate>
@property(nonatomic, strong) RTCMTLNV12Renderer *renderer;
@property(nonatomic, strong) RTCMTLI420Renderer *rendererI420;
@property(nonatomic, strong) RTCMTLNV12Renderer *rendererNV12;
@property(nonatomic, strong) MTKView *metalView;
@property(atomic, strong) RTCVideoFrame *videoFrame;
@end
@implementation RTCMTLVideoView
@synthesize renderer = _renderer;
@synthesize rendererI420 = _rendererI420;
@synthesize rendererNV12 = _rendererNV12;
@synthesize metalView = _metalView;
@synthesize videoFrame = _videoFrame;
@ -66,30 +70,19 @@
return view;
}
+ (RTCMTLNV12Renderer *)createMetalRenderer {
RTCMTLNV12Renderer *renderer = [[RTCMTLNV12RendererClass alloc] init];
return renderer;
+ (RTCMTLNV12Renderer *)createNV12Renderer {
return [[RTCMTLNV12RendererClass alloc] init];
}
+ (RTCMTLI420Renderer *)createI420Renderer {
return [[RTCMTLI420RendererClass alloc] init];
}
- (void)configure {
if (![RTCMTLVideoView isMetalAvailable]) {
RTCLog("Metal unavailable");
return;
}
NSAssert([RTCMTLVideoView isMetalAvailable], @"Metal not availiable on this device");
_metalView = [RTCMTLVideoView createMetalView:self.bounds];
_renderer = [RTCMTLVideoView createMetalRenderer];
if ([self configureMetalRenderer]) {
[self configureMetalView];
} else {
_renderer = nil;
RTCLogError("Metal configuration falied.");
}
}
- (BOOL)configureMetalRenderer {
return [_renderer addRenderingDestination:_metalView];
[self configureMetalView];
}
- (void)configureMetalView {
@ -110,7 +103,32 @@
- (void)drawInMTKView:(nonnull MTKView *)view {
NSAssert(view == self.metalView, @"Receiving draw callbacks from foreign instance.");
[self.renderer drawFrame:self.videoFrame];
if (!self.videoFrame) {
return;
}
id<RTCMTLRenderer> renderer = nil;
if (self.videoFrame.nativeHandle) {
if (!self.rendererNV12) {
self.rendererNV12 = [RTCMTLVideoView createNV12Renderer];
if (![self.rendererNV12 addRenderingDestination:self.metalView]) {
self.rendererNV12 = nil;
RTCLogError(@"Failed to create NV12 renderer");
}
}
renderer = self.rendererNV12;
} else {
if (!self.rendererI420) {
self.rendererI420 = [RTCMTLVideoView createI420Renderer];
if (![self.rendererI420 addRenderingDestination:self.metalView]) {
self.rendererI420 = nil;
RTCLogError(@"Failed to create I420 renderer");
}
}
renderer = self.rendererI420;
}
[renderer drawFrame:self.videoFrame];
}
- (void)mtkView:(MTKView *)view drawableSizeWillChange:(CGSize)size {

2
webrtc/sdk/objc/Framework/Headers/WebRTC/RTCMTLNSVideoView.h
View File

@ -12,6 +12,6 @@
#import "WebRTC/RTCVideoRenderer.h"
NS_AVAILABLE_MAC(10.11)
@interface RTCMTLNSVideoView : NSView <RTCVideoRenderer>
@interface RTCMTLNSVideoView : NSView<RTCVideoRenderer>
@end

2
webrtc/sdk/objc/Framework/Headers/WebRTC/RTCMTLVideoView.h
View File

@ -25,6 +25,8 @@ NS_ASSUME_NONNULL_BEGIN
*
* It has id<RTCVideoRenderer> property that renders video frames in the view's
* bounds using Metal.
* NOTE: always check if metal is available on the running device via
* RTC_SUPPORTS_METAL macro before initializing this class.
*/
NS_CLASS_AVAILABLE_IOS(9)

179
webrtc/sdk/objc/Framework/UnitTests/RTCMTLVideoViewTests.mm
View File

@ -18,138 +18,148 @@
// Extension of RTCMTLVideoView for testing purposes.
@interface RTCMTLVideoView (Testing)
@property(nonatomic, strong) id<RTCMTLRenderer> renderer;
@property(nonatomic, strong) UIView* metalView;
@property(atomic, strong) RTCVideoFrame* videoFrame;
+ (BOOL)isMetalAvailable;
+ (UIView*)createMetalView:(CGRect)frame;
+ (id<RTCMTLRenderer>)createMetalRenderer;
+ (id<RTCMTLRenderer>)createNV12Renderer;
+ (id<RTCMTLRenderer>)createI420Renderer;
- (void)drawInMTKView:(id)view;
@end
@interface RTCMTLVideoViewTests : NSObject
@property(nonatomic, strong) id classMock;
@property(nonatomic, strong) id metalViewMock;
@property(nonatomic, strong) id rendererMock;
@property(nonatomic, strong) id rendererNV12Mock;
@property(nonatomic, strong) id rendererI420Mock;
@property(nonatomic, strong) id frameMock;
@end
@implementation RTCMTLVideoViewTests
@synthesize classMock = _classMock;
@synthesize metalViewMock = _metalViewMock;
@synthesize rendererMock = _rendererMock;
@synthesize rendererNV12Mock = _rendererNV12Mock;
@synthesize rendererI420Mock = _rendererI420Mock;
@synthesize frameMock = _frameMock;
- (void)setup {
self.classMock = OCMClassMock([RTCMTLVideoView class]);
self.metalViewMock = OCMClassMock([RTCMTLVideoViewTests class]);
// NOTE: OCMock doesen't provide modern syntax for -ignoringNonObjectArgs.
[[[[self.classMock stub] ignoringNonObjectArgs] andReturn:self.metalViewMock]
createMetalView:CGRectZero];
self.rendererMock = OCMProtocolMock(@protocol(RTCMTLRenderer));
OCMStub([self.classMock createMetalRenderer]).andReturn(self.rendererMock);
}
- (void)tearDown {
[self.classMock stopMocking];
[self.rendererMock stopMocking];
[self.rendererI420Mock stopMocking];
[self.rendererNV12Mock stopMocking];
[self.metalViewMock stopMocking];
[self.frameMock stopMocking];
self.classMock = nil;
self.rendererMock = nil;
self.rendererI420Mock = nil;
self.rendererNV12Mock = nil;
self.metalViewMock = nil;
self.frameMock = nil;
}
- (void)testMetalConfigureNotExecuted {
// when
OCMStub([self.classMock isMetalAvailable]).andReturn(NO);
RTCMTLVideoView *realView = [[RTCMTLVideoView alloc] init];
// then
EXPECT_TRUE(realView.renderer == nil);
EXPECT_TRUE(realView.metalView == nil);
- (id)frameMockWithNativeHandle:(BOOL)hasNativeHandle {
id frameMock = OCMClassMock([RTCVideoFrame class]);
if (hasNativeHandle) {
OCMStub([frameMock nativeHandle]).andReturn((CVPixelBufferRef)[OCMArg anyPointer]);
} else {
OCMStub([frameMock nativeHandle]).andReturn((CVPixelBufferRef) nullptr);
}
return frameMock;
}
- (void)testMetalConfigureExecuted {
// given
OCMStub([self.classMock isMetalAvailable]).andReturn(YES);
OCMStub([self.rendererMock addRenderingDestination:self.metalViewMock])
.andReturn(NO);
- (id)rendererMockWithSuccessfulSetup:(BOOL)sucess {
id rendererMock = OCMProtocolMock(@protocol(RTCMTLRenderer));
OCMStub([rendererMock addRenderingDestination:[OCMArg any]]).andReturn(sucess);
// when
RTCMTLVideoView *realView = [[RTCMTLVideoView alloc] init];
// then
EXPECT_TRUE(realView.renderer == nil);
EXPECT_TRUE(realView.metalView != nil);
return rendererMock;
}
- (void)testMetalDrawCallback {
#pragma mark - Test cases
- (void)testInitAssertsIfMetalUnavailabe {
// given
OCMStub([self.classMock isMetalAvailable]).andReturn(NO);
OCMExpect([self.rendererMock drawFrame:[OCMArg any]]);
RTCMTLVideoView *realView = [[RTCMTLVideoView alloc] init];
realView.metalView = self.metalViewMock;
realView.renderer = self.rendererMock;
// when
[realView drawInMTKView:self.metalViewMock];
BOOL asserts = NO;
@try {
RTCMTLVideoView *realView = [[RTCMTLVideoView alloc] initWithFrame:CGRectZero];
(void)realView;
} @catch (NSException *ex) {
asserts = YES;
}
// then
[self.rendererMock verify];
EXPECT_TRUE(asserts);
}
- (void)testRTCVideoRenderNilFrameCallback {
// given
OCMStub([self.classMock isMetalAvailable]).andReturn(NO);
OCMStub([self.classMock isMetalAvailable]).andReturn(YES);
RTCMTLVideoView *realView = [[RTCMTLVideoView alloc] init];
self.frameMock = OCMClassMock([RTCVideoFrame class]);
[[self.frameMock reject] nativeHandle];
[[self.classMock reject] createNV12Renderer];
[[self.classMock reject] createI420Renderer];
// when
[realView renderFrame:nil];
// then
EXPECT_TRUE(realView.videoFrame == nil);
}
- (void)testRTCVideoRenderFrameCallback {
// given
OCMStub([self.classMock isMetalAvailable]).andReturn(NO);
RTCMTLVideoView *realView = [[RTCMTLVideoView alloc] init];
id frame = OCMClassMock([RTCVideoFrame class]);
realView.metalView = self.metalViewMock;
realView.renderer = self.rendererMock;
OCMExpect([self.rendererMock drawFrame:frame]);
// when
[realView renderFrame:frame];
[realView drawInMTKView:self.metalViewMock];
// then
EXPECT_EQ(realView.videoFrame, frame);
[self.rendererMock verify];
[self.frameMock verify];
[self.classMock verify];
}
- (void)testRTCVideoRenderFrameCallbackI420 {
// given
OCMStub([self.classMock isMetalAvailable]).andReturn(YES);
self.rendererI420Mock = [self rendererMockWithSuccessfulSetup:YES];
self.frameMock = [self frameMockWithNativeHandle:NO];
OCMExpect([self.rendererI420Mock drawFrame:self.frameMock]);
OCMExpect([self.classMock createI420Renderer]).andReturn(self.rendererI420Mock);
[[self.classMock reject] createNV12Renderer];
RTCMTLVideoView *realView = [[RTCMTLVideoView alloc] init];
// when
[realView renderFrame:self.frameMock];
[realView drawInMTKView:self.metalViewMock];
// then
[self.rendererI420Mock verify];
[self.classMock verify];
}
- (void)testRTCVideoRenderFrameCallbackNV12 {
// given
OCMStub([self.classMock isMetalAvailable]).andReturn(YES);
self.rendererNV12Mock = [self rendererMockWithSuccessfulSetup:YES];
self.frameMock = [self frameMockWithNativeHandle:YES];
OCMExpect([self.rendererNV12Mock drawFrame:self.frameMock]);
OCMExpect([self.classMock createNV12Renderer]).andReturn(self.rendererNV12Mock);
[[self.classMock reject] createI420Renderer];
RTCMTLVideoView *realView = [[RTCMTLVideoView alloc] init];
// when
[realView renderFrame:self.frameMock];
[realView drawInMTKView:self.metalViewMock];
// then
[self.rendererNV12Mock verify];
[self.classMock verify];
}
@end
TEST(RTCMTLVideoViewTests, MetalConfigureNotExecuted) {
TEST(RTCMTLVideoViewTests, InitAssertsIfMetalUnavailabe) {
RTCMTLVideoViewTests *test = [[RTCMTLVideoViewTests alloc] init];
[test setup];
[test testMetalConfigureNotExecuted];
[test tearDown];
}
TEST(RTCMTLVideoViewTests, MetalConfigureExecuted) {
RTCMTLVideoViewTests *test = [[RTCMTLVideoViewTests alloc] init];
[test setup];
[test testMetalConfigureExecuted];
[test tearDown];
}
TEST(RTCMTLVideoViewTests, MetalDrawCallback) {
RTCMTLVideoViewTests *test = [[RTCMTLVideoViewTests alloc] init];
[test setup];
[test testMetalDrawCallback];
[test testInitAssertsIfMetalUnavailabe];
[test tearDown];
}
@ -160,9 +170,18 @@ TEST(RTCMTLVideoViewTests, RTCVideoRenderNilFrameCallback) {
[test tearDown];
}
TEST(RTCMTLVideoViewTests, RTCVideoRenderFrameCallback) {
TEST(RTCMTLVideoViewTests, RTCVideoRenderFrameCallbackI420) {
RTCMTLVideoViewTests *test = [[RTCMTLVideoViewTests alloc] init];
[test setup];
[test testRTCVideoRenderFrameCallback];
[test testRTCVideoRenderFrameCallbackI420];
[test tearDown];
}
TEST(RTCMTLVideoViewTests, RTCVideoRenderFrameCallbackNV12) {
RTCMTLVideoViewTests *test = [[RTCMTLVideoViewTests alloc] init];
[test setup];
[test testRTCVideoRenderFrameCallbackNV12];
[test tearDown];
}