diff --git a/webrtc/sdk/BUILD.gn b/webrtc/sdk/BUILD.gn index 4acf3783f5..68d7f5f901 100644 --- a/webrtc/sdk/BUILD.gn +++ b/webrtc/sdk/BUILD.gn @@ -67,22 +67,7 @@ if (is_ios || is_mac) { "objc/Framework/Headers/WebRTC/RTCCameraPreviewView.h", "objc/Framework/Headers/WebRTC/UIDevice+RTCDevice.h", ] - if (current_cpu == "arm64") { - sources += [ - "objc/Framework/Classes/Metal/RTCMTLNV12Renderer.h", - "objc/Framework/Classes/Metal/RTCMTLNV12Renderer.mm", - "objc/Framework/Classes/Metal/RTCMTLVideoView.m", - ] - } - libs = [ "AVFoundation.framework" ] - if (current_cpu == "arm64") { - libs += [ - "CoreVideo.framework", - "Metal.framework", - "MetalKit.framework", - ] - } } if (!build_with_chromium) { sources += [ @@ -306,7 +291,6 @@ if (is_ios || is_mac) { "objc/Framework/Headers/WebRTC/RTCMediaStreamTrack.h", "objc/Framework/Headers/WebRTC/RTCMetrics.h", "objc/Framework/Headers/WebRTC/RTCMetricsSampleInfo.h", - "objc/Framework/Headers/WebRTC/RTCMTLVideoView.h", "objc/Framework/Headers/WebRTC/RTCPeerConnection.h", "objc/Framework/Headers/WebRTC/RTCPeerConnectionFactory.h", "objc/Framework/Headers/WebRTC/RTCRtpCodecParameters.h", diff --git a/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLNV12Renderer.h b/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLNV12Renderer.h deleted file mode 100644 index 0508e78db5..0000000000 --- a/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLNV12Renderer.h +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2017 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#import -#import - -#import "WebRTC/RTCVideoFrame.h" - -/** - * Protocol defining ability to render RTCVideoFrame in Metal enabled views. - */ -@protocol RTCMTLRenderer - -/** - * Method to be implemented to perform actual rendering of the provided frame. - * - * @param frame The frame to be rendered. - */ -- (void)drawFrame:(RTCVideoFrame *)frame; -@end - -/** - * Implementation of RTCMTLRenderer protocol for rendering native nv12 video frames. - */ -@interface RTCMTLNV12Renderer : NSObject - -/** - * Sets the provided view as rendering destination if possible. - * - * If not possible method returns NO and callers of the method are responisble for performing - * cleanups. - */ -- (BOOL)addRenderingDestination:(__kindof MTKView *)view; -@end diff --git a/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLNV12Renderer.mm b/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLNV12Renderer.mm deleted file mode 100644 index 13bd3fb322..0000000000 --- a/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLNV12Renderer.mm +++ /dev/null @@ -1,319 +0,0 @@ -/* - * Copyright 2017 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#import "RTCMTLNV12Renderer.h" - -#import -#import - -#import "WebRTC/RTCLogging.h" -#import "WebRTC/RTCVideoFrame.h" - -#include "webrtc/api/video/video_rotation.h" - -#define MTL_STRINGIFY(s) @ #s - -// As defined in shaderSource. -static NSString *const vertexFunctionName = @"vertexPassthrough"; -static NSString *const fragmentFunctionName = @"fragmentColorConversion"; - -static NSString *const pipelineDescriptorLabel = @"RTCPipeline"; -static NSString *const commandBufferLabel = @"RTCCommandBuffer"; -static NSString *const renderEncoderLabel = @"RTCEncoder"; -static NSString *const renderEncoderDebugGroup = @"RTCDrawFrame"; - -static const float cubeVertexData[64] = { - -1.0, -1.0, 0.0, 1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 0.0, - - // rotation = 90, offset = 16. - -1.0, -1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 0.0, -1.0, 1.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0, - - // rotation = 180, offset = 32. - -1.0, -1.0, 1.0, 0.0, 1.0, -1.0, 0.0, 0.0, -1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 1.0, - - // rotation = 270, offset = 48. - -1.0, -1.0, 0.0, 0.0, 1.0, -1.0, 0.0, 1.0, -1.0, 1.0, 1.0, 0.0, 1.0, 1.0, 1.0, 1.0, -}; - -static inline int offsetForRotation(webrtc::VideoRotation rotation) { - switch (rotation) { - case webrtc::kVideoRotation_0: - return 0; - case webrtc::kVideoRotation_90: - return 16; - case webrtc::kVideoRotation_180: - return 32; - case webrtc::kVideoRotation_270: - return 48; - } - return 0; -} - -static NSString *const shaderSource = MTL_STRINGIFY( - using namespace metal; typedef struct { - packed_float2 position; - packed_float2 texcoord; - } Vertex; - - typedef struct { - float4 position[[position]]; - float2 texcoord; - } Varyings; - - vertex Varyings vertexPassthrough(device Vertex * verticies[[buffer(0)]], - unsigned int vid[[vertex_id]]) { - Varyings out; - device Vertex &v = verticies[vid]; - out.position = float4(float2(v.position), 0.0, 1.0); - out.texcoord = v.texcoord; - - return out; - } - - // Receiving YCrCb textures. - fragment half4 fragmentColorConversion( - Varyings in[[stage_in]], texture2d textureY[[texture(0)]], - texture2d textureCbCr[[texture(1)]]) { - constexpr sampler s(address::clamp_to_edge, filter::linear); - float y; - float2 uv; - y = textureY.sample(s, in.texcoord).r; - uv = textureCbCr.sample(s, in.texcoord).rg - float2(0.5, 0.5); - - // Conversion for YUV to rgb from http://www.fourcc.org/fccyvrgb.php - float4 out = float4(y + 1.403 * uv.y, y - 0.344 * uv.x - 0.714 * uv.y, y + 1.770 * uv.x, 1.0); - - return half4(out); - }); - -// The max number of command buffers in flight (submitted to GPU). -// For now setting it up to 1. -// In future we might use triple buffering method if it improves performance. -static const NSInteger kMaxInflightBuffers = 1; - -@implementation RTCMTLNV12Renderer { - __kindof MTKView *_view; - - // Controller. - dispatch_semaphore_t _inflight_semaphore; - - // Renderer. - id _device; - id _commandQueue; - id _defaultLibrary; - id _pipelineState; - - // Textures. - CVMetalTextureCacheRef _textureCache; - id _yTexture; - id _CrCbTexture; - - // Buffers. - id _vertexBuffer; - - // RTC Frame parameters. - int _offset; -} - -- (instancetype)init { - if (self = [super init]) { - // _offset of 0 is equal to rotation of 0. - _offset = 0; - _inflight_semaphore = dispatch_semaphore_create(kMaxInflightBuffers); - } - - return self; -} - -- (BOOL)addRenderingDestination:(__kindof MTKView *)view { - return [self setupWithView:view]; -} - -#pragma mark - Private - -- (BOOL)setupWithView:(__kindof MTKView *)view { - BOOL success = NO; - if ([self setupMetal]) { - [self setupView:view]; - [self loadAssets]; - [self setupBuffers]; - [self initializeTextureCache]; - success = YES; - } - return success; -} - -#pragma mark - GPU methods - -- (BOOL)setupMetal { - // Set the view to use the default device. - _device = MTLCreateSystemDefaultDevice(); - if (!_device) { - return NO; - } - - // Create a new command queue. - _commandQueue = [_device newCommandQueue]; - - // Load metal library from source. - NSError *libraryError = nil; - - id sourceLibrary = - [_device newLibraryWithSource:shaderSource options:NULL error:&libraryError]; - - if (libraryError) { - RTCLogError(@"Metal: Library with source failed\n%@", libraryError); - return NO; - } - - if (!sourceLibrary) { - RTCLogError(@"Metal: Failed to load library. %@", libraryError); - return NO; - } - _defaultLibrary = sourceLibrary; - - return YES; -} - -- (void)setupView:(__kindof MTKView *)view { - view.device = _device; - - view.preferredFramesPerSecond = 30; - view.autoResizeDrawable = NO; - - // We need to keep reference to the view as it's needed down the rendering pipeline. - _view = view; -} - -- (void)loadAssets { - id vertexFunction = [_defaultLibrary newFunctionWithName:vertexFunctionName]; - id fragmentFunction = - [_defaultLibrary newFunctionWithName:fragmentFunctionName]; - - MTLRenderPipelineDescriptor *pipelineDescriptor = [[MTLRenderPipelineDescriptor alloc] init]; - pipelineDescriptor.label = pipelineDescriptorLabel; - pipelineDescriptor.vertexFunction = vertexFunction; - pipelineDescriptor.fragmentFunction = fragmentFunction; - pipelineDescriptor.colorAttachments[0].pixelFormat = _view.colorPixelFormat; - pipelineDescriptor.depthAttachmentPixelFormat = MTLPixelFormatInvalid; - NSError *error = nil; - _pipelineState = [_device newRenderPipelineStateWithDescriptor:pipelineDescriptor error:&error]; - - if (!_pipelineState) { - RTCLogError(@"Metal: Failed to create pipeline state. %@", error); - } -} - -- (void)setupBuffers { - _vertexBuffer = [_device newBufferWithBytes:cubeVertexData - length:sizeof(cubeVertexData) - options:MTLResourceOptionCPUCacheModeDefault]; -} - -- (void)initializeTextureCache { - CVReturn status = - CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, _device, nil, &_textureCache); - if (status != kCVReturnSuccess) { - RTCLogError(@"Metal: Failed to initialize metal texture cache. Return status is %d", status); - } -} - -- (void)render { - // Wait until the inflight (curently sent to GPU) command buffer - // has completed the GPU work. - dispatch_semaphore_wait(_inflight_semaphore, DISPATCH_TIME_FOREVER); - - id commandBuffer = [_commandQueue commandBuffer]; - commandBuffer.label = commandBufferLabel; - - __block dispatch_semaphore_t block_semaphore = _inflight_semaphore; - [commandBuffer addCompletedHandler:^(id _Nonnull) { - // GPU work completed. - dispatch_semaphore_signal(block_semaphore); - }]; - - MTLRenderPassDescriptor *_renderPassDescriptor = _view.currentRenderPassDescriptor; - if (_renderPassDescriptor) { // Valid drawable. - id renderEncoder = - [commandBuffer renderCommandEncoderWithDescriptor:_renderPassDescriptor]; - renderEncoder.label = renderEncoderLabel; - - // Set context state. - [renderEncoder pushDebugGroup:renderEncoderDebugGroup]; - [renderEncoder setRenderPipelineState:_pipelineState]; - [renderEncoder setVertexBuffer:_vertexBuffer offset:_offset * sizeof(float) atIndex:0]; - [renderEncoder setFragmentTexture:_yTexture atIndex:0]; - [renderEncoder setFragmentTexture:_CrCbTexture atIndex:1]; - - [renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip - vertexStart:0 - vertexCount:4 - instanceCount:1]; - [renderEncoder popDebugGroup]; - [renderEncoder endEncoding]; - - [commandBuffer presentDrawable:_view.currentDrawable]; - } - - // CPU work is completed, GPU work can be started. - [commandBuffer commit]; -} - -#pragma mark - RTCMTLRenderer - -- (void)drawFrame:(RTCVideoFrame *)frame { - [self setupTexturesForFrame:frame]; - @autoreleasepool { - [self render]; - } -} - -- (void)setupTexturesForFrame:(nonnull RTCVideoFrame *)frame { - CVPixelBufferRef pixelBuffer = frame.nativeHandle; - - id lumaTexture = nil; - id chromaTexture = nil; - CVMetalTextureRef outTexture = nullptr; - - // Luma (y) texture. - int lumaWidth = CVPixelBufferGetWidthOfPlane(pixelBuffer, 0); - int lumaHeight = CVPixelBufferGetHeightOfPlane(pixelBuffer, 0); - - int indexPlane = 0; - CVReturn result = CVMetalTextureCacheCreateTextureFromImage( - kCFAllocatorDefault, _textureCache, pixelBuffer, nil, MTLPixelFormatR8Unorm, lumaWidth, - lumaHeight, indexPlane, &outTexture); - - if (result == kCVReturnSuccess) { - lumaTexture = CVMetalTextureGetTexture(outTexture); - } - - // Same as CFRelease except it can be passed NULL without crashing. - CVBufferRelease(outTexture); - outTexture = nullptr; - - // Chroma (CrCb) texture. - indexPlane = 1; - result = CVMetalTextureCacheCreateTextureFromImage( - kCFAllocatorDefault, _textureCache, pixelBuffer, nil, MTLPixelFormatRG8Unorm, lumaWidth / 2, - lumaHeight / 2, indexPlane, &outTexture); - if (result == kCVReturnSuccess) { - chromaTexture = CVMetalTextureGetTexture(outTexture); - } - CVBufferRelease(outTexture); - - if (lumaTexture != nil && chromaTexture != nil) { - _yTexture = lumaTexture; - _CrCbTexture = chromaTexture; - _offset = offsetForRotation((webrtc::VideoRotation)frame.rotation); - } -} - -@end diff --git a/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLVideoView.m b/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLVideoView.m deleted file mode 100644 index 2a18736b25..0000000000 --- a/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLVideoView.m +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright 2017 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#import "WebRTC/RTCMTLVideoView.h" - -#import -#import - -#import "WebRTC/RTCLogging.h" -#import "WebRTC/RTCVideoFrame.h" - -#import "RTCMTLNV12Renderer.h" - -@interface RTCMTLVideoView () -@property(nonatomic, strong) id renderer; -@property(nonatomic, strong) MTKView *metalView; -@property(atomic, strong) RTCVideoFrame *videoFrame; -@end - -@implementation RTCMTLVideoView { - id _renderer; -} - -@synthesize renderer = _renderer; -@synthesize metalView = _metalView; -@synthesize videoFrame = _videoFrame; - -- (instancetype)initWithFrame:(CGRect)frameRect { - self = [super initWithFrame:frameRect]; - if (self) { - [self configure]; - } - return self; -} - -- (instancetype)initWithCoder:(NSCoder *)aCoder { - self = [super initWithCoder:aCoder]; - if (self) { - [self configure]; - } - return self; -} - -#pragma mark - Private - -+ (BOOL)isMetalAvailable { -#if defined(__OBJC__) && COREVIDEO_SUPPORTS_METAL - return YES; -#elif - return NO; -#endif -} - -- (void)configure { - if ([RTCMTLVideoView isMetalAvailable]) { - _metalView = [[MTKView alloc] initWithFrame:self.bounds]; - [self addSubview:_metalView]; - _metalView.delegate = self; - _metalView.contentMode = UIViewContentModeScaleAspectFit; - _metalView.translatesAutoresizingMaskIntoConstraints = NO; - UILayoutGuide *margins = self.layoutMarginsGuide; - [_metalView.topAnchor constraintEqualToAnchor:margins.topAnchor].active = YES; - [_metalView.bottomAnchor constraintEqualToAnchor:margins.bottomAnchor].active = YES; - [_metalView.leftAnchor constraintEqualToAnchor:margins.leftAnchor].active = YES; - [_metalView.rightAnchor constraintEqualToAnchor:margins.rightAnchor].active = YES; - - _renderer = [[RTCMTLNV12Renderer alloc] init]; - if (![(RTCMTLNV12Renderer *)_renderer addRenderingDestination:_metalView]) { - _renderer = nil; - }; - } else { - RTCLogError("Metal configuration falied."); - } -} -#pragma mark - MTKViewDelegate methods - -- (void)drawInMTKView:(nonnull MTKView *)view { - NSAssert(view == self.metalView, @"Receiving draw callbacks from foreign instance."); - [_renderer drawFrame:self.videoFrame]; -} - -- (void)mtkView:(MTKView *)view drawableSizeWillChange:(CGSize)size { -} - -#pragma mark - RTCVideoRenderer - -- (void)setSize:(CGSize)size { - _metalView.drawableSize = size; - [_metalView draw]; -} - -- (void)renderFrame:(nullable RTCVideoFrame *)frame { - if (frame == nil) { - RTCLogInfo(@"Incoming frame is nil. Exiting render callback."); - return; - } - self.videoFrame = frame; -} - -@end diff --git a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCMTLVideoView.h b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCMTLVideoView.h deleted file mode 100644 index 681c457516..0000000000 --- a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCMTLVideoView.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2017 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#import - -#import "WebRTC/RTCVideoRenderer.h" - -NS_ASSUME_NONNULL_BEGIN -RTC_EXPORT - -/** - * RTCMTLVideoView is thin wrapper around MTKView. - * - * It has id property that renders video frames in the view's - * bounds using Metal. - */ -NS_CLASS_AVAILABLE_IOS(9) -@interface RTCMTLVideoView : UIView - -@end -NS_ASSUME_NONNULL_END