diff --git a/webrtc/sdk/BUILD.gn b/webrtc/sdk/BUILD.gn index 78e1002580..d82788c7f6 100644 --- a/webrtc/sdk/BUILD.gn +++ b/webrtc/sdk/BUILD.gn @@ -105,8 +105,11 @@ if (is_ios || is_mac) { sources = [ "objc/Framework/Classes/Video/RTCAVFoundationVideoCapturerInternal.h", "objc/Framework/Classes/Video/RTCAVFoundationVideoCapturerInternal.mm", + "objc/Framework/Classes/Video/RTCCVPixelBuffer.mm", "objc/Framework/Classes/Video/RTCDefaultShader.h", "objc/Framework/Classes/Video/RTCDefaultShader.mm", + "objc/Framework/Classes/Video/RTCI420Buffer+Private.h", + "objc/Framework/Classes/Video/RTCI420Buffer.mm", "objc/Framework/Classes/Video/RTCI420TextureCache.h", "objc/Framework/Classes/Video/RTCI420TextureCache.mm", "objc/Framework/Classes/Video/RTCOpenGLDefines.h", @@ -116,8 +119,11 @@ if (is_ios || is_mac) { "objc/Framework/Classes/Video/avfoundationformatmapper.mm", "objc/Framework/Classes/Video/avfoundationvideocapturer.h", "objc/Framework/Classes/Video/avfoundationvideocapturer.mm", + "objc/Framework/Classes/Video/objc_frame_buffer.h", + "objc/Framework/Classes/Video/objc_frame_buffer.mm", "objc/Framework/Classes/Video/objcvideotracksource.h", "objc/Framework/Classes/Video/objcvideotracksource.mm", + "objc/Framework/Headers/WebRTC/RTCVideoFrameBuffer.h", ] libs = [] if (is_ios) { @@ -142,8 +148,6 @@ if (is_ios || is_mac) { deps = [ ":objc_common", - ":objc_corevideoframebuffer", - ":objc_videotoolbox", "../api:libjingle_peerconnection_api", "../base:rtc_base", "../common_video", @@ -270,7 +274,6 @@ if (is_ios || is_mac) { "objc/Framework/Classes/PeerConnection/RTCSessionDescription.mm", "objc/Framework/Classes/PeerConnection/RTCTracing.mm", "objc/Framework/Classes/PeerConnection/RTCVideoCapturer.m", - "objc/Framework/Classes/PeerConnection/RTCVideoFrame+Private.h", "objc/Framework/Classes/PeerConnection/RTCVideoFrame.mm", "objc/Framework/Classes/PeerConnection/RTCVideoRendererAdapter+Private.h", "objc/Framework/Classes/PeerConnection/RTCVideoRendererAdapter.h", @@ -307,6 +310,7 @@ if (is_ios || is_mac) { "objc/Framework/Headers/WebRTC/RTCTracing.h", "objc/Framework/Headers/WebRTC/RTCVideoCapturer.h", "objc/Framework/Headers/WebRTC/RTCVideoFrame.h", + "objc/Framework/Headers/WebRTC/RTCVideoFrameBuffer.h", "objc/Framework/Headers/WebRTC/RTCVideoRenderer.h", "objc/Framework/Headers/WebRTC/RTCVideoSource.h", "objc/Framework/Headers/WebRTC/RTCVideoTrack.h", @@ -335,10 +339,11 @@ if (is_ios || is_mac) { deps = [ ":objc_common", - ":objc_corevideoframebuffer", ":objc_video", + ":objc_videotoolbox", "../api:video_frame_api", "../base:rtc_base", + "../common_video", "../media:rtc_media_base", "../pc:libjingle_peerconnection", ] @@ -454,6 +459,7 @@ if (is_ios || is_mac) { "objc/Framework/Headers/WebRTC/RTCTracing.h", "objc/Framework/Headers/WebRTC/RTCVideoCapturer.h", "objc/Framework/Headers/WebRTC/RTCVideoFrame.h", + "objc/Framework/Headers/WebRTC/RTCVideoFrameBuffer.h", "objc/Framework/Headers/WebRTC/RTCVideoRenderer.h", "objc/Framework/Headers/WebRTC/RTCVideoSource.h", "objc/Framework/Headers/WebRTC/RTCVideoTrack.h", @@ -545,13 +551,14 @@ if (is_ios || is_mac) { "objc/Framework/Classes/VideoToolbox/nalu_rewriter.h", "objc/Framework/Classes/VideoToolbox/videocodecfactory.h", "objc/Framework/Classes/VideoToolbox/videocodecfactory.mm", + "objc/Framework/Headers/WebRTC/RTCVideoFrameBuffer.h", ] configs += [ "..:common_objc" ] deps = [ ":objc_common", - ":objc_corevideoframebuffer", + ":objc_video", "../base:rtc_base_approved", "../common_video", "../media:rtc_media", diff --git a/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLI420Renderer.mm b/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLI420Renderer.mm index 7ca1d4e766..027db05b39 100644 --- a/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLI420Renderer.mm +++ b/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLI420Renderer.mm @@ -9,6 +9,7 @@ */ #import "RTCMTLI420Renderer.h" +#import "WebRTC/RTCVideoFrameBuffer.h" #import #import @@ -96,6 +97,8 @@ static NSString *const shaderSource = MTL_STRINGIFY( return NO; } + id buffer = [frame.buffer toI420]; + // Luma (y) texture. if (!_descriptor || (_width != frame.width && _height != frame.height)) { _width = frame.width; @@ -111,8 +114,8 @@ static NSString *const shaderSource = MTL_STRINGIFY( // Chroma (u,v) textures [_yTexture replaceRegion:MTLRegionMake2D(0, 0, _width, _height) mipmapLevel:0 - withBytes:frame.dataY - bytesPerRow:frame.strideY]; + withBytes:buffer.dataY + bytesPerRow:buffer.strideY]; if (!_chromaDescriptor || (_chromaWidth != frame.width / 2 && _chromaHeight != frame.height / 2)) { @@ -130,12 +133,12 @@ static NSString *const shaderSource = MTL_STRINGIFY( [_uTexture replaceRegion:MTLRegionMake2D(0, 0, _chromaWidth, _chromaHeight) mipmapLevel:0 - withBytes:frame.dataU - bytesPerRow:frame.strideU]; + withBytes:buffer.dataU + bytesPerRow:buffer.strideU]; [_vTexture replaceRegion:MTLRegionMake2D(0, 0, _chromaWidth, _chromaHeight) mipmapLevel:0 - withBytes:frame.dataV - bytesPerRow:frame.strideV]; + withBytes:buffer.dataV + bytesPerRow:buffer.strideV]; return (_uTexture != nil) && (_yTexture != nil) && (_vTexture != nil); } diff --git a/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLNV12Renderer.mm b/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLNV12Renderer.mm index b5b9a0f59b..70ecf549ab 100644 --- a/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLNV12Renderer.mm +++ b/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLNV12Renderer.mm @@ -15,6 +15,7 @@ #import "WebRTC/RTCLogging.h" #import "WebRTC/RTCVideoFrame.h" +#import "WebRTC/RTCVideoFrameBuffer.h" #import "RTCMTLRenderer+Private.h" @@ -85,7 +86,7 @@ static NSString *const shaderSource = MTL_STRINGIFY( - (BOOL)setupTexturesForFrame:(nonnull RTCVideoFrame *)frame { [super setupTexturesForFrame:frame]; - CVPixelBufferRef pixelBuffer = frame.nativeHandle; + CVPixelBufferRef pixelBuffer = ((RTCCVPixelBuffer *)frame.buffer).pixelBuffer; id lumaTexture = nil; id chromaTexture = nil; diff --git a/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLVideoView.m b/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLVideoView.m index 96778c4095..e36cb3ab4c 100644 --- a/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLVideoView.m +++ b/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLVideoView.m @@ -15,6 +15,7 @@ #import "WebRTC/RTCLogging.h" #import "WebRTC/RTCVideoFrame.h" +#import "WebRTC/RTCVideoFrameBuffer.h" #import "RTCMTLI420Renderer.h" #import "RTCMTLNV12Renderer.h" @@ -108,7 +109,7 @@ } id renderer = nil; - if (self.videoFrame.nativeHandle) { + if ([self.videoFrame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) { if (!self.rendererNV12) { self.rendererNV12 = [RTCMTLVideoView createNV12Renderer]; if (![self.rendererNV12 addRenderingDestination:self.metalView]) { diff --git a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCCameraVideoCapturer.m b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCCameraVideoCapturer.m index cbeddfe627..a2290c2458 100644 --- a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCCameraVideoCapturer.m +++ b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCCameraVideoCapturer.m @@ -12,6 +12,7 @@ #import "WebRTC/RTCCameraVideoCapturer.h" #import "WebRTC/RTCLogging.h" +#import "WebRTC/RTCVideoFrameBuffer.h" #if TARGET_OS_IPHONE #import "WebRTC/UIDevice+RTCDevice.h" @@ -191,11 +192,12 @@ static inline BOOL IsMediaSubTypeSupported(FourCharCode mediaSubType) { return; } + RTCCVPixelBuffer *rtcPixelBuffer = [[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBuffer]; int64_t timeStampNs = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer)) * kNanosecondsPerSecond; - RTCVideoFrame *videoFrame = [[RTCVideoFrame alloc] initWithPixelBuffer:pixelBuffer - rotation:_rotation - timeStampNs:timeStampNs]; + RTCVideoFrame *videoFrame = [[RTCVideoFrame alloc] initWithBuffer:rtcPixelBuffer + rotation:_rotation + timeStampNs:timeStampNs]; [self.delegate capturer:self didCaptureVideoFrame:videoFrame]; } diff --git a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCFileVideoCapturer.m b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCFileVideoCapturer.m index 933207df44..178a958041 100644 --- a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCFileVideoCapturer.m +++ b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCFileVideoCapturer.m @@ -11,6 +11,7 @@ #import "RTCFileVideoCapturer.h" #import "WebRTC/RTCLogging.h" +#import "WebRTC/RTCVideoFrameBuffer.h" @implementation RTCFileVideoCapturer { AVAssetReader *_reader; @@ -133,10 +134,11 @@ return; } + RTCCVPixelBuffer *rtcPixelBuffer = [[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBuffer]; NSTimeInterval timeStampSeconds = CACurrentMediaTime(); int64_t timeStampNs = lroundf(timeStampSeconds * NSEC_PER_SEC); RTCVideoFrame *videoFrame = - [[RTCVideoFrame alloc] initWithPixelBuffer:pixelBuffer rotation:0 timeStampNs:timeStampNs]; + [[RTCVideoFrame alloc] initWithBuffer:rtcPixelBuffer rotation:0 timeStampNs:timeStampNs]; CFRelease(sampleBuffer); dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ diff --git a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoFrame+Private.h b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoFrame+Private.h deleted file mode 100644 index 3b36f5b008..0000000000 --- a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoFrame+Private.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#import "WebRTC/RTCVideoFrame.h" - -#include "webrtc/api/video/video_frame_buffer.h" - -NS_ASSUME_NONNULL_BEGIN - -@interface RTCVideoFrame () - -@property(nonatomic, readonly) rtc::scoped_refptr videoBuffer; - -- (instancetype)initWithVideoBuffer: - (rtc::scoped_refptr)videoBuffer - rotation:(RTCVideoRotation)rotation - timeStampNs:(int64_t)timeStampNs - NS_DESIGNATED_INITIALIZER; - -@end - -NS_ASSUME_NONNULL_END diff --git a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoFrame.mm b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoFrame.mm index 20e8f927e2..ef93fef8cd 100644 --- a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoFrame.mm +++ b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoFrame.mm @@ -8,22 +8,22 @@ * be found in the AUTHORS file in the root of the source tree. */ -#import "RTCVideoFrame+Private.h" - -#include "webrtc/sdk/objc/Framework/Classes/Video/corevideo_frame_buffer.h" +#import "webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrame.h" +#import "webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrameBuffer.h" @implementation RTCVideoFrame { - rtc::scoped_refptr _videoBuffer; RTCVideoRotation _rotation; int64_t _timeStampNs; } +@synthesize buffer = _buffer; + - (int)width { - return _videoBuffer->width(); + return _buffer.width; } - (int)height { - return _videoBuffer->height(); + return _buffer.height; } - (RTCVideoRotation)rotation { @@ -31,27 +31,51 @@ } - (const uint8_t *)dataY { - return _videoBuffer->GetI420()->DataY(); + if ([_buffer conformsToProtocol:@protocol(RTCI420Buffer)]) { + return ((id)_buffer).dataY; + } else { + return nullptr; + } } - (const uint8_t *)dataU { - return _videoBuffer->GetI420()->DataU(); + if ([_buffer conformsToProtocol:@protocol(RTCI420Buffer)]) { + return ((id)_buffer).dataU; + } else { + return nullptr; + } } - (const uint8_t *)dataV { - return _videoBuffer->GetI420()->DataV(); + if ([_buffer conformsToProtocol:@protocol(RTCI420Buffer)]) { + return ((id)_buffer).dataV; + } else { + return nullptr; + } } - (int)strideY { - return _videoBuffer->GetI420()->StrideY(); + if ([_buffer conformsToProtocol:@protocol(RTCI420Buffer)]) { + return ((id)_buffer).strideY; + } else { + return 0; + } } - (int)strideU { - return _videoBuffer->GetI420()->StrideU(); + if ([_buffer conformsToProtocol:@protocol(RTCI420Buffer)]) { + return ((id)_buffer).strideU; + } else { + return 0; + } } - (int)strideV { - return _videoBuffer->GetI420()->StrideV(); + if ([_buffer conformsToProtocol:@protocol(RTCI420Buffer)]) { + return ((id)_buffer).strideV; + } else { + return 0; + } } - (int64_t)timeStampNs { @@ -59,26 +83,25 @@ } - (CVPixelBufferRef)nativeHandle { - return (_videoBuffer->type() == webrtc::VideoFrameBuffer::Type::kNative) ? - static_cast(_videoBuffer.get())->pixel_buffer() : - nil; + if ([_buffer isKindOfClass:[RTCCVPixelBuffer class]]) { + return ((RTCCVPixelBuffer *)_buffer).pixelBuffer; + } else { + return nullptr; + } } - (RTCVideoFrame *)newI420VideoFrame { - return [[RTCVideoFrame alloc] - initWithVideoBuffer:_videoBuffer->ToI420() - rotation:_rotation - timeStampNs:_timeStampNs]; + return [[RTCVideoFrame alloc] initWithBuffer:[_buffer toI420] + rotation:_rotation + timeStampNs:_timeStampNs]; } - (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer rotation:(RTCVideoRotation)rotation timeStampNs:(int64_t)timeStampNs { - rtc::scoped_refptr videoBuffer( - new rtc::RefCountedObject(pixelBuffer)); - return [self initWithVideoBuffer:videoBuffer - rotation:rotation - timeStampNs:timeStampNs]; + return [self initWithBuffer:[[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBuffer] + rotation:rotation + timeStampNs:timeStampNs]; } - (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer @@ -90,33 +113,26 @@ cropY:(int)cropY rotation:(RTCVideoRotation)rotation timeStampNs:(int64_t)timeStampNs { - rtc::scoped_refptr videoBuffer( - new rtc::RefCountedObject( - pixelBuffer, - scaledWidth, scaledHeight, - cropWidth, cropHeight, - cropX, cropY)); - return [self initWithVideoBuffer:videoBuffer - rotation:rotation - timeStampNs:timeStampNs]; + RTCCVPixelBuffer *rtcPixelBuffer = [[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBuffer + adaptedWidth:scaledWidth + adaptedHeight:scaledHeight + cropWidth:cropWidth + cropHeight:cropHeight + cropX:cropX + cropY:cropY]; + return [self initWithBuffer:rtcPixelBuffer rotation:rotation timeStampNs:timeStampNs]; } -#pragma mark - Private - -- (instancetype)initWithVideoBuffer: - (rtc::scoped_refptr)videoBuffer - rotation:(RTCVideoRotation)rotation - timeStampNs:(int64_t)timeStampNs { +- (instancetype)initWithBuffer:(id)buffer + rotation:(RTCVideoRotation)rotation + timeStampNs:(int64_t)timeStampNs { if (self = [super init]) { - _videoBuffer = videoBuffer; + _buffer = buffer; _rotation = rotation; _timeStampNs = timeStampNs; } + return self; } -- (rtc::scoped_refptr)videoBuffer { - return _videoBuffer; -} - @end diff --git a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoRendererAdapter.mm b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoRendererAdapter.mm index 1d5107b1b0..736803b913 100644 --- a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoRendererAdapter.mm +++ b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoRendererAdapter.mm @@ -8,9 +8,11 @@ * be found in the AUTHORS file in the root of the source tree. */ +#import "RTCI420Buffer+Private.h" #import "RTCVideoRendererAdapter+Private.h" - -#import "RTCVideoFrame+Private.h" +#import "WebRTC/RTCVideoFrame.h" +#import "WebRTC/RTCVideoFrameBuffer.h" +#import "objc_frame_buffer.h" #include @@ -25,12 +27,20 @@ class VideoRendererAdapter } void OnFrame(const webrtc::VideoFrame& nativeVideoFrame) override { + rtc::scoped_refptr video_frame_buffer = nativeVideoFrame.video_frame_buffer(); + id rtc_frame_buffer; + if (video_frame_buffer->type() == VideoFrameBuffer::Type::kNative) { + rtc::scoped_refptr objc_frame_buffer( + static_cast(video_frame_buffer.get())); + rtc_frame_buffer = (id)objc_frame_buffer->wrapped_frame_buffer(); + } else { + rtc_frame_buffer = [[RTCI420Buffer alloc] initWithFrameBuffer:video_frame_buffer->ToI420()]; + } RTCVideoFrame* videoFrame = [[RTCVideoFrame alloc] - initWithVideoBuffer:nativeVideoFrame.video_frame_buffer() - rotation:static_cast( - nativeVideoFrame.rotation()) - timeStampNs:nativeVideoFrame.timestamp_us() * - rtc::kNumNanosecsPerMicrosec]; + initWithBuffer:rtc_frame_buffer + rotation:static_cast(nativeVideoFrame.rotation()) + timeStampNs:nativeVideoFrame.timestamp_us() * rtc::kNumNanosecsPerMicrosec]; + CGSize current_size = (videoFrame.rotation % 180 == 0) ? CGSizeMake(videoFrame.width, videoFrame.height) : CGSizeMake(videoFrame.height, videoFrame.width); diff --git a/webrtc/sdk/objc/Framework/Classes/UI/RTCEAGLVideoView.m b/webrtc/sdk/objc/Framework/Classes/UI/RTCEAGLVideoView.m index 851c4b4a9e..89b7dfcc2e 100644 --- a/webrtc/sdk/objc/Framework/Classes/UI/RTCEAGLVideoView.m +++ b/webrtc/sdk/objc/Framework/Classes/UI/RTCEAGLVideoView.m @@ -17,6 +17,7 @@ #import "RTCNV12TextureCache.h" #import "WebRTC/RTCLogging.h" #import "WebRTC/RTCVideoFrame.h" +#import "WebRTC/RTCVideoFrameBuffer.h" // RTCDisplayLinkTimer wraps a CADisplayLink and is set to fire every two screen // refreshes, which should be 30fps. We wrap the display link in order to avoid @@ -220,7 +221,7 @@ } [self ensureGLContext]; glClear(GL_COLOR_BUFFER_BIT); - if (frame.nativeHandle) { + if ([frame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) { if (!_nv12TextureCache) { _nv12TextureCache = [[RTCNV12TextureCache alloc] initWithContext:_glContext]; } diff --git a/webrtc/sdk/objc/Framework/Classes/Video/RTCCVPixelBuffer.mm b/webrtc/sdk/objc/Framework/Classes/Video/RTCCVPixelBuffer.mm new file mode 100644 index 0000000000..a45c212713 --- /dev/null +++ b/webrtc/sdk/objc/Framework/Classes/Video/RTCCVPixelBuffer.mm @@ -0,0 +1,188 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "WebRTC/RTCVideoFrameBuffer.h" + +#include "webrtc/base/checks.h" +#include "webrtc/base/logging.h" +#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" + +@implementation RTCCVPixelBuffer { + int _width; + int _height; + int _bufferWidth; + int _bufferHeight; + int _cropWidth; + int _cropHeight; + int _cropX; + int _cropY; +} + +@synthesize pixelBuffer = _pixelBuffer; + +- (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer { + return [self initWithPixelBuffer:pixelBuffer + adaptedWidth:CVPixelBufferGetWidth(pixelBuffer) + adaptedHeight:CVPixelBufferGetHeight(pixelBuffer) + cropWidth:CVPixelBufferGetWidth(pixelBuffer) + cropHeight:CVPixelBufferGetHeight(pixelBuffer) + cropX:0 + cropY:0]; +} + +- (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer + adaptedWidth:(int)adaptedWidth + adaptedHeight:(int)adaptedHeight + cropWidth:(int)cropWidth + cropHeight:(int)cropHeight + cropX:(int)cropX + cropY:(int)cropY { + if (self = [super init]) { + _width = adaptedWidth; + _height = adaptedHeight; + _pixelBuffer = pixelBuffer; + _bufferWidth = CVPixelBufferGetWidth(_pixelBuffer); + _bufferHeight = CVPixelBufferGetHeight(_pixelBuffer); + _cropWidth = cropWidth; + _cropHeight = cropHeight; + // Can only crop at even pixels. + _cropX = cropX & ~1; + _cropY = cropY & ~1; + CVBufferRetain(_pixelBuffer); + } + + return self; +} + +- (void)dealloc { + CVBufferRelease(_pixelBuffer); +} + +- (int)width { + return _width; +} + +- (int)height { + return _height; +} + +- (BOOL)requiresCropping { + return _cropWidth != _bufferWidth || _cropHeight != _bufferHeight; +} + +- (BOOL)requiresScalingToWidth:(int)width height:(int)height { + return _cropWidth != width || _cropHeight != height; +} + +- (int)bufferSizeForCroppingAndScalingToWidth:(int)width height:(int)height { + int srcChromaWidth = (_cropWidth + 1) / 2; + int srcChromaHeight = (_cropHeight + 1) / 2; + int dstChromaWidth = (width + 1) / 2; + int dstChromaHeight = (height + 1) / 2; + + return srcChromaWidth * srcChromaHeight * 2 + dstChromaWidth * dstChromaHeight * 2; +} + +- (BOOL)cropAndScaleTo:(CVPixelBufferRef)outputPixelBuffer withTempBuffer:(uint8_t*)tmpBuffer { + // Prepare output pointers. + RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(outputPixelBuffer), + kCVPixelFormatType_420YpCbCr8BiPlanarFullRange); + CVReturn cvRet = CVPixelBufferLockBaseAddress(outputPixelBuffer, 0); + if (cvRet != kCVReturnSuccess) { + LOG(LS_ERROR) << "Failed to lock base address: " << cvRet; + return NO; + } + const int dstWidth = CVPixelBufferGetWidth(outputPixelBuffer); + const int dstHeight = CVPixelBufferGetHeight(outputPixelBuffer); + uint8_t* dstY = + reinterpret_cast(CVPixelBufferGetBaseAddressOfPlane(outputPixelBuffer, 0)); + const int dstYStride = CVPixelBufferGetBytesPerRowOfPlane(outputPixelBuffer, 0); + uint8_t* dstUV = + reinterpret_cast(CVPixelBufferGetBaseAddressOfPlane(outputPixelBuffer, 1)); + const int dstUVStride = CVPixelBufferGetBytesPerRowOfPlane(outputPixelBuffer, 1); + + // Prepare source pointers. + const OSType srcPixelFormat = CVPixelBufferGetPixelFormatType(_pixelBuffer); + RTC_DCHECK(srcPixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange || + srcPixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange); + CVPixelBufferLockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly); + const uint8_t* srcY = + static_cast(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 0)); + const int srcYStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 0); + const uint8_t* srcUV = + static_cast(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 1)); + const int srcUVStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 1); + + // Crop just by modifying pointers. + srcY += srcYStride * _cropY + _cropX; + srcUV += srcUVStride * (_cropY / 2) + _cropX; + + webrtc::NV12Scale(tmpBuffer, + srcY, + srcYStride, + srcUV, + srcUVStride, + _cropWidth, + _cropHeight, + dstY, + dstYStride, + dstUV, + dstUVStride, + dstWidth, + dstHeight); + + CVPixelBufferUnlockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly); + CVPixelBufferUnlockBaseAddress(outputPixelBuffer, 0); + + return YES; +} + +- (id)toI420 { + const OSType pixelFormat = CVPixelBufferGetPixelFormatType(_pixelBuffer); + RTC_DCHECK(pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange || + pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange); + + CVPixelBufferLockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly); + const uint8_t* srcY = + static_cast(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 0)); + const int srcYStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 0); + const uint8_t* srcUV = + static_cast(CVPixelBufferGetBaseAddressOfPlane(_pixelBuffer, 1)); + const int srcUVStride = CVPixelBufferGetBytesPerRowOfPlane(_pixelBuffer, 1); + + // Crop just by modifying pointers. + srcY += srcYStride * _cropY + _cropX; + srcUV += srcUVStride * (_cropY / 2) + _cropX; + + // TODO(magjed): Use a frame buffer pool. + webrtc::NV12ToI420Scaler nv12ToI420Scaler; + RTCMutableI420Buffer* i420Buffer = + [[RTCMutableI420Buffer alloc] initWithWidth:[self width] height:[self height]]; + nv12ToI420Scaler.NV12ToI420Scale(srcY, + srcYStride, + srcUV, + srcUVStride, + _cropWidth, + _cropHeight, + i420Buffer.mutableDataY, + i420Buffer.strideY, + i420Buffer.mutableDataU, + i420Buffer.strideU, + i420Buffer.mutableDataV, + i420Buffer.strideV, + i420Buffer.width, + i420Buffer.height); + + CVPixelBufferUnlockBaseAddress(_pixelBuffer, kCVPixelBufferLock_ReadOnly); + + return i420Buffer; +} + +@end diff --git a/webrtc/sdk/objc/Framework/Classes/Video/RTCI420Buffer+Private.h b/webrtc/sdk/objc/Framework/Classes/Video/RTCI420Buffer+Private.h new file mode 100644 index 0000000000..7883c414b5 --- /dev/null +++ b/webrtc/sdk/objc/Framework/Classes/Video/RTCI420Buffer+Private.h @@ -0,0 +1,24 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "WebRTC/RTCVideoFrameBuffer.h" + +#include "webrtc/api/video/i420_buffer.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface RTCI420Buffer () + +/** Initialize an RTCI420Buffer with its backing I420BufferInterface. */ +- (instancetype)initWithFrameBuffer:(rtc::scoped_refptr)i420Buffer; + +@end + +NS_ASSUME_NONNULL_END diff --git a/webrtc/sdk/objc/Framework/Classes/Video/RTCI420Buffer.mm b/webrtc/sdk/objc/Framework/Classes/Video/RTCI420Buffer.mm new file mode 100644 index 0000000000..042fcc06ba --- /dev/null +++ b/webrtc/sdk/objc/Framework/Classes/Video/RTCI420Buffer.mm @@ -0,0 +1,108 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "WebRTC/RTCVideoFrameBuffer.h" + +#include "webrtc/api/video/i420_buffer.h" + +@implementation RTCI420Buffer { + @protected + rtc::scoped_refptr _i420Buffer; +} + +- (instancetype)initWithWidth:(int)width height:(int)height { + if (self = [super init]) { + _i420Buffer = webrtc::I420Buffer::Create(width, height); + } + + return self; +} + +- (instancetype)initWithWidth:(int)width + height:(int)height + strideY:(int)strideY + strideU:(int)strideU + strideV:(int)strideV { + if (self = [super init]) { + _i420Buffer = webrtc::I420Buffer::Create(width, height, strideY, strideU, strideV); + } + + return self; +} + +- (instancetype)initWithFrameBuffer:(rtc::scoped_refptr)i420Buffer { + if (self = [super init]) { + _i420Buffer = i420Buffer; + } + + return self; +} + +- (int)width { + return _i420Buffer->width(); +} + +- (int)height { + return _i420Buffer->height(); +} + +- (int)strideY { + return _i420Buffer->StrideY(); +} + +- (int)strideU { + return _i420Buffer->StrideU(); +} + +- (int)strideV { + return _i420Buffer->StrideV(); +} + +- (int)chromaWidth { + return _i420Buffer->ChromaWidth(); +} + +- (int)chromaHeight { + return _i420Buffer->ChromaHeight(); +} + +- (const uint8_t *)dataY { + return _i420Buffer->DataY(); +} + +- (const uint8_t *)dataU { + return _i420Buffer->DataU(); +} + +- (const uint8_t *)dataV { + return _i420Buffer->DataV(); +} + +- (id)toI420 { + return self; +} + +@end + +@implementation RTCMutableI420Buffer + +- (uint8_t *)mutableDataY { + return static_cast(_i420Buffer.get())->MutableDataY(); +} + +- (uint8_t *)mutableDataU { + return static_cast(_i420Buffer.get())->MutableDataU(); +} + +- (uint8_t *)mutableDataV { + return static_cast(_i420Buffer.get())->MutableDataV(); +} + +@end diff --git a/webrtc/sdk/objc/Framework/Classes/Video/RTCI420TextureCache.mm b/webrtc/sdk/objc/Framework/Classes/Video/RTCI420TextureCache.mm index eeae867bc0..b603130ea7 100644 --- a/webrtc/sdk/objc/Framework/Classes/Video/RTCI420TextureCache.mm +++ b/webrtc/sdk/objc/Framework/Classes/Video/RTCI420TextureCache.mm @@ -9,6 +9,7 @@ */ #import "RTCI420TextureCache.h" +#import "WebRTC/RTCVideoFrameBuffer.h" #if TARGET_OS_IPHONE #import @@ -123,31 +124,32 @@ static const GLsizei kNumTextures = kNumTexturesPerSet * kNumTextureSets; - (void)uploadFrameToTextures:(RTCVideoFrame *)frame { _currentTextureSet = (_currentTextureSet + 1) % kNumTextureSets; - const int chromaWidth = (frame.width + 1) / 2; - const int chromaHeight = (frame.height + 1) / 2; - if (frame.strideY != frame.width || - frame.strideU != chromaWidth || - frame.strideV != chromaWidth) { - _planeBuffer.resize(frame.width * frame.height); + id buffer = [frame.buffer toI420]; + + const int chromaWidth = buffer.chromaWidth; + const int chromaHeight = buffer.chromaHeight; + if (buffer.strideY != frame.width || buffer.strideU != chromaWidth || + buffer.strideV != chromaWidth) { + _planeBuffer.resize(buffer.width * buffer.height); } - [self uploadPlane:frame.dataY + [self uploadPlane:buffer.dataY texture:self.yTexture - width:frame.width - height:frame.height - stride:frame.strideY]; + width:buffer.width + height:buffer.height + stride:buffer.strideY]; - [self uploadPlane:frame.dataU + [self uploadPlane:buffer.dataU texture:self.uTexture width:chromaWidth height:chromaHeight - stride:frame.strideU]; + stride:buffer.strideU]; - [self uploadPlane:frame.dataV + [self uploadPlane:buffer.dataV texture:self.vTexture width:chromaWidth height:chromaHeight - stride:frame.strideV]; + stride:buffer.strideV]; } @end diff --git a/webrtc/sdk/objc/Framework/Classes/Video/RTCNV12TextureCache.m b/webrtc/sdk/objc/Framework/Classes/Video/RTCNV12TextureCache.m index e259cee193..20a6082a78 100644 --- a/webrtc/sdk/objc/Framework/Classes/Video/RTCNV12TextureCache.m +++ b/webrtc/sdk/objc/Framework/Classes/Video/RTCNV12TextureCache.m @@ -11,6 +11,7 @@ #import "RTCNV12TextureCache.h" #import "WebRTC/RTCVideoFrame.h" +#import "WebRTC/RTCVideoFrameBuffer.h" @implementation RTCNV12TextureCache { CVOpenGLESTextureCacheRef _textureCache; @@ -73,8 +74,10 @@ } - (BOOL)uploadFrameToTextures:(RTCVideoFrame *)frame { - CVPixelBufferRef pixelBuffer = frame.nativeHandle; - NSParameterAssert(pixelBuffer); + NSAssert([frame.buffer isKindOfClass:[RTCCVPixelBuffer class]], + @"frame must be CVPixelBuffer backed"); + RTCCVPixelBuffer *rtcPixelBuffer = (RTCCVPixelBuffer *)frame.buffer; + CVPixelBufferRef pixelBuffer = rtcPixelBuffer.pixelBuffer; return [self loadTexture:&_yTextureRef pixelBuffer:pixelBuffer planeIndex:0 diff --git a/webrtc/sdk/objc/Framework/Classes/Video/avfoundationvideocapturer.mm b/webrtc/sdk/objc/Framework/Classes/Video/avfoundationvideocapturer.mm index a060b51c16..c2e20564b6 100644 --- a/webrtc/sdk/objc/Framework/Classes/Video/avfoundationvideocapturer.mm +++ b/webrtc/sdk/objc/Framework/Classes/Video/avfoundationvideocapturer.mm @@ -15,6 +15,7 @@ #import "RTCAVFoundationVideoCapturerInternal.h" #import "RTCDispatcher+Private.h" #import "WebRTC/RTCLogging.h" +#import "WebRTC/RTCVideoFrameBuffer.h" #include "avfoundationformatmapper.h" @@ -23,7 +24,7 @@ #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" #include "webrtc/base/thread.h" -#include "webrtc/sdk/objc/Framework/Classes/Video/corevideo_frame_buffer.h" +#include "webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.h" namespace webrtc { @@ -150,12 +151,15 @@ void AVFoundationVideoCapturer::CaptureSampleBuffer( return; } + RTCCVPixelBuffer* rtcPixelBuffer = [[RTCCVPixelBuffer alloc] initWithPixelBuffer:image_buffer + adaptedWidth:adapted_width + adaptedHeight:adapted_height + cropWidth:crop_width + cropHeight:crop_height + cropX:crop_x + cropY:crop_y]; rtc::scoped_refptr buffer = - new rtc::RefCountedObject( - image_buffer, - adapted_width, adapted_height, - crop_width, crop_height, - crop_x, crop_y); + new rtc::RefCountedObject(rtcPixelBuffer); // Applying rotation is only supported for legacy reasons and performance is // not critical here. diff --git a/webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.h b/webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.h new file mode 100644 index 0000000000..71099e446b --- /dev/null +++ b/webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.h @@ -0,0 +1,44 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_VIDEO_OBJC_FRAME_BUFFER_H_ +#define WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_VIDEO_OBJC_FRAME_BUFFER_H_ + +#import + +#include "webrtc/common_video/include/video_frame_buffer.h" + +@protocol RTCVideoFrameBuffer; + +namespace webrtc { + +class ObjCFrameBuffer : public VideoFrameBuffer { + public: + explicit ObjCFrameBuffer(id); + ~ObjCFrameBuffer() override; + + Type type() const override; + + int width() const override; + int height() const override; + + rtc::scoped_refptr ToI420() override; + + id wrapped_frame_buffer() const; + + private: + id frame_buffer_; + int width_; + int height_; +}; + +} // namespace webrtc + +#endif // WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_VIDEO_OBJC_FRAME_BUFFER_H_ diff --git a/webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.mm b/webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.mm new file mode 100644 index 0000000000..74e9c13477 --- /dev/null +++ b/webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.mm @@ -0,0 +1,78 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.h" + +#import "WebRTC/RTCVideoFrameBuffer.h" + +namespace webrtc { + +namespace { + +/** ObjCFrameBuffer that conforms to I420BufferInterface by wrapping RTCI420Buffer */ +class ObjCI420FrameBuffer : public I420BufferInterface { + public: + explicit ObjCI420FrameBuffer(id frame_buffer) + : frame_buffer_(frame_buffer), width_(frame_buffer.width), height_(frame_buffer.height) {} + ~ObjCI420FrameBuffer() override{}; + + int width() const override { return width_; } + + int height() const override { return height_; } + + const uint8_t* DataY() const override { return frame_buffer_.dataY; } + + const uint8_t* DataU() const override { return frame_buffer_.dataU; } + + const uint8_t* DataV() const override { return frame_buffer_.dataV; } + + int StrideY() const override { return frame_buffer_.strideY; } + + int StrideU() const override { return frame_buffer_.strideU; } + + int StrideV() const override { return frame_buffer_.strideV; } + + private: + id frame_buffer_; + int width_; + int height_; +}; + +} // namespace + +ObjCFrameBuffer::ObjCFrameBuffer(id frame_buffer) + : frame_buffer_(frame_buffer), width_(frame_buffer.width), height_(frame_buffer.height) {} + +ObjCFrameBuffer::~ObjCFrameBuffer() {} + +VideoFrameBuffer::Type ObjCFrameBuffer::type() const { + return Type::kNative; +} + +int ObjCFrameBuffer::width() const { + return width_; +} + +int ObjCFrameBuffer::height() const { + return height_; +} + +rtc::scoped_refptr ObjCFrameBuffer::ToI420() { + rtc::scoped_refptr buffer = + new rtc::RefCountedObject([frame_buffer_ toI420]); + + return buffer; +} + +id ObjCFrameBuffer::wrapped_frame_buffer() const { + return frame_buffer_; +} + +} // namespace webrtc diff --git a/webrtc/sdk/objc/Framework/Classes/Video/objcvideotracksource.mm b/webrtc/sdk/objc/Framework/Classes/Video/objcvideotracksource.mm index 387bb1bd13..4200cbc891 100644 --- a/webrtc/sdk/objc/Framework/Classes/Video/objcvideotracksource.mm +++ b/webrtc/sdk/objc/Framework/Classes/Video/objcvideotracksource.mm @@ -10,10 +10,11 @@ #include "webrtc/sdk/objc/Framework/Classes/Video/objcvideotracksource.h" -#import "RTCVideoFrame+Private.h" +#import "WebRTC/RTCVideoFrame.h" +#import "WebRTC/RTCVideoFrameBuffer.h" #include "webrtc/api/video/i420_buffer.h" -#include "webrtc/sdk/objc/Framework/Classes/Video/corevideo_frame_buffer.h" +#include "webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.h" namespace webrtc { @@ -43,18 +44,24 @@ void ObjcVideoTrackSource::OnCapturedFrame(RTCVideoFrame* frame) { rtc::scoped_refptr buffer; if (adapted_width == frame.width && adapted_height == frame.height) { // No adaption - optimized path. - buffer = frame.videoBuffer; - } else if (frame.nativeHandle) { + buffer = new rtc::RefCountedObject(frame.buffer); + } else if ([frame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) { // Adapted CVPixelBuffer frame. - buffer = new rtc::RefCountedObject( - static_cast(frame.nativeHandle), adapted_width, adapted_height, - crop_width, crop_height, crop_x, crop_y); + RTCCVPixelBuffer *rtcPixelBuffer = (RTCCVPixelBuffer *)frame.buffer; + buffer = new rtc::RefCountedObject([[RTCCVPixelBuffer alloc] + initWithPixelBuffer:rtcPixelBuffer.pixelBuffer + adaptedWidth:adapted_width + adaptedHeight:adapted_height + cropWidth:crop_width + cropHeight:crop_height + cropX:crop_x + cropY:crop_y]); } else { // Adapted I420 frame. // TODO(magjed): Optimize this I420 path. rtc::scoped_refptr i420_buffer = I420Buffer::Create(adapted_width, adapted_height); - i420_buffer->CropAndScaleFrom( - *frame.videoBuffer->ToI420(), crop_x, crop_y, crop_width, crop_height); + buffer = new rtc::RefCountedObject(frame.buffer); + i420_buffer->CropAndScaleFrom(*buffer->ToI420(), crop_x, crop_y, crop_width, crop_height); buffer = i420_buffer; } diff --git a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/decoder.mm b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/decoder.mm index 1698477e82..c039442eac 100644 --- a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/decoder.mm +++ b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/decoder.mm @@ -18,9 +18,11 @@ #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" #include "webrtc/common_video/include/video_frame.h" -#include "webrtc/sdk/objc/Framework/Classes/Video/corevideo_frame_buffer.h" +#include "webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.h" #include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/nalu_rewriter.h" +#import "WebRTC/RTCVideoFrameBuffer.h" + #if defined(WEBRTC_IOS) #import "Common/RTCUIApplicationStatusObserver.h" #endif @@ -64,8 +66,8 @@ void VTDecompressionOutputCallback(void* decoder, return; } // TODO(tkchin): Handle CVO properly. - rtc::scoped_refptr buffer = - new rtc::RefCountedObject(image_buffer); + rtc::scoped_refptr buffer = new rtc::RefCountedObject( + [[RTCCVPixelBuffer alloc] initWithPixelBuffer:image_buffer]); VideoFrame decoded_frame(buffer, decode_params->timestamp, CMTimeGetSeconds(timestamp) * kMsPerSec, kVideoRotation_0); diff --git a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.mm b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.mm index 1d37ebaed6..2bd485fc78 100644 --- a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.mm +++ b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.mm @@ -19,11 +19,12 @@ #import "Common/RTCUIApplicationStatusObserver.h" #import "WebRTC/UIDevice+RTCDevice.h" #endif +#import "WebRTC/RTCVideoFrameBuffer.h" #include "libyuv/convert_from.h" #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" #include "webrtc/common_video/h264/profile_level_id.h" -#include "webrtc/sdk/objc/Framework/Classes/Video/corevideo_frame_buffer.h" +#include "webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.h" #include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/nalu_rewriter.h" #include "webrtc/system_wrappers/include/clock.h" @@ -411,29 +412,49 @@ int H264VideoToolboxEncoder::Encode( } #endif - CVPixelBufferRef pixel_buffer; + CVPixelBufferRef pixel_buffer = nullptr; if (frame.video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative) { - rtc::scoped_refptr core_video_frame_buffer( - static_cast(frame.video_frame_buffer().get())); - if (!core_video_frame_buffer->RequiresCropping()) { - pixel_buffer = core_video_frame_buffer->pixel_buffer(); - // This pixel buffer might have a higher resolution than what the - // compression session is configured to. The compression session can - // handle that and will output encoded frames in the configured - // resolution regardless of the input pixel buffer resolution. - CVBufferRetain(pixel_buffer); - } else { - // Cropping required, we need to crop and scale to a new pixel buffer. - pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool); - if (!pixel_buffer) { - return WEBRTC_VIDEO_CODEC_ERROR; - } - if (!core_video_frame_buffer->CropAndScaleTo(&nv12_scale_buffer_, - pixel_buffer)) { - return WEBRTC_VIDEO_CODEC_ERROR; + // Native frame. + rtc::scoped_refptr objc_frame_buffer( + static_cast(frame.video_frame_buffer().get())); + id wrapped_frame_buffer = + (id)objc_frame_buffer->wrapped_frame_buffer(); + + if ([wrapped_frame_buffer isKindOfClass:[RTCCVPixelBuffer class]]) { + RTCCVPixelBuffer* rtc_pixel_buffer = (RTCCVPixelBuffer*)wrapped_frame_buffer; + if (![rtc_pixel_buffer requiresCropping]) { + // This pixel buffer might have a higher resolution than what the + // compression session is configured to. The compression session can + // handle that and will output encoded frames in the configured + // resolution regardless of the input pixel buffer resolution. + pixel_buffer = rtc_pixel_buffer.pixelBuffer; + CVBufferRetain(pixel_buffer); + } else { + // Cropping required, we need to crop and scale to a new pixel buffer. + pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool); + if (!pixel_buffer) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + int dst_width = CVPixelBufferGetWidth(pixel_buffer); + int dst_height = CVPixelBufferGetHeight(pixel_buffer); + if ([rtc_pixel_buffer requiresScalingToWidth:dst_width height:dst_height]) { + int size = + [rtc_pixel_buffer bufferSizeForCroppingAndScalingToWidth:dst_width height:dst_height]; + nv12_scale_buffer_.resize(size); + } else { + nv12_scale_buffer_.clear(); + } + nv12_scale_buffer_.shrink_to_fit(); + if (![rtc_pixel_buffer cropAndScaleTo:pixel_buffer + withTempBuffer:nv12_scale_buffer_.data()]) { + return WEBRTC_VIDEO_CODEC_ERROR; + } } } - } else { + } + + if (!pixel_buffer) { + // We did not have a native frame, or the ObjCVideoFrame wrapped a non-native frame pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool); if (!pixel_buffer) { return WEBRTC_VIDEO_CODEC_ERROR; diff --git a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrame.h b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrame.h index ddf8a6b628..78eee5f224 100644 --- a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrame.h +++ b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrame.h @@ -22,6 +22,8 @@ typedef NS_ENUM(NSInteger, RTCVideoRotation) { RTCVideoRotation_270 = 270, }; +@protocol RTCVideoFrameBuffer; + // RTCVideoFrame is an ObjectiveC version of webrtc::VideoFrame. RTC_EXPORT @interface RTCVideoFrame : NSObject @@ -36,27 +38,35 @@ RTC_EXPORT * is null. It is always possible to get such a frame by calling * newI420VideoFrame. */ -@property(nonatomic, readonly, nullable) const uint8_t *dataY; -@property(nonatomic, readonly, nullable) const uint8_t *dataU; -@property(nonatomic, readonly, nullable) const uint8_t *dataV; -@property(nonatomic, readonly) int strideY; -@property(nonatomic, readonly) int strideU; -@property(nonatomic, readonly) int strideV; +@property(nonatomic, readonly, nullable) + const uint8_t *dataY DEPRECATED_MSG_ATTRIBUTE("use [buffer toI420]"); +@property(nonatomic, readonly, nullable) + const uint8_t *dataU DEPRECATED_MSG_ATTRIBUTE("use [buffer toI420]"); +@property(nonatomic, readonly, nullable) + const uint8_t *dataV DEPRECATED_MSG_ATTRIBUTE("use [buffer toI420]"); +@property(nonatomic, readonly) int strideY DEPRECATED_MSG_ATTRIBUTE("use [buffer toI420]"); +@property(nonatomic, readonly) int strideU DEPRECATED_MSG_ATTRIBUTE("use [buffer toI420]"); +@property(nonatomic, readonly) int strideV DEPRECATED_MSG_ATTRIBUTE("use [buffer toI420]"); /** Timestamp in nanoseconds. */ @property(nonatomic, readonly) int64_t timeStampNs; /** The native handle should be a pixel buffer on iOS. */ -@property(nonatomic, readonly) CVPixelBufferRef nativeHandle; +@property(nonatomic, readonly) + CVPixelBufferRef nativeHandle DEPRECATED_MSG_ATTRIBUTE("use buffer instead"); + +@property(nonatomic, readonly) id buffer; - (instancetype)init NS_UNAVAILABLE; - (instancetype)new NS_UNAVAILABLE; /** Initialize an RTCVideoFrame from a pixel buffer, rotation, and timestamp. + * Deprecated - initialize with a RTCCVPixelBuffer instead */ - (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer rotation:(RTCVideoRotation)rotation - timeStampNs:(int64_t)timeStampNs; + timeStampNs:(int64_t)timeStampNs + DEPRECATED_MSG_ATTRIBUTE("use initWithBuffer instead"); /** Initialize an RTCVideoFrame from a pixel buffer combined with cropping and * scaling. Cropping will be applied first on the pixel buffer, followed by @@ -70,7 +80,14 @@ RTC_EXPORT cropX:(int)cropX cropY:(int)cropY rotation:(RTCVideoRotation)rotation - timeStampNs:(int64_t)timeStampNs; + timeStampNs:(int64_t)timeStampNs + DEPRECATED_MSG_ATTRIBUTE("use initWithBuffer instead"); + +/** Initialize an RTCVideoFrame from a frame buffer, rotation, and timestamp. + */ +- (instancetype)initWithBuffer:(id)frameBuffer + rotation:(RTCVideoRotation)rotation + timeStampNs:(int64_t)timeStampNs; /** Return a frame that is guaranteed to be I420, i.e. it is possible to access * the YUV data on it. diff --git a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrameBuffer.h b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrameBuffer.h new file mode 100644 index 0000000000..59986b8da3 --- /dev/null +++ b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrameBuffer.h @@ -0,0 +1,99 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +@protocol RTCI420Buffer; + +// RTCVideoFrameBuffer is an ObjectiveC version of webrtc::VideoFrameBuffer. +RTC_EXPORT +@protocol RTCVideoFrameBuffer + +@property(nonatomic, readonly) int width; +@property(nonatomic, readonly) int height; + +- (id)toI420; + +@end + +/** Protocol for RTCVideoFrameBuffers containing YUV planar data. */ +@protocol RTCYUVPlanarBuffer + +@property(nonatomic, readonly) int chromaWidth; +@property(nonatomic, readonly) int chromaHeight; +@property(nonatomic, readonly) const uint8_t *dataY; +@property(nonatomic, readonly) const uint8_t *dataU; +@property(nonatomic, readonly) const uint8_t *dataV; +@property(nonatomic, readonly) int strideY; +@property(nonatomic, readonly) int strideU; +@property(nonatomic, readonly) int strideV; + +- (instancetype)initWithWidth:(int)width height:(int)height; +- (instancetype)initWithWidth:(int)width + height:(int)height + strideY:(int)strideY + strideU:(int)strideU + strideV:(int)strideV; + +@end + +/** Extension of the YUV planar data buffer with mutable data access */ +@protocol RTCMutableYUVPlanarBuffer + +@property(nonatomic, readonly) uint8_t *mutableDataY; +@property(nonatomic, readonly) uint8_t *mutableDataU; +@property(nonatomic, readonly) uint8_t *mutableDataV; + +@end + +/** Protocol for RTCYUVPlanarBuffers containing I420 data */ +@protocol RTCI420Buffer +@end + +/** Extension of the I420 buffer with mutable data access */ +@protocol RTCMutableI420Buffer +@end + +/** RTCVideoFrameBuffer containing a CVPixelBufferRef */ +@interface RTCCVPixelBuffer : NSObject + +@property(nonatomic, readonly) CVPixelBufferRef pixelBuffer; + +- (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer; +- (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer + adaptedWidth:(int)adaptedWidth + adaptedHeight:(int)adaptedHeight + cropWidth:(int)cropWidth + cropHeight:(int)cropHeight + cropX:(int)cropX + cropY:(int)cropY; + +- (BOOL)requiresCropping; +- (BOOL)requiresScalingToWidth:(int)width height:(int)height; +- (int)bufferSizeForCroppingAndScalingToWidth:(int)width height:(int)height; +/** The minimum size of the |tmpBuffer| must be the number of bytes returned from the + * bufferSizeForCroppingAndScalingToWidth:height: method. + */ +- (BOOL)cropAndScaleTo:(CVPixelBufferRef)outputPixelBuffer withTempBuffer:(uint8_t *)tmpBuffer; + +@end + +/** RTCI420Buffer implements the RTCI420Buffer protocol */ +@interface RTCI420Buffer : NSObject +@end + +/** Mutable version of RTCI420Buffer */ +@interface RTCMutableI420Buffer : RTCI420Buffer +@end + +NS_ASSUME_NONNULL_END diff --git a/webrtc/sdk/objc/Framework/UnitTests/RTCMTLVideoViewTests.mm b/webrtc/sdk/objc/Framework/UnitTests/RTCMTLVideoViewTests.mm index 99004ea515..3743326cdc 100644 --- a/webrtc/sdk/objc/Framework/UnitTests/RTCMTLVideoViewTests.mm +++ b/webrtc/sdk/objc/Framework/UnitTests/RTCMTLVideoViewTests.mm @@ -15,6 +15,7 @@ #include #include +#include // Extension of RTCMTLVideoView for testing purposes. @interface RTCMTLVideoView (Testing) @@ -59,12 +60,14 @@ self.frameMock = nil; } -- (id)frameMockWithNativeHandle:(BOOL)hasNativeHandle { +- (id)frameMockWithCVPixelBuffer:(BOOL)hasCVPixelBuffer { id frameMock = OCMClassMock([RTCVideoFrame class]); - if (hasNativeHandle) { - OCMStub([frameMock nativeHandle]).andReturn((CVPixelBufferRef)[OCMArg anyPointer]); + if (hasCVPixelBuffer) { + OCMStub([frameMock buffer]) + .andReturn( + [[RTCCVPixelBuffer alloc] initWithPixelBuffer:(CVPixelBufferRef)[OCMArg anyPointer]]); } else { - OCMStub([frameMock nativeHandle]).andReturn((CVPixelBufferRef) nullptr); + OCMStub([frameMock buffer]).andReturn([[RTCI420Buffer alloc] initWithWidth:200 height:200]); } return frameMock; } @@ -99,7 +102,7 @@ RTCMTLVideoView *realView = [[RTCMTLVideoView alloc] init]; self.frameMock = OCMClassMock([RTCVideoFrame class]); - [[self.frameMock reject] nativeHandle]; + [[self.frameMock reject] buffer]; [[self.classMock reject] createNV12Renderer]; [[self.classMock reject] createI420Renderer]; @@ -116,7 +119,7 @@ // given OCMStub([self.classMock isMetalAvailable]).andReturn(YES); self.rendererI420Mock = [self rendererMockWithSuccessfulSetup:YES]; - self.frameMock = [self frameMockWithNativeHandle:NO]; + self.frameMock = [self frameMockWithCVPixelBuffer:NO]; OCMExpect([self.rendererI420Mock drawFrame:self.frameMock]); OCMExpect([self.classMock createI420Renderer]).andReturn(self.rendererI420Mock); @@ -137,7 +140,7 @@ // given OCMStub([self.classMock isMetalAvailable]).andReturn(YES); self.rendererNV12Mock = [self rendererMockWithSuccessfulSetup:YES]; - self.frameMock = [self frameMockWithNativeHandle:YES]; + self.frameMock = [self frameMockWithCVPixelBuffer:YES]; OCMExpect([self.rendererNV12Mock drawFrame:self.frameMock]); OCMExpect([self.classMock createNV12Renderer]).andReturn(self.rendererNV12Mock);