From 11dd52bcc69d074f2ea9473fbf95dd42a0cfde1e Mon Sep 17 00:00:00 2001 From: magjed Date: Tue, 12 Jul 2016 01:26:42 -0700 Subject: [PATCH] H264VideoToolboxEncoder: Add support for native handles Support encoding from CVPixelBuffers directly in H264VideoToolboxEncoder. If the frame needs to be scaled, it will fall back to the previous slow path: CVPixelBuffer -> NV12 -> I420 -> (scale) I420 -> NV12 -> CVPixelBuffer. BUG=webrtc:4081 Review-Url: https://codereview.webrtc.org/2140573002 Cr-Commit-Position: refs/heads/master@{#13439} --- .../codecs/h264/h264_video_toolbox_encoder.cc | 84 +++++++++++-------- .../codecs/h264/h264_video_toolbox_encoder.h | 2 + 2 files changed, 53 insertions(+), 33 deletions(-) diff --git a/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.cc b/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.cc index a7bd5e2040..bbf3f0f8f6 100644 --- a/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.cc +++ b/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.cc @@ -250,7 +250,16 @@ H264VideoToolboxEncoder::GetScaledBufferOnEncode( const rtc::scoped_refptr& frame) { rtc::CritScope lock(&quality_scaler_crit_); quality_scaler_.OnEncodeFrame(frame->width(), frame->height()); - return quality_scaler_.GetScaledBuffer(frame); + if (!frame->native_handle()) + return quality_scaler_.GetScaledBuffer(frame); + + // Handle native (CVImageRef) scaling. + const QualityScaler::Resolution res = quality_scaler_.GetScaledResolution(); + if (res.width == frame->width() && res.height == frame->height()) + return frame; + // TODO(magjed): Implement efficient CVImageRef -> CVImageRef scaling instead + // of doing it via I420. + return quality_scaler_.GetScaledBuffer(frame->NativeToI420Buffer()); } int H264VideoToolboxEncoder::Encode( @@ -280,40 +289,45 @@ int H264VideoToolboxEncoder::Encode( return ret; } - // Get a pixel buffer from the pool and copy frame data over. - CVPixelBufferPoolRef pixel_buffer_pool = - VTCompressionSessionGetPixelBufferPool(compression_session_); -#if defined(WEBRTC_IOS) - if (!pixel_buffer_pool) { - // Kind of a hack. On backgrounding, the compression session seems to get - // invalidated, which causes this pool call to fail when the application - // is foregrounded and frames are being sent for encoding again. - // Resetting the session when this happens fixes the issue. - // In addition we request a keyframe so video can recover quickly. - ResetCompressionSession(); - pixel_buffer_pool = + CVPixelBufferRef pixel_buffer = + static_cast(input_image->native_handle()); + if (pixel_buffer) { + CVBufferRetain(pixel_buffer); + } else { + // Get a pixel buffer from the pool and copy frame data over. + CVPixelBufferPoolRef pixel_buffer_pool = VTCompressionSessionGetPixelBufferPool(compression_session_); - is_keyframe_required = true; - } +#if defined(WEBRTC_IOS) + if (!pixel_buffer_pool) { + // Kind of a hack. On backgrounding, the compression session seems to get + // invalidated, which causes this pool call to fail when the application + // is foregrounded and frames are being sent for encoding again. + // Resetting the session when this happens fixes the issue. + // In addition we request a keyframe so video can recover quickly. + ResetCompressionSession(); + pixel_buffer_pool = + VTCompressionSessionGetPixelBufferPool(compression_session_); + is_keyframe_required = true; + } #endif - if (!pixel_buffer_pool) { - LOG(LS_ERROR) << "Failed to get pixel buffer pool."; - return WEBRTC_VIDEO_CODEC_ERROR; - } - CVPixelBufferRef pixel_buffer = nullptr; - CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(nullptr, pixel_buffer_pool, - &pixel_buffer); - if (ret != kCVReturnSuccess) { - LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret; - // We probably want to drop frames here, since failure probably means - // that the pool is empty. - return WEBRTC_VIDEO_CODEC_ERROR; - } - RTC_DCHECK(pixel_buffer); - if (!internal::CopyVideoFrameToPixelBuffer(input_image, pixel_buffer)) { - LOG(LS_ERROR) << "Failed to copy frame data."; - CVBufferRelease(pixel_buffer); - return WEBRTC_VIDEO_CODEC_ERROR; + if (!pixel_buffer_pool) { + LOG(LS_ERROR) << "Failed to get pixel buffer pool."; + return WEBRTC_VIDEO_CODEC_ERROR; + } + CVReturn ret = CVPixelBufferPoolCreatePixelBuffer( + nullptr, pixel_buffer_pool, &pixel_buffer); + if (ret != kCVReturnSuccess) { + LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret; + // We probably want to drop frames here, since failure probably means + // that the pool is empty. + return WEBRTC_VIDEO_CODEC_ERROR; + } + RTC_DCHECK(pixel_buffer); + if (!internal::CopyVideoFrameToPixelBuffer(input_image, pixel_buffer)) { + LOG(LS_ERROR) << "Failed to copy frame data."; + CVBufferRelease(pixel_buffer); + return WEBRTC_VIDEO_CODEC_ERROR; + } } // Check if we need a keyframe. @@ -488,6 +502,10 @@ const char* H264VideoToolboxEncoder::ImplementationName() const { return "VideoToolbox"; } +bool H264VideoToolboxEncoder::SupportsNativeHandle() const { + return true; +} + void H264VideoToolboxEncoder::SetBitrateBps(uint32_t bitrate_bps) { if (encoder_bitrate_bps_ != bitrate_bps) { SetEncoderBitrateBps(bitrate_bps); diff --git a/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.h b/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.h index c8ae495eb3..08cb537a58 100644 --- a/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.h +++ b/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.h @@ -56,6 +56,8 @@ class H264VideoToolboxEncoder : public H264Encoder { const char* ImplementationName() const override; + bool SupportsNativeHandle() const override; + void OnEncodedFrame(OSStatus status, VTEncodeInfoFlags info_flags, CMSampleBufferRef sample_buffer,