From a831dc3a7d10a1fbaa258ee6b1ca6cfc7e91c5ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Bostr=C3=B6m?= Date: Mon, 1 Jun 2015 20:06:42 +0200 Subject: [PATCH] Convert native handles to buffers before encoding. Required to permit conversion of NV12 handles on iOS to I420 for VP8 software encoding, which blocks texture-based capture. This change enforces that all texture-based input provides a method for converting native handles to I420 if they are ever used with software encoders that do not understand the native handles. BUG=4081 R=emircan@chromium.org, glaznev@webrtc.org, hbos@webrtc.org, magjed@webrtc.org, mflodman@webrtc.org, stefan@webrtc.org, tkchin@webrtc.org Review URL: https://webrtc-codereview.appspot.com/50909005 Cr-Commit-Position: refs/heads/master@{#9347} --- .../java/jni/androidmediadecoder_jni.cc | 6 +- talk/app/webrtc/java/jni/native_handle_impl.h | 20 +++++++ .../media/webrtc/webrtcvideoframe_unittest.cc | 19 ++++--- webrtc/common_video/i420_buffer_pool.cc | 5 ++ .../common_video/i420_video_frame_unittest.cc | 40 ++----------- .../interface/video_frame_buffer.h | 29 +++++----- webrtc/common_video/video_frame.cc | 24 +++----- webrtc/common_video/video_frame_buffer.cc | 39 ++++++------- .../main/source/generic_encoder.cc | 4 ++ .../main/source/generic_encoder.h | 2 + .../video_coding/main/source/video_sender.cc | 10 +++- webrtc/test/fake_texture_frame.h | 56 +++++++++++++++++++ webrtc/video/video_encoder.cc | 6 ++ webrtc/video/video_encoder_unittest.cc | 20 +++++++ webrtc/video/video_send_stream_tests.cc | 32 ++++------- webrtc/video_encoder.h | 4 +- webrtc/video_engine/vie_capturer_unittest.cc | 22 ++++---- webrtc/video_engine/vie_encoder.cc | 5 -- webrtc/video_frame.h | 11 ++-- 19 files changed, 211 insertions(+), 143 deletions(-) create mode 100644 webrtc/test/fake_texture_frame.h diff --git a/talk/app/webrtc/java/jni/androidmediadecoder_jni.cc b/talk/app/webrtc/java/jni/androidmediadecoder_jni.cc index b92b2de8ff..9952e26d2e 100644 --- a/talk/app/webrtc/java/jni/androidmediadecoder_jni.cc +++ b/talk/app/webrtc/java/jni/androidmediadecoder_jni.cc @@ -654,9 +654,9 @@ bool MediaCodecVideoDecoder::DeliverPendingOutputs( int32_t callback_status = WEBRTC_VIDEO_CODEC_OK; if (use_surface_) { native_handle_.SetTextureObject(surface_texture_, texture_id); - VideoFrame texture_image(&native_handle_, width, height, output_timestamp_, - 0, webrtc::kVideoRotation_0, - rtc::Callback0()); + VideoFrame texture_image(new rtc::RefCountedObject( + &native_handle_, width, height), + output_timestamp_, 0, webrtc::kVideoRotation_0); texture_image.set_ntp_time_ms(output_ntp_time_ms_); callback_status = callback_->Decoded(texture_image); } else { diff --git a/talk/app/webrtc/java/jni/native_handle_impl.h b/talk/app/webrtc/java/jni/native_handle_impl.h index 8c87696118..cdb72ff4d5 100644 --- a/talk/app/webrtc/java/jni/native_handle_impl.h +++ b/talk/app/webrtc/java/jni/native_handle_impl.h @@ -29,6 +29,9 @@ #ifndef TALK_APP_WEBRTC_JAVA_JNI_NATIVE_HANDLE_IMPL_H_ #define TALK_APP_WEBRTC_JAVA_JNI_NATIVE_HANDLE_IMPL_H_ +#include "webrtc/base/checks.h" +#include "webrtc/common_video/interface/video_frame_buffer.h" + namespace webrtc_jni { // Wrapper for texture object. @@ -52,6 +55,23 @@ class NativeHandleImpl { int32_t texture_id_; }; +class JniNativeHandleBuffer : public webrtc::NativeHandleBuffer { + public: + JniNativeHandleBuffer(void* native_handle, int width, int height) + : NativeHandleBuffer(native_handle, width, height) {} + + // TODO(pbos): Override destructor to release native handle, at the moment the + // native handle is not released based on refcount. + + private: + rtc::scoped_refptr NativeToI420Buffer() override { + // TODO(pbos): Implement before using this in the encoder pipeline (or + // remove the CHECK() in VideoCapture). + RTC_NOTREACHED(); + return nullptr; + } +}; + } // namespace webrtc_jni #endif // TALK_APP_WEBRTC_JAVA_JNI_NATIVE_HANDLE_IMPL_H_ diff --git a/talk/media/webrtc/webrtcvideoframe_unittest.cc b/talk/media/webrtc/webrtcvideoframe_unittest.cc index 7386f51e7b..daa8ffa46e 100644 --- a/talk/media/webrtc/webrtcvideoframe_unittest.cc +++ b/talk/media/webrtc/webrtcvideoframe_unittest.cc @@ -29,6 +29,7 @@ #include "talk/media/base/videoframe_unittest.h" #include "talk/media/webrtc/webrtcvideoframe.h" +#include "webrtc/test/fake_texture_frame.h" namespace { @@ -297,10 +298,11 @@ TEST_F(WebRtcVideoFrameTest, InitRotated90DontApplyRotation) { } TEST_F(WebRtcVideoFrameTest, TextureInitialValues) { - void* dummy_handle = reinterpret_cast(0x1); - webrtc::TextureBuffer* buffer = - new rtc::RefCountedObject(dummy_handle, 640, 480, - rtc::Callback0()); + webrtc::test::FakeNativeHandle* dummy_handle = + new webrtc::test::FakeNativeHandle(); + webrtc::NativeHandleBuffer* buffer = + new rtc::RefCountedObject( + dummy_handle, 640, 480); cricket::WebRtcVideoFrame frame(buffer, 100, 200, webrtc::kVideoRotation_0); EXPECT_EQ(dummy_handle, frame.GetNativeHandle()); EXPECT_EQ(640u, frame.GetWidth()); @@ -314,10 +316,11 @@ TEST_F(WebRtcVideoFrameTest, TextureInitialValues) { } TEST_F(WebRtcVideoFrameTest, CopyTextureFrame) { - void* dummy_handle = reinterpret_cast(0x1); - webrtc::TextureBuffer* buffer = - new rtc::RefCountedObject(dummy_handle, 640, 480, - rtc::Callback0()); + webrtc::test::FakeNativeHandle* dummy_handle = + new webrtc::test::FakeNativeHandle(); + webrtc::NativeHandleBuffer* buffer = + new rtc::RefCountedObject( + dummy_handle, 640, 480); cricket::WebRtcVideoFrame frame1(buffer, 100, 200, webrtc::kVideoRotation_0); cricket::VideoFrame* frame2 = frame1.Copy(); EXPECT_EQ(frame1.GetNativeHandle(), frame2->GetNativeHandle()); diff --git a/webrtc/common_video/i420_buffer_pool.cc b/webrtc/common_video/i420_buffer_pool.cc index 35a6c10953..4b8695824c 100644 --- a/webrtc/common_video/i420_buffer_pool.cc +++ b/webrtc/common_video/i420_buffer_pool.cc @@ -40,6 +40,11 @@ class PooledI420Buffer : public webrtc::VideoFrameBuffer { } void* native_handle() const override { return nullptr; } + rtc::scoped_refptr NativeToI420Buffer() override { + RTC_NOTREACHED(); + return nullptr; + } + friend class rtc::RefCountedObject; rtc::scoped_refptr buffer_; }; diff --git a/webrtc/common_video/i420_video_frame_unittest.cc b/webrtc/common_video/i420_video_frame_unittest.cc index eab8789f9a..da3996b9bd 100644 --- a/webrtc/common_video/i420_video_frame_unittest.cc +++ b/webrtc/common_video/i420_video_frame_unittest.cc @@ -16,27 +16,16 @@ #include "testing/gtest/include/gtest/gtest.h" #include "webrtc/base/bind.h" #include "webrtc/base/scoped_ptr.h" +#include "webrtc/test/fake_texture_frame.h" namespace webrtc { -class NativeHandleImpl { - public: - NativeHandleImpl() : no_longer_needed_(false) {} - virtual ~NativeHandleImpl() {} - bool no_longer_needed() const { return no_longer_needed_; } - void SetNoLongerNeeded() { no_longer_needed_ = true; } - - private: - bool no_longer_needed_; -}; - bool EqualPlane(const uint8_t* data1, const uint8_t* data2, int stride, int width, int height); bool EqualFrames(const VideoFrame& frame1, const VideoFrame& frame2); -bool EqualTextureFrames(const VideoFrame& frame1, const VideoFrame& frame2); int ExpectedSize(int plane_stride, int image_height, PlaneType type); TEST(TestVideoFrame, InitialValues) { @@ -254,14 +243,14 @@ TEST(TestVideoFrame, FailToReuseAllocation) { } TEST(TestVideoFrame, TextureInitialValues) { - NativeHandleImpl handle; - VideoFrame frame(&handle, 640, 480, 100, 10, webrtc::kVideoRotation_0, - rtc::Callback0()); + test::FakeNativeHandle* handle = new test::FakeNativeHandle(); + VideoFrame frame = test::CreateFakeNativeHandleFrame( + handle, 640, 480, 100, 10, webrtc::kVideoRotation_0); EXPECT_EQ(640, frame.width()); EXPECT_EQ(480, frame.height()); EXPECT_EQ(100u, frame.timestamp()); EXPECT_EQ(10, frame.render_time_ms()); - EXPECT_EQ(&handle, frame.native_handle()); + EXPECT_EQ(handle, frame.native_handle()); frame.set_timestamp(200); EXPECT_EQ(200u, frame.timestamp()); @@ -269,17 +258,6 @@ TEST(TestVideoFrame, TextureInitialValues) { EXPECT_EQ(20, frame.render_time_ms()); } -TEST(TestVideoFrame, NoLongerNeeded) { - NativeHandleImpl handle; - ASSERT_FALSE(handle.no_longer_needed()); - VideoFrame* frame = - new VideoFrame(&handle, 640, 480, 100, 200, webrtc::kVideoRotation_0, - rtc::Bind(&NativeHandleImpl::SetNoLongerNeeded, &handle)); - EXPECT_FALSE(handle.no_longer_needed()); - delete frame; - EXPECT_TRUE(handle.no_longer_needed()); -} - bool EqualPlane(const uint8_t* data1, const uint8_t* data2, int stride, @@ -315,14 +293,6 @@ bool EqualFrames(const VideoFrame& frame1, const VideoFrame& frame2) { frame1.stride(kVPlane), half_width, half_height); } -bool EqualTextureFrames(const VideoFrame& frame1, const VideoFrame& frame2) { - return ((frame1.native_handle() == frame2.native_handle()) && - (frame1.width() == frame2.width()) && - (frame1.height() == frame2.height()) && - (frame1.timestamp() == frame2.timestamp()) && - (frame1.render_time_ms() == frame2.render_time_ms())); -} - int ExpectedSize(int plane_stride, int image_height, PlaneType type) { if (type == kYPlane) { return (plane_stride * image_height); diff --git a/webrtc/common_video/interface/video_frame_buffer.h b/webrtc/common_video/interface/video_frame_buffer.h index 1aefa6c611..4079019ccc 100644 --- a/webrtc/common_video/interface/video_frame_buffer.h +++ b/webrtc/common_video/interface/video_frame_buffer.h @@ -52,6 +52,10 @@ class VideoFrameBuffer : public rtc::RefCountInterface { // frame is backed by a texture. virtual void* native_handle() const = 0; + // Returns a new memory-backed frame buffer converted from this buffer's + // native handle. + virtual rtc::scoped_refptr NativeToI420Buffer() = 0; + protected: virtual ~VideoFrameBuffer(); }; @@ -68,6 +72,7 @@ class I420Buffer : public VideoFrameBuffer { uint8_t* data(PlaneType type) override; int stride(PlaneType type) const override; void* native_handle() const override; + rtc::scoped_refptr NativeToI420Buffer() override; protected: ~I420Buffer() override; @@ -81,15 +86,14 @@ class I420Buffer : public VideoFrameBuffer { const rtc::scoped_ptr data_; }; -// Texture buffer is a VideoFrameBuffer wrapper around a |native_handle|. -// |native_handle| must be valid for the lifetime of an instance of this object. -// |no_longer_used| can be used to manage the lifetime of |native_handle|. -class TextureBuffer : public VideoFrameBuffer { +// Base class for native-handle buffer is a wrapper around a |native_handle|. +// This is used for convenience as most native-handle implementations can share +// many VideoFrame implementations, but need to implement a few others (such +// as their own destructors or conversion methods back to software I420). +class NativeHandleBuffer : public VideoFrameBuffer { public: - TextureBuffer(void* native_handle, - int width, - int height, - const rtc::Callback0& no_longer_used); + NativeHandleBuffer(void* native_handle, int width, int height); + int width() const override; int height() const override; const uint8_t* data(PlaneType type) const override; @@ -97,15 +101,10 @@ class TextureBuffer : public VideoFrameBuffer { int stride(PlaneType type) const override; void* native_handle() const override; - private: - friend class rtc::RefCountedObject; - ~TextureBuffer() override; - - // |native_handle_| is a raw pointer and not owned by TextureBuffer. + protected: void* native_handle_; const int width_; const int height_; - rtc::Callback0 no_longer_used_cb_; }; class WrappedI420Buffer : public webrtc::VideoFrameBuffer { @@ -130,6 +129,8 @@ class WrappedI420Buffer : public webrtc::VideoFrameBuffer { int stride(PlaneType type) const override; void* native_handle() const override; + rtc::scoped_refptr NativeToI420Buffer() override; + private: friend class rtc::RefCountedObject; ~WrappedI420Buffer() override; diff --git a/webrtc/common_video/video_frame.cc b/webrtc/common_video/video_frame.cc index 9c127602df..208a31b618 100644 --- a/webrtc/common_video/video_frame.cc +++ b/webrtc/common_video/video_frame.cc @@ -36,22 +36,6 @@ VideoFrame::VideoFrame(const rtc::scoped_refptr& buffer, rotation_(rotation) { } -VideoFrame::VideoFrame(void* native_handle, - int width, - int height, - uint32_t timestamp, - int64_t render_time_ms, - VideoRotation rotation, - const rtc::Callback0& no_longer_used) - : VideoFrame(new rtc::RefCountedObject(native_handle, - width, - height, - no_longer_used), - timestamp, - render_time_ms, - rotation) { -} - int VideoFrame::CreateEmptyFrame(int width, int height, int stride_y, @@ -211,4 +195,12 @@ void VideoFrame::set_video_frame_buffer( video_frame_buffer_ = buffer; } +VideoFrame VideoFrame::ConvertNativeToI420Frame() const { + DCHECK(native_handle()); + VideoFrame frame; + frame.ShallowCopy(*this); + frame.set_video_frame_buffer(video_frame_buffer_->NativeToI420Buffer()); + return frame; +} + } // namespace webrtc diff --git a/webrtc/common_video/video_frame_buffer.cc b/webrtc/common_video/video_frame_buffer.cc index 908c97272d..2cf427ce42 100644 --- a/webrtc/common_video/video_frame_buffer.cc +++ b/webrtc/common_video/video_frame_buffer.cc @@ -94,51 +94,47 @@ void* I420Buffer::native_handle() const { return nullptr; } -TextureBuffer::TextureBuffer(void* native_handle, - int width, - int height, - const rtc::Callback0& no_longer_used) - : native_handle_(native_handle), - width_(width), - height_(height), - no_longer_used_cb_(no_longer_used) { +rtc::scoped_refptr I420Buffer::NativeToI420Buffer() { + RTC_NOTREACHED(); + return nullptr; +} + +NativeHandleBuffer::NativeHandleBuffer(void* native_handle, + int width, + int height) + : native_handle_(native_handle), width_(width), height_(height) { DCHECK(native_handle != nullptr); DCHECK_GT(width, 0); DCHECK_GT(height, 0); } -TextureBuffer::~TextureBuffer() { - no_longer_used_cb_(); -} - -int TextureBuffer::width() const { +int NativeHandleBuffer::width() const { return width_; } -int TextureBuffer::height() const { +int NativeHandleBuffer::height() const { return height_; } -const uint8_t* TextureBuffer::data(PlaneType type) const { +const uint8_t* NativeHandleBuffer::data(PlaneType type) const { RTC_NOTREACHED(); // Should not be called. return nullptr; } -uint8_t* TextureBuffer::data(PlaneType type) { +uint8_t* NativeHandleBuffer::data(PlaneType type) { RTC_NOTREACHED(); // Should not be called. return nullptr; } -int TextureBuffer::stride(PlaneType type) const { +int NativeHandleBuffer::stride(PlaneType type) const { RTC_NOTREACHED(); // Should not be called. return 0; } -void* TextureBuffer::native_handle() const { +void* NativeHandleBuffer::native_handle() const { return native_handle_; } - WrappedI420Buffer::WrappedI420Buffer(int desired_width, int desired_height, int width, @@ -220,4 +216,9 @@ void* WrappedI420Buffer::native_handle() const { return nullptr; } +rtc::scoped_refptr WrappedI420Buffer::NativeToI420Buffer() { + RTC_NOTREACHED(); + return nullptr; +} + } // namespace webrtc diff --git a/webrtc/modules/video_coding/main/source/generic_encoder.cc b/webrtc/modules/video_coding/main/source/generic_encoder.cc index 44db07c61f..109150101f 100644 --- a/webrtc/modules/video_coding/main/source/generic_encoder.cc +++ b/webrtc/modules/video_coding/main/source/generic_encoder.cc @@ -200,6 +200,10 @@ void VCMGenericEncoder::OnDroppedFrame() { encoder_->OnDroppedFrame(); } +bool VCMGenericEncoder::SupportsNativeHandle() const { + return encoder_->SupportsNativeHandle(); +} + /*************************** * Callback Implementation ***************************/ diff --git a/webrtc/modules/video_coding/main/source/generic_encoder.h b/webrtc/modules/video_coding/main/source/generic_encoder.h index 78cc598130..a72277214e 100644 --- a/webrtc/modules/video_coding/main/source/generic_encoder.h +++ b/webrtc/modules/video_coding/main/source/generic_encoder.h @@ -138,6 +138,8 @@ public: void OnDroppedFrame(); + bool SupportsNativeHandle() const; + private: VideoEncoder* const encoder_; VideoEncoderRateObserver* const rate_observer_; diff --git a/webrtc/modules/video_coding/main/source/video_sender.cc b/webrtc/modules/video_coding/main/source/video_sender.cc index 5da529f7ed..9b855da16a 100644 --- a/webrtc/modules/video_coding/main/source/video_sender.cc +++ b/webrtc/modules/video_coding/main/source/video_sender.cc @@ -321,8 +321,16 @@ int32_t VideoSender::AddVideoFrame(const VideoFrame& videoFrame, LOG(LS_ERROR) << "Incoming frame doesn't match set resolution. Dropping."; return VCM_PARAMETER_ERROR; } + VideoFrame converted_frame = videoFrame; + if (converted_frame.native_handle() && !_encoder->SupportsNativeHandle()) { + // This module only supports software encoding. + // TODO(pbos): Offload conversion from the encoder thread. + converted_frame = converted_frame.ConvertNativeToI420Frame(); + CHECK(!converted_frame.IsZeroSize()) + << "Frame conversion failed, won't be able to encode frame."; + } int32_t ret = - _encoder->Encode(videoFrame, codecSpecificInfo, _nextFrameTypes); + _encoder->Encode(converted_frame, codecSpecificInfo, _nextFrameTypes); if (ret < 0) { LOG(LS_ERROR) << "Failed to encode frame. Error code: " << ret; return ret; diff --git a/webrtc/test/fake_texture_frame.h b/webrtc/test/fake_texture_frame.h new file mode 100644 index 0000000000..682e7b6bae --- /dev/null +++ b/webrtc/test/fake_texture_frame.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef WEBRTC_TEST_FAKE_TEXTURE_FRAME_H_ +#define WEBRTC_TEST_FAKE_TEXTURE_FRAME_H_ + +#include "webrtc/base/checks.h" +#include "webrtc/common_video/interface/video_frame_buffer.h" +#include "webrtc/video_frame.h" + +namespace webrtc { +namespace test { + +class FakeNativeHandle {}; + +class FakeNativeHandleBuffer : public NativeHandleBuffer { + public: + FakeNativeHandleBuffer(void* native_handle, int width, int height) + : NativeHandleBuffer(native_handle, width, height) {} + + ~FakeNativeHandleBuffer() { + delete reinterpret_cast(native_handle_); + } + + private: + rtc::scoped_refptr NativeToI420Buffer() override { + rtc::scoped_refptr buffer( + new rtc::RefCountedObject(width_, height_)); + int half_height = (height_ + 1) / 2; + int half_width = (width_ + 1) / 2; + memset(buffer->data(kYPlane), 0, height_ * width_); + memset(buffer->data(kUPlane), 0, half_height * half_width); + memset(buffer->data(kVPlane), 0, half_height * half_width); + return buffer; + } +}; + +static VideoFrame CreateFakeNativeHandleFrame(FakeNativeHandle* native_handle, + int width, + int height, + uint32_t timestamp, + int64_t render_time_ms, + VideoRotation rotation) { + return VideoFrame(new rtc::RefCountedObject( + native_handle, width, height), + timestamp, render_time_ms, rotation); +} +} // namespace test +} // namespace webrtc +#endif // WEBRTC_TEST_FAKE_TEXTURE_FRAME_H_ diff --git a/webrtc/video/video_encoder.cc b/webrtc/video/video_encoder.cc index 66918af98f..381b776909 100644 --- a/webrtc/video/video_encoder.cc +++ b/webrtc/video/video_encoder.cc @@ -122,4 +122,10 @@ void VideoEncoderSoftwareFallbackWrapper::OnDroppedFrame() { return encoder_->OnDroppedFrame(); } +bool VideoEncoderSoftwareFallbackWrapper::SupportsNativeHandle() const { + if (fallback_encoder_) + return fallback_encoder_->SupportsNativeHandle(); + return encoder_->SupportsNativeHandle(); +} + } // namespace webrtc diff --git a/webrtc/video/video_encoder_unittest.cc b/webrtc/video/video_encoder_unittest.cc index 99cf643120..be0170cdbe 100644 --- a/webrtc/video/video_encoder_unittest.cc +++ b/webrtc/video/video_encoder_unittest.cc @@ -60,6 +60,11 @@ class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { void OnDroppedFrame() override { ++on_dropped_frame_count_; } + bool SupportsNativeHandle() const override { + ++supports_native_handle_count_; + return false; + } + int init_encode_count_ = 0; int32_t init_encode_return_code_ = WEBRTC_VIDEO_CODEC_OK; int encode_count_ = 0; @@ -68,6 +73,7 @@ class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { int set_channel_parameters_count_ = 0; int set_rates_count_ = 0; int on_dropped_frame_count_ = 0; + mutable int supports_native_handle_count_ = 0; }; class FakeEncodedImageCallback : public EncodedImageCallback { @@ -197,4 +203,18 @@ TEST_F(VideoEncoderSoftwareFallbackWrapperTest, EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release()); } +TEST_F(VideoEncoderSoftwareFallbackWrapperTest, + SupportsNativeHandleForwardedWithoutFallback) { + fallback_wrapper_.SupportsNativeHandle(); + EXPECT_EQ(1, fake_encoder_.supports_native_handle_count_); +} + +TEST_F(VideoEncoderSoftwareFallbackWrapperTest, + SupportsNativeHandleNotForwardedDuringFallback) { + UtilizeFallbackEncoder(); + fallback_wrapper_.SupportsNativeHandle(); + EXPECT_EQ(0, fake_encoder_.supports_native_handle_count_); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release()); +} + } // namespace webrtc diff --git a/webrtc/video/video_send_stream_tests.cc b/webrtc/video/video_send_stream_tests.cc index 7e31ade396..10f5055246 100644 --- a/webrtc/video/video_send_stream_tests.cc +++ b/webrtc/video/video_send_stream_tests.cc @@ -30,6 +30,7 @@ #include "webrtc/system_wrappers/interface/thread_wrapper.h" #include "webrtc/test/call_test.h" #include "webrtc/test/configurable_frame_size_encoder.h" +#include "webrtc/test/fake_texture_frame.h" #include "webrtc/test/null_transport.h" #include "webrtc/test/testsupport/perf_test.h" #include "webrtc/video/send_statistics_proxy.h" @@ -50,16 +51,6 @@ void ExpectEqualFramesVector(const std::vector& frames1, const std::vector& frames2); VideoFrame CreateVideoFrame(int width, int height, uint8_t data); -class FakeNativeHandle { - public: - FakeNativeHandle() {} - ~FakeNativeHandle() {} -}; - -void DeleteNativeHandle(FakeNativeHandle* handle) { - delete handle; -} - class VideoSendStreamTest : public test::CallTest { protected: void TestNackRetransmission(uint32_t retransmit_ssrc, @@ -1084,20 +1075,17 @@ TEST_F(VideoSendStreamTest, CapturesTextureAndVideoFrames) { std::vector input_frames; int width = static_cast(encoder_config_.streams[0].width); int height = static_cast(encoder_config_.streams[0].height); - FakeNativeHandle* handle1 = new FakeNativeHandle(); - FakeNativeHandle* handle2 = new FakeNativeHandle(); - FakeNativeHandle* handle3 = new FakeNativeHandle(); - input_frames.push_back(VideoFrame(handle1, width, height, 1, 1, - kVideoRotation_0, - rtc::Bind(&DeleteNativeHandle, handle1))); - input_frames.push_back(VideoFrame(handle2, width, height, 2, 2, - kVideoRotation_0, - rtc::Bind(&DeleteNativeHandle, handle2))); + test::FakeNativeHandle* handle1 = new test::FakeNativeHandle(); + test::FakeNativeHandle* handle2 = new test::FakeNativeHandle(); + test::FakeNativeHandle* handle3 = new test::FakeNativeHandle(); + input_frames.push_back(test::CreateFakeNativeHandleFrame( + handle1, width, height, 1, 1, kVideoRotation_0)); + input_frames.push_back(test::CreateFakeNativeHandleFrame( + handle2, width, height, 2, 2, kVideoRotation_0)); input_frames.push_back(CreateVideoFrame(width, height, 3)); input_frames.push_back(CreateVideoFrame(width, height, 4)); - input_frames.push_back(VideoFrame(handle3, width, height, 5, 5, - kVideoRotation_0, - rtc::Bind(&DeleteNativeHandle, handle3))); + input_frames.push_back(test::CreateFakeNativeHandleFrame( + handle3, width, height, 5, 5, kVideoRotation_0)); send_stream_->Start(); for (size_t i = 0; i < input_frames.size(); i++) { diff --git a/webrtc/video_encoder.h b/webrtc/video_encoder.h index 5ef5fa64c7..87cbb98a7e 100644 --- a/webrtc/video_encoder.h +++ b/webrtc/video_encoder.h @@ -123,7 +123,8 @@ class VideoEncoder { virtual int32_t CodecConfigParameters(uint8_t* /*buffer*/, int32_t /*size*/) { return -1; } - virtual void OnDroppedFrame() {}; + virtual void OnDroppedFrame() {} + virtual bool SupportsNativeHandle() const { return false; } }; // Class used to wrap external VideoEncoders to provide a fallback option on @@ -149,6 +150,7 @@ class VideoEncoderSoftwareFallbackWrapper : public VideoEncoder { int32_t SetRates(uint32_t bitrate, uint32_t framerate) override; void OnDroppedFrame() override; + bool SupportsNativeHandle() const override; private: const EncoderType encoder_type_; diff --git a/webrtc/video_engine/vie_capturer_unittest.cc b/webrtc/video_engine/vie_capturer_unittest.cc index f144c2de0c..3733b3d68c 100644 --- a/webrtc/video_engine/vie_capturer_unittest.cc +++ b/webrtc/video_engine/vie_capturer_unittest.cc @@ -24,6 +24,7 @@ #include "webrtc/system_wrappers/interface/event_wrapper.h" #include "webrtc/system_wrappers/interface/ref_count.h" #include "webrtc/system_wrappers/interface/scoped_vector.h" +#include "webrtc/test/fake_texture_frame.h" using ::testing::_; using ::testing::Invoke; @@ -159,11 +160,10 @@ TEST_F(ViECapturerTest, TestRtpTimeStampSet) { TEST_F(ViECapturerTest, TestTextureFrames) { const int kNumFrame = 3; for (int i = 0 ; i < kNumFrame; ++i) { - void* dummy_handle = reinterpret_cast(i+1); + test::FakeNativeHandle* dummy_handle = new test::FakeNativeHandle(); // Add one to |i| so that width/height > 0. - input_frames_.push_back(new VideoFrame(dummy_handle, i + 1, i + 1, i + 1, - i + 1, webrtc::kVideoRotation_0, - rtc::Callback0())); + input_frames_.push_back(new VideoFrame(test::CreateFakeNativeHandleFrame( + dummy_handle, i + 1, i + 1, i + 1, i + 1, webrtc::kVideoRotation_0))); AddInputFrame(input_frames_[i]); WaitOutputFrame(); EXPECT_EQ(dummy_handle, output_frames_[i]->native_handle()); @@ -190,10 +190,9 @@ TEST_F(ViECapturerTest, TestI420Frames) { } TEST_F(ViECapturerTest, TestI420FrameAfterTextureFrame) { - void* dummy_handle = &input_frames_; - input_frames_.push_back(new VideoFrame(dummy_handle, 1, 1, 1, 1, - webrtc::kVideoRotation_0, - rtc::Callback0())); + test::FakeNativeHandle* dummy_handle = new test::FakeNativeHandle(); + input_frames_.push_back(new VideoFrame(test::CreateFakeNativeHandleFrame( + dummy_handle, 1, 1, 1, 1, webrtc::kVideoRotation_0))); AddInputFrame(input_frames_[0]); WaitOutputFrame(); EXPECT_EQ(dummy_handle, output_frames_[0]->native_handle()); @@ -210,10 +209,9 @@ TEST_F(ViECapturerTest, TestTextureFrameAfterI420Frame) { AddInputFrame(input_frames_[0]); WaitOutputFrame(); - void* dummy_handle = &input_frames_; - input_frames_.push_back(new VideoFrame(dummy_handle, 1, 1, 2, 2, - webrtc::kVideoRotation_0, - rtc::Callback0())); + test::FakeNativeHandle* dummy_handle = new test::FakeNativeHandle(); + input_frames_.push_back(new VideoFrame(test::CreateFakeNativeHandleFrame( + dummy_handle, 1, 1, 2, 2, webrtc::kVideoRotation_0))); AddInputFrame(input_frames_[1]); WaitOutputFrame(); diff --git a/webrtc/video_engine/vie_encoder.cc b/webrtc/video_engine/vie_encoder.cc index 2787e09a6d..f94d69387b 100644 --- a/webrtc/video_engine/vie_encoder.cc +++ b/webrtc/video_engine/vie_encoder.cc @@ -543,11 +543,6 @@ void ViEEncoder::DeliverFrame(VideoFrame video_frame) { const VideoFrame* output_frame = (decimated_frame != NULL) ? decimated_frame : &video_frame; - if (video_frame.native_handle() != NULL) { - // TODO(wuchengli): add texture support. http://crbug.com/362437 - return; - } - #ifdef VIDEOCODEC_VP8 if (vcm_->SendCodec() == webrtc::kVideoCodecVP8) { webrtc::CodecSpecificInfo codec_specific_info; diff --git a/webrtc/video_frame.h b/webrtc/video_frame.h index ea550aba88..d70a746dae 100644 --- a/webrtc/video_frame.h +++ b/webrtc/video_frame.h @@ -25,13 +25,6 @@ class VideoFrame { uint32_t timestamp, int64_t render_time_ms, VideoRotation rotation); - VideoFrame(void* native_handle, - int width, - int height, - uint32_t timestamp, - int64_t render_time_ms, - VideoRotation rotation, - const rtc::Callback0& no_longer_used); // TODO(pbos): Make all create/copy functions void, they should not be able to // fail (which should be DCHECK/CHECKed instead). @@ -160,6 +153,10 @@ class VideoFrame { void set_video_frame_buffer( const rtc::scoped_refptr& buffer); + // Convert native-handle frame to memory-backed I420 frame. Should not be + // called on a non-native-handle frame. + VideoFrame ConvertNativeToI420Frame() const; + private: // An opaque reference counted handle that stores the pixel data. rtc::scoped_refptr video_frame_buffer_;