diff --git a/api/video_codecs/BUILD.gn b/api/video_codecs/BUILD.gn index a3072eb31b..5475af97e5 100644 --- a/api/video_codecs/BUILD.gn +++ b/api/video_codecs/BUILD.gn @@ -134,6 +134,7 @@ rtc_library("rtc_software_fallback_wrappers") { deps = [ ":video_codecs_api", "..:fec_controller_api", + "../../api/video:video_frame_i420", "../../media:rtc_h264_profile_id", "../../media:rtc_media_base", "../../modules/video_coding:video_codec_interface", diff --git a/api/video_codecs/test/BUILD.gn b/api/video_codecs/test/BUILD.gn index e3172643a7..243b78267f 100644 --- a/api/video_codecs/test/BUILD.gn +++ b/api/video_codecs/test/BUILD.gn @@ -32,6 +32,7 @@ if (rtc_include_tests) { "../../../rtc_base:rtc_base_tests_utils", "../../../test:field_trial", "../../../test:test_support", + "../../../test:video_test_common", "../../video:encoded_image", "../../video:video_bitrate_allocation", "../../video:video_frame", diff --git a/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc b/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc index 1fa62eb4c0..d2c3666d86 100644 --- a/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc +++ b/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc @@ -35,6 +35,7 @@ #include "modules/video_coding/include/video_error_codes.h" #include "modules/video_coding/utility/simulcast_rate_allocator.h" #include "rtc_base/fake_clock.h" +#include "test/fake_texture_frame.h" #include "test/field_trial.h" #include "test/gmock.h" #include "test/gtest.h" @@ -90,17 +91,16 @@ class FakeEncodedImageCallback : public EncodedImageCallback { }; } // namespace -class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { +class VideoEncoderSoftwareFallbackWrapperTestBase : public ::testing::Test { protected: - VideoEncoderSoftwareFallbackWrapperTest() - : VideoEncoderSoftwareFallbackWrapperTest("") {} - explicit VideoEncoderSoftwareFallbackWrapperTest( - const std::string& field_trials) + VideoEncoderSoftwareFallbackWrapperTestBase( + const std::string& field_trials, + std::unique_ptr sw_encoder) : override_field_trials_(field_trials), fake_encoder_(new CountingFakeEncoder()), wrapper_initialized_(false), fallback_wrapper_(CreateVideoEncoderSoftwareFallbackWrapper( - std::unique_ptr(VP8Encoder::Create()), + std::move(sw_encoder), std::unique_ptr(fake_encoder_), false)) {} @@ -120,6 +120,7 @@ class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { int32_t Encode(const VideoFrame& frame, const std::vector* frame_types) override { ++encode_count_; + last_video_frame_ = frame; if (encode_complete_callback_ && encode_return_code_ == WEBRTC_VIDEO_CODEC_OK) { encode_complete_callback_->OnEncodedImage(EncodedImage(), nullptr, @@ -146,7 +147,7 @@ class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { EncoderInfo info; info.scaling_settings = ScalingSettings(kLowThreshold, kHighThreshold); info.supports_native_handle = supports_native_handle_; - info.implementation_name = "fake-encoder"; + info.implementation_name = implementation_name_; return info; } @@ -158,6 +159,8 @@ class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { int release_count_ = 0; mutable int supports_native_handle_count_ = 0; bool supports_native_handle_ = false; + std::string implementation_name_ = "fake-encoder"; + absl::optional last_video_frame_; }; void InitEncode(); @@ -174,6 +177,7 @@ class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { FakeEncodedImageCallback callback_; // |fake_encoder_| is owned and released by |fallback_wrapper_|. CountingFakeEncoder* fake_encoder_; + CountingFakeEncoder* fake_sw_encoder_; bool wrapper_initialized_; std::unique_ptr fallback_wrapper_; VideoCodec codec_ = {}; @@ -181,11 +185,29 @@ class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { std::unique_ptr rate_allocator_; }; -void VideoEncoderSoftwareFallbackWrapperTest::EncodeFrame() { +class VideoEncoderSoftwareFallbackWrapperTest + : public VideoEncoderSoftwareFallbackWrapperTestBase { + protected: + VideoEncoderSoftwareFallbackWrapperTest() + : VideoEncoderSoftwareFallbackWrapperTest(new CountingFakeEncoder()) {} + explicit VideoEncoderSoftwareFallbackWrapperTest( + CountingFakeEncoder* fake_sw_encoder) + : VideoEncoderSoftwareFallbackWrapperTestBase( + "", + std::unique_ptr(fake_sw_encoder)), + fake_sw_encoder_(fake_sw_encoder) { + fake_sw_encoder_->implementation_name_ = "fake_sw_encoder"; + } + + CountingFakeEncoder* fake_sw_encoder_; +}; + +void VideoEncoderSoftwareFallbackWrapperTestBase::EncodeFrame() { EncodeFrame(WEBRTC_VIDEO_CODEC_OK); } -void VideoEncoderSoftwareFallbackWrapperTest::EncodeFrame(int expected_ret) { +void VideoEncoderSoftwareFallbackWrapperTestBase::EncodeFrame( + int expected_ret) { rtc::scoped_refptr buffer = I420Buffer::Create(codec_.width, codec_.height); I420Buffer::SetBlack(buffer); @@ -200,7 +222,7 @@ void VideoEncoderSoftwareFallbackWrapperTest::EncodeFrame(int expected_ret) { EXPECT_EQ(expected_ret, fallback_wrapper_->Encode(*frame_, &types)); } -void VideoEncoderSoftwareFallbackWrapperTest::InitEncode() { +void VideoEncoderSoftwareFallbackWrapperTestBase::InitEncode() { if (!wrapper_initialized_) { fallback_wrapper_->RegisterEncodeCompleteCallback(&callback_); EXPECT_EQ(&callback_, fake_encoder_->encode_complete_callback_); @@ -231,7 +253,7 @@ void VideoEncoderSoftwareFallbackWrapperTest::InitEncode() { wrapper_initialized_ = true; } -void VideoEncoderSoftwareFallbackWrapperTest::UtilizeFallbackEncoder() { +void VideoEncoderSoftwareFallbackWrapperTestBase::UtilizeFallbackEncoder() { if (!wrapper_initialized_) { fallback_wrapper_->RegisterEncodeCompleteCallback(&callback_); EXPECT_EQ(&callback_, fake_encoder_->encode_complete_callback_); @@ -264,7 +286,7 @@ void VideoEncoderSoftwareFallbackWrapperTest::UtilizeFallbackEncoder() { EXPECT_EQ(callback_count + 1, callback_.callback_count_); } -void VideoEncoderSoftwareFallbackWrapperTest::FallbackFromEncodeRequest() { +void VideoEncoderSoftwareFallbackWrapperTestBase::FallbackFromEncodeRequest() { fallback_wrapper_->RegisterEncodeCompleteCallback(&callback_); codec_.codecType = kVideoCodecVP8; codec_.maxFramerate = kFramerate; @@ -402,9 +424,52 @@ TEST_F(VideoEncoderSoftwareFallbackWrapperTest, ReportsImplementationName) { TEST_F(VideoEncoderSoftwareFallbackWrapperTest, ReportsFallbackImplementationName) { UtilizeFallbackEncoder(); - // Hard coded expected value since libvpx is the software implementation name - // for VP8. Change accordingly if the underlying implementation does. - CheckLastEncoderName("libvpx"); + CheckLastEncoderName(fake_sw_encoder_->implementation_name_.c_str()); +} + +TEST_F(VideoEncoderSoftwareFallbackWrapperTest, + OnEncodeFallbackNativeFrameScaledIfFallbackDoesNotSupportNativeFrames) { + fake_encoder_->supports_native_handle_ = true; + fake_sw_encoder_->supports_native_handle_ = false; + InitEncode(); + int width = codec_.width * 2; + int height = codec_.height * 2; + VideoFrame native_frame = test::FakeNativeBuffer::CreateFrame( + width, height, 0, 0, VideoRotation::kVideoRotation_0); + std::vector types(1, VideoFrameType::kVideoFrameKey); + fake_encoder_->encode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + fallback_wrapper_->Encode(native_frame, &types)); + EXPECT_EQ(1, fake_sw_encoder_->encode_count_); + ASSERT_TRUE(fake_sw_encoder_->last_video_frame_.has_value()); + EXPECT_NE(VideoFrameBuffer::Type::kNative, + fake_sw_encoder_->last_video_frame_->video_frame_buffer()->type()); + EXPECT_EQ(codec_.width, fake_sw_encoder_->last_video_frame_->width()); + EXPECT_EQ(codec_.height, fake_sw_encoder_->last_video_frame_->height()); +} + +TEST_F(VideoEncoderSoftwareFallbackWrapperTest, + OnEncodeFallbackNativeFrameForwardedToFallbackIfItSupportsNativeFrames) { + fake_encoder_->supports_native_handle_ = true; + fake_sw_encoder_->supports_native_handle_ = true; + InitEncode(); + int width = codec_.width * 2; + int height = codec_.height * 2; + VideoFrame native_frame = test::FakeNativeBuffer::CreateFrame( + width, height, 0, 0, VideoRotation::kVideoRotation_0); + std::vector types(1, VideoFrameType::kVideoFrameKey); + fake_encoder_->encode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + fallback_wrapper_->Encode(native_frame, &types)); + EXPECT_EQ(1, fake_sw_encoder_->encode_count_); + ASSERT_TRUE(fake_sw_encoder_->last_video_frame_.has_value()); + EXPECT_EQ(VideoFrameBuffer::Type::kNative, + fake_sw_encoder_->last_video_frame_->video_frame_buffer()->type()); + EXPECT_EQ(native_frame.width(), fake_sw_encoder_->last_video_frame_->width()); + EXPECT_EQ(native_frame.height(), + fake_sw_encoder_->last_video_frame_->height()); } namespace { @@ -413,10 +478,11 @@ const int kMinPixelsPerFrame = 1; const char kFieldTrial[] = "WebRTC-VP8-Forced-Fallback-Encoder-v2"; } // namespace -class ForcedFallbackTest : public VideoEncoderSoftwareFallbackWrapperTest { +class ForcedFallbackTest : public VideoEncoderSoftwareFallbackWrapperTestBase { public: explicit ForcedFallbackTest(const std::string& field_trials) - : VideoEncoderSoftwareFallbackWrapperTest(field_trials) {} + : VideoEncoderSoftwareFallbackWrapperTestBase(field_trials, + VP8Encoder::Create()) {} ~ForcedFallbackTest() override {} @@ -668,6 +734,8 @@ TEST(SoftwareFallbackEncoderTest, HwRateControllerTrusted) { EXPECT_TRUE(wrapper->GetEncoderInfo().has_trusted_rate_controller); VideoCodec codec_ = {}; + codec_.width = 100; + codec_.height = 100; wrapper->InitEncode(&codec_, kSettings); // Trigger fallback to software. @@ -711,6 +779,8 @@ TEST(SoftwareFallbackEncoderTest, ReportsHardwareAccelerated) { EXPECT_TRUE(wrapper->GetEncoderInfo().is_hardware_accelerated); VideoCodec codec_ = {}; + codec_.width = 100; + codec_.height = 100; wrapper->InitEncode(&codec_, kSettings); // Trigger fallback to software. @@ -738,6 +808,8 @@ TEST(SoftwareFallbackEncoderTest, ReportsInternalSource) { EXPECT_TRUE(wrapper->GetEncoderInfo().has_internal_source); VideoCodec codec_ = {}; + codec_.width = 100; + codec_.height = 100; wrapper->InitEncode(&codec_, kSettings); // Trigger fallback to software. diff --git a/api/video_codecs/video_encoder_software_fallback_wrapper.cc b/api/video_codecs/video_encoder_software_fallback_wrapper.cc index 9edc9b074b..3b2ad4f18f 100644 --- a/api/video_codecs/video_encoder_software_fallback_wrapper.cc +++ b/api/video_codecs/video_encoder_software_fallback_wrapper.cc @@ -19,6 +19,7 @@ #include "absl/types/optional.h" #include "api/fec_controller_override.h" +#include "api/video/i420_buffer.h" #include "api/video/video_bitrate_allocation.h" #include "api/video/video_frame.h" #include "api/video_codecs/video_codec.h" @@ -194,6 +195,8 @@ class VideoEncoderSoftwareFallbackWrapper final : public VideoEncoder { EncodedImageCallback* callback_; const absl::optional fallback_params_; + int32_t EncodeWithMainEncoder(const VideoFrame& frame, + const std::vector* frame_types); }; VideoEncoderSoftwareFallbackWrapper::VideoEncoderSoftwareFallbackWrapper( @@ -335,22 +338,47 @@ int32_t VideoEncoderSoftwareFallbackWrapper::Encode( case EncoderState::kUninitialized: return WEBRTC_VIDEO_CODEC_ERROR; case EncoderState::kMainEncoderUsed: { - int32_t ret = encoder_->Encode(frame, frame_types); - // If requested, try a software fallback. - bool fallback_requested = (ret == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE); - if (fallback_requested && InitFallbackEncoder(/*is_forced=*/false)) { - // Start using the fallback with this frame. - PrimeEncoder(current_encoder()); - return fallback_encoder_->Encode(frame, frame_types); - } - // Fallback encoder failed too, return original error code. - return ret; + return EncodeWithMainEncoder(frame, frame_types); } case EncoderState::kFallbackDueToFailure: case EncoderState::kForcedFallback: return fallback_encoder_->Encode(frame, frame_types); } } +int32_t VideoEncoderSoftwareFallbackWrapper::EncodeWithMainEncoder( + const VideoFrame& frame, + const std::vector* frame_types) { + int32_t ret = encoder_->Encode(frame, frame_types); + // If requested, try a software fallback. + bool fallback_requested = (ret == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE); + if (fallback_requested && InitFallbackEncoder(/*is_forced=*/false)) { + // Start using the fallback with this frame. + PrimeEncoder(current_encoder()); + if (frame.video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative && + fallback_encoder_->GetEncoderInfo().supports_native_handle) { + return fallback_encoder_->Encode(frame, frame_types); + } else { + RTC_LOG(INFO) << "Fallback encoder does not support native handle - " + "converting frame to I420"; + rtc::scoped_refptr src_buffer = + frame.video_frame_buffer()->ToI420(); + if (!src_buffer) { + RTC_LOG(LS_ERROR) << "Failed to convert from to I420"; + return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; + } + rtc::scoped_refptr dst_buffer = + I420Buffer::Create(codec_settings_.width, codec_settings_.height); + dst_buffer->ScaleFrom(*src_buffer); + VideoFrame scaled_frame = frame; + scaled_frame.set_video_frame_buffer(dst_buffer); + scaled_frame.set_update_rect(VideoFrame::UpdateRect{ + 0, 0, scaled_frame.width(), scaled_frame.height()}); + return fallback_encoder_->Encode(scaled_frame, frame_types); + } + } + // Fallback encoder failed too, return original error code. + return ret; +} void VideoEncoderSoftwareFallbackWrapper::SetRates( const RateControlParameters& parameters) {