From 22c76c4e65679edabfc99d050d40c944cd5ce092 Mon Sep 17 00:00:00 2001 From: asapersson Date: Wed, 16 Aug 2017 00:53:59 -0700 Subject: [PATCH] Add support for a forced software encoder fallback. Make it possible to switch from VP8 HW -> VP8 SW -> VP8 HW depending on bitrate and resolution. BUG=webrtc:6634 Review-Url: https://codereview.webrtc.org/2988963002 Cr-Commit-Position: refs/heads/master@{#19362} --- .../videoencodersoftwarefallbackwrapper.cc | 177 +++++++++- .../videoencodersoftwarefallbackwrapper.h | 30 ++ ...encodersoftwarefallbackwrapper_unittest.cc | 308 +++++++++++++++++- .../codecs/vp8/default_temporal_layers.cc | 13 +- 4 files changed, 509 insertions(+), 19 deletions(-) diff --git a/webrtc/media/engine/videoencodersoftwarefallbackwrapper.cc b/webrtc/media/engine/videoencodersoftwarefallbackwrapper.cc index d9d6f32935..9f5dedabc0 100644 --- a/webrtc/media/engine/videoencodersoftwarefallbackwrapper.cc +++ b/webrtc/media/engine/videoencodersoftwarefallbackwrapper.cc @@ -12,9 +12,58 @@ #include "webrtc/media/engine/internalencoderfactory.h" #include "webrtc/modules/video_coding/include/video_error_codes.h" +#include "webrtc/rtc_base/checks.h" #include "webrtc/rtc_base/logging.h" +#include "webrtc/rtc_base/timeutils.h" +#include "webrtc/system_wrappers/include/field_trial.h" namespace webrtc { +namespace { +const char kVp8ForceFallbackEncoderFieldTrial[] = + "WebRTC-VP8-Forced-Fallback-Encoder"; + +bool EnableForcedFallback(const cricket::VideoCodec& codec) { + if (!webrtc::field_trial::IsEnabled(kVp8ForceFallbackEncoderFieldTrial)) + return false; + + return (PayloadNameToCodecType(codec.name).value_or(kVideoCodecUnknown) == + kVideoCodecVP8); +} + +bool IsForcedFallbackPossible(const VideoCodec& codec_settings) { + return codec_settings.codecType == kVideoCodecVP8 && + codec_settings.numberOfSimulcastStreams <= 1 && + codec_settings.VP8().numberOfTemporalLayers == 1; +} + +void GetForcedFallbackParamsFromFieldTrialGroup(uint32_t* param_low_kbps, + uint32_t* param_high_kbps, + int64_t* param_min_low_ms) { + RTC_DCHECK(param_low_kbps); + RTC_DCHECK(param_high_kbps); + RTC_DCHECK(param_min_low_ms); + std::string group = + webrtc::field_trial::FindFullName(kVp8ForceFallbackEncoderFieldTrial); + if (group.empty()) + return; + + int low_kbps; + int high_kbps; + int min_low_ms; + if (sscanf(group.c_str(), "Enabled-%d,%d,%d", &low_kbps, &high_kbps, + &min_low_ms) != 3) { + LOG(LS_WARNING) << "Invalid number of forced fallback parameters provided."; + return; + } + if (min_low_ms <= 0 || low_kbps <= 0 || high_kbps <= low_kbps) { + LOG(LS_WARNING) << "Invalid forced fallback parameter value provided."; + return; + } + *param_low_kbps = low_kbps; + *param_high_kbps = high_kbps; + *param_min_low_ms = min_low_ms; +} +} // namespace VideoEncoderSoftwareFallbackWrapper::VideoEncoderSoftwareFallbackWrapper( const cricket::VideoCodec& codec, @@ -28,7 +77,14 @@ VideoEncoderSoftwareFallbackWrapper::VideoEncoderSoftwareFallbackWrapper( rtt_(0), codec_(codec), encoder_(encoder), - callback_(nullptr) {} + callback_(nullptr), + forced_fallback_possible_(EnableForcedFallback(codec)) { + if (forced_fallback_possible_) { + GetForcedFallbackParamsFromFieldTrialGroup(&forced_fallback_.low_kbps, + &forced_fallback_.high_kbps, + &forced_fallback_.min_low_ms); + } +} bool VideoEncoderSoftwareFallbackWrapper::InitFallbackEncoder() { cricket::InternalEncoderFactory internal_factory; @@ -76,6 +132,13 @@ int32_t VideoEncoderSoftwareFallbackWrapper::InitEncode( // Clear stored rate/channel parameters. rates_set_ = false; channel_parameters_set_ = false; + ValidateSettingsForForcedFallback(); + + // Try to reinit forced software codec if it is in use. + if (TryReInitForcedFallbackEncoder()) { + return WEBRTC_VIDEO_CODEC_OK; + } + forced_fallback_.Reset(); int32_t ret = encoder_->InitEncode(codec_settings, number_of_cores, max_payload_size); @@ -117,11 +180,26 @@ int32_t VideoEncoderSoftwareFallbackWrapper::Encode( const VideoFrame& frame, const CodecSpecificInfo* codec_specific_info, const std::vector* frame_types) { + if (TryReleaseForcedFallbackEncoder()) { + // Frame may have been converted from kNative to kI420 during fallback. + if (encoder_->SupportsNativeHandle() && + frame.video_frame_buffer()->type() != VideoFrameBuffer::Type::kNative) { + LOG(LS_WARNING) << "Encoder supports native frames, dropping one frame " + << "to avoid possible reconfig due to format change."; + return WEBRTC_VIDEO_CODEC_ERROR; + } + } if (fallback_encoder_) return fallback_encoder_->Encode(frame, codec_specific_info, frame_types); int32_t ret = encoder_->Encode(frame, codec_specific_info, frame_types); // If requested, try a software fallback. - if (ret == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE && InitFallbackEncoder()) { + bool fallback_requested = + (ret == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE) || + (ret == WEBRTC_VIDEO_CODEC_OK && RequestForcedFallback()); + if (fallback_requested && InitFallbackEncoder()) { + // Fallback was successful. + if (ret == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE) + forced_fallback_.Reset(); // Not a forced fallback. if (frame.video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative && !fallback_encoder_->SupportsNativeHandle()) { LOG(LS_WARNING) << "Fallback encoder doesn't support native frames, " @@ -129,7 +207,7 @@ int32_t VideoEncoderSoftwareFallbackWrapper::Encode( return WEBRTC_VIDEO_CODEC_ERROR; } - // Fallback was successful, so start using it with this frame. + // Start using the fallback with this frame. return fallback_encoder_->Encode(frame, codec_specific_info, frame_types); } return ret; @@ -176,4 +254,97 @@ const char *VideoEncoderSoftwareFallbackWrapper::ImplementationName() const { return encoder_->ImplementationName(); } +bool VideoEncoderSoftwareFallbackWrapper::IsForcedFallbackActive() const { + return (forced_fallback_possible_ && fallback_encoder_ && + forced_fallback_.start_ms); +} + +bool VideoEncoderSoftwareFallbackWrapper::RequestForcedFallback() { + if (!forced_fallback_possible_ || fallback_encoder_ || !rates_set_) + return false; + + // No fallback encoder. + return forced_fallback_.ShouldStart(bitrate_allocation_.get_sum_kbps(), + codec_settings_); +} + +bool VideoEncoderSoftwareFallbackWrapper::TryReleaseForcedFallbackEncoder() { + if (!IsForcedFallbackActive()) + return false; + + if (!forced_fallback_.ShouldStop(bitrate_allocation_.get_sum_kbps())) + return false; + + // Release the forced fallback encoder. + if (encoder_->InitEncode(&codec_settings_, number_of_cores_, + max_payload_size_) == WEBRTC_VIDEO_CODEC_OK) { + LOG(LS_INFO) << "Stop forced SW encoder fallback, max bitrate exceeded."; + fallback_encoder_->Release(); + fallback_encoder_.reset(); + forced_fallback_.Reset(); + return true; + } + return false; +} + +bool VideoEncoderSoftwareFallbackWrapper::TryReInitForcedFallbackEncoder() { + if (!IsForcedFallbackActive()) + return false; + + // Encoder reconfigured. + if (!forced_fallback_.IsValid(codec_settings_)) { + LOG(LS_INFO) << "Stop forced SW encoder fallback, max pixels exceeded."; + return false; + } + // Settings valid, reinitialize the forced fallback encoder. + if (fallback_encoder_->InitEncode(&codec_settings_, number_of_cores_, + max_payload_size_) != + WEBRTC_VIDEO_CODEC_OK) { + LOG(LS_ERROR) << "Failed to init forced SW encoder fallback."; + return false; + } + return true; +} + +void VideoEncoderSoftwareFallbackWrapper::ValidateSettingsForForcedFallback() { + if (!forced_fallback_possible_) + return; + + if (!IsForcedFallbackPossible(codec_settings_)) { + if (IsForcedFallbackActive()) { + fallback_encoder_->Release(); + fallback_encoder_.reset(); + } + LOG(LS_INFO) << "Disable forced_fallback_possible_ due to settings."; + forced_fallback_possible_ = false; + } +} + +bool VideoEncoderSoftwareFallbackWrapper::ForcedFallbackParams::ShouldStart( + uint32_t bitrate_kbps, + const VideoCodec& codec) { + if (bitrate_kbps > low_kbps || !IsValid(codec)) { + start_ms.reset(); + return false; + } + + // Has bitrate been below |low_kbps| for long enough duration. + int64_t now_ms = rtc::TimeMillis(); + if (!start_ms) + start_ms.emplace(now_ms); + + if ((now_ms - *start_ms) >= min_low_ms) { + LOG(LS_INFO) << "Request forced SW encoder fallback."; + // In case the request fails, update time to avoid too frequent requests. + start_ms.emplace(now_ms); + return true; + } + return false; +} + +bool VideoEncoderSoftwareFallbackWrapper::ForcedFallbackParams::ShouldStop( + uint32_t bitrate_kbps) const { + return bitrate_kbps >= high_kbps; +} + } // namespace webrtc diff --git a/webrtc/media/engine/videoencodersoftwarefallbackwrapper.h b/webrtc/media/engine/videoencodersoftwarefallbackwrapper.h index 61c1a11c8d..ea7e13d353 100644 --- a/webrtc/media/engine/videoencodersoftwarefallbackwrapper.h +++ b/webrtc/media/engine/videoencodersoftwarefallbackwrapper.h @@ -49,6 +49,33 @@ class VideoEncoderSoftwareFallbackWrapper : public VideoEncoder { private: bool InitFallbackEncoder(); + // If |forced_fallback_possible_| is true: + // The forced fallback is requested if the target bitrate is below |low_kbps| + // for more than |min_low_ms| and the input video resolution is not larger + // than |kMaxPixels|. + // If the bitrate is above |high_kbps|, the forced fallback is requested to + // immediately be stopped. + class ForcedFallbackParams { + public: + bool ShouldStart(uint32_t bitrate_kbps, const VideoCodec& codec); + bool ShouldStop(uint32_t bitrate_kbps) const; + void Reset() { start_ms.reset(); } + bool IsValid(const VideoCodec& codec) const { + return codec.width * codec.height <= kMaxPixels; + } + rtc::Optional start_ms; // Set when bitrate is below |low_kbps|. + uint32_t low_kbps = 100; + uint32_t high_kbps = 150; + int64_t min_low_ms = 10000; + const int kMaxPixels = 320 * 240; + }; + + bool RequestForcedFallback(); + bool TryReleaseForcedFallbackEncoder(); + bool TryReInitForcedFallbackEncoder(); + void ValidateSettingsForForcedFallback(); + bool IsForcedFallbackActive() const; + // Settings used in the last InitEncode call and used if a dynamic fallback to // software is required. VideoCodec codec_settings_; @@ -71,6 +98,9 @@ class VideoEncoderSoftwareFallbackWrapper : public VideoEncoder { std::unique_ptr fallback_encoder_; std::string fallback_implementation_name_; EncodedImageCallback* callback_; + + bool forced_fallback_possible_; + ForcedFallbackParams forced_fallback_; }; } // namespace webrtc diff --git a/webrtc/media/engine/videoencodersoftwarefallbackwrapper_unittest.cc b/webrtc/media/engine/videoencodersoftwarefallbackwrapper_unittest.cc index e196587f05..f66551958f 100644 --- a/webrtc/media/engine/videoencodersoftwarefallbackwrapper_unittest.cc +++ b/webrtc/media/engine/videoencodersoftwarefallbackwrapper_unittest.cc @@ -18,18 +18,27 @@ #include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/modules/video_coding/include/video_error_codes.h" #include "webrtc/rtc_base/checks.h" +#include "webrtc/rtc_base/fakeclock.h" +#include "webrtc/test/field_trial.h" #include "webrtc/test/gtest.h" namespace webrtc { - +namespace { const int kWidth = 320; const int kHeight = 240; +const int kNumCores = 2; +const uint32_t kFramerate = 30; const size_t kMaxPayloadSize = 800; +} // namespace class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { protected: VideoEncoderSoftwareFallbackWrapperTest() - : fallback_wrapper_(cricket::VideoCodec("VP8"), &fake_encoder_) {} + : VideoEncoderSoftwareFallbackWrapperTest("") {} + explicit VideoEncoderSoftwareFallbackWrapperTest( + const std::string& field_trials) + : override_field_trials_(field_trials), + fallback_wrapper_(cricket::VideoCodec("VP8"), &fake_encoder_) {} class CountingFakeEncoder : public VideoEncoder { public: @@ -77,7 +86,7 @@ class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { bool SupportsNativeHandle() const override { ++supports_native_handle_count_; - return false; + return supports_native_handle_; } const char* ImplementationName() const override { @@ -93,6 +102,7 @@ class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { int set_channel_parameters_count_ = 0; int set_rates_count_ = 0; mutable int supports_native_handle_count_ = 0; + bool supports_native_handle_ = false; }; class FakeEncodedImageCallback : public EncodedImageCallback { @@ -112,10 +122,12 @@ class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { void UtilizeFallbackEncoder(); void FallbackFromEncodeRequest(); void EncodeFrame(); + void EncodeFrame(int expected_ret); void CheckLastEncoderName(const char* expected_name) { EXPECT_STREQ(expected_name, callback_.last_codec_name_.c_str()); } + test::ScopedFieldTrials override_field_trials_; FakeEncodedImageCallback callback_; CountingFakeEncoder fake_encoder_; VideoEncoderSoftwareFallbackWrapper fallback_wrapper_; @@ -125,13 +137,16 @@ class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { }; void VideoEncoderSoftwareFallbackWrapperTest::EncodeFrame() { + EncodeFrame(WEBRTC_VIDEO_CODEC_OK); +} + +void VideoEncoderSoftwareFallbackWrapperTest::EncodeFrame(int expected_ret) { rtc::scoped_refptr buffer = I420Buffer::Create(kWidth, kHeight); I420Buffer::SetBlack(buffer); std::vector types(1, kVideoFrameKey); frame_.reset(new VideoFrame(buffer, 0, 0, webrtc::kVideoRotation_0)); - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, - fallback_wrapper_.Encode(*frame_, nullptr, &types)); + EXPECT_EQ(expected_ret, fallback_wrapper_.Encode(*frame_, nullptr, &types)); } void VideoEncoderSoftwareFallbackWrapperTest::UtilizeFallbackEncoder() { @@ -140,7 +155,7 @@ void VideoEncoderSoftwareFallbackWrapperTest::UtilizeFallbackEncoder() { // Register with failing fake encoder. Should succeed with VP8 fallback. codec_.codecType = kVideoCodecVP8; - codec_.maxFramerate = 30; + codec_.maxFramerate = kFramerate; codec_.width = kWidth; codec_.height = kHeight; codec_.VP8()->numberOfTemporalLayers = 1; @@ -152,10 +167,11 @@ void VideoEncoderSoftwareFallbackWrapperTest::UtilizeFallbackEncoder() { fake_encoder_.init_encode_return_code_ = WEBRTC_VIDEO_CODEC_ERROR; EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, - fallback_wrapper_.InitEncode(&codec_, 2, kMaxPayloadSize)); - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, - fallback_wrapper_.SetRateAllocation( - rate_allocator_->GetAllocation(300000, 30), 30)); + fallback_wrapper_.InitEncode(&codec_, kNumCores, kMaxPayloadSize)); + EXPECT_EQ( + WEBRTC_VIDEO_CODEC_OK, + fallback_wrapper_.SetRateAllocation( + rate_allocator_->GetAllocation(300000, kFramerate), kFramerate)); int callback_count = callback_.callback_count_; int encode_count = fake_encoder_.encode_count_; @@ -167,7 +183,7 @@ void VideoEncoderSoftwareFallbackWrapperTest::UtilizeFallbackEncoder() { void VideoEncoderSoftwareFallbackWrapperTest::FallbackFromEncodeRequest() { fallback_wrapper_.RegisterEncodeCompleteCallback(&callback_); codec_.codecType = kVideoCodecVP8; - codec_.maxFramerate = 30; + codec_.maxFramerate = kFramerate; codec_.width = kWidth; codec_.height = kHeight; codec_.VP8()->numberOfTemporalLayers = 1; @@ -177,9 +193,10 @@ void VideoEncoderSoftwareFallbackWrapperTest::FallbackFromEncodeRequest() { rate_allocator_.reset( new SimulcastRateAllocator(codec_, std::move(tl_factory))); fallback_wrapper_.InitEncode(&codec_, 2, kMaxPayloadSize); - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, - fallback_wrapper_.SetRateAllocation( - rate_allocator_->GetAllocation(300000, 30), 30)); + EXPECT_EQ( + WEBRTC_VIDEO_CODEC_OK, + fallback_wrapper_.SetRateAllocation( + rate_allocator_->GetAllocation(300000, kFramerate), kFramerate)); EXPECT_EQ(1, fake_encoder_.init_encode_count_); // Have the non-fallback encoder request a software fallback. @@ -284,7 +301,7 @@ TEST_F(VideoEncoderSoftwareFallbackWrapperTest, TEST_F(VideoEncoderSoftwareFallbackWrapperTest, ReportsImplementationName) { VideoCodec codec = {}; fallback_wrapper_.RegisterEncodeCompleteCallback(&callback_); - fallback_wrapper_.InitEncode(&codec, 2, kMaxPayloadSize); + fallback_wrapper_.InitEncode(&codec, kNumCores, kMaxPayloadSize); EncodeFrame(); CheckLastEncoderName("fake-encoder"); } @@ -297,4 +314,265 @@ TEST_F(VideoEncoderSoftwareFallbackWrapperTest, CheckLastEncoderName("libvpx"); } +namespace { +const int kLowKbps = 220; +const int kHighKbps = 300; +const int kMinLowDurationMs = 4000; +const std::string kFieldTrial = "WebRTC-VP8-Forced-Fallback-Encoder"; +} // namespace + +class ForcedFallbackTest : public VideoEncoderSoftwareFallbackWrapperTest { + public: + ForcedFallbackTest(const std::string& field_trials) + : VideoEncoderSoftwareFallbackWrapperTest(field_trials) {} + + ~ForcedFallbackTest() override {} + + protected: + void SetUp() override { + clock_.SetTimeMicros(1234); + ConfigureVp8Codec(); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.InitEncode( + &codec_, kNumCores, kMaxPayloadSize)); + EXPECT_EQ(1, fake_encoder_.init_encode_count_); + } + + void TearDown() override { + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release()); + } + + void ConfigureVp8Codec() { + fallback_wrapper_.RegisterEncodeCompleteCallback(&callback_); + std::unique_ptr tl_factory( + new TemporalLayersFactory()); + codec_.codecType = kVideoCodecVP8; + codec_.maxFramerate = kFramerate; + codec_.width = kWidth; + codec_.height = kHeight; + codec_.VP8()->numberOfTemporalLayers = 1; + codec_.VP8()->tl_factory = tl_factory.get(); + rate_allocator_.reset( + new SimulcastRateAllocator(codec_, std::move(tl_factory))); + } + + void SetRateAllocation(uint32_t bitrate_kbps) { + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.SetRateAllocation( + rate_allocator_->GetAllocation( + bitrate_kbps * 1000, kFramerate), + kFramerate)); + } + + void EncodeFrameAndVerifyLastName(const char* expected_name) { + EncodeFrameAndVerifyLastName(expected_name, WEBRTC_VIDEO_CODEC_OK); + } + + void EncodeFrameAndVerifyLastName(const char* expected_name, + int expected_ret) { + EncodeFrame(expected_ret); + CheckLastEncoderName(expected_name); + } + + rtc::ScopedFakeClock clock_; +}; + +class ForcedFallbackTestEnabled : public ForcedFallbackTest { + public: + ForcedFallbackTestEnabled() + : ForcedFallbackTest(kFieldTrial + "/Enabled-" + + std::to_string(kLowKbps) + "," + + std::to_string(kHighKbps) + "," + + std::to_string(kMinLowDurationMs) + "/") {} +}; + +class ForcedFallbackTestDisabled : public ForcedFallbackTest { + public: + ForcedFallbackTestDisabled() + : ForcedFallbackTest(kFieldTrial + "/Disabled/") {} +}; + +TEST_F(ForcedFallbackTestDisabled, NoFallbackWithoutFieldTrial) { + // Bitrate at low threshold. + SetRateAllocation(kLowKbps); + EncodeFrameAndVerifyLastName("fake-encoder"); + // Duration passed, expect no fallback. + clock_.AdvanceTime(rtc::TimeDelta::FromMilliseconds(kMinLowDurationMs)); + EncodeFrameAndVerifyLastName("fake-encoder"); +} + +TEST_F(ForcedFallbackTestEnabled, FallbackIfAtLowLimit) { + // Bitrate at low threshold. + SetRateAllocation(kLowKbps); + EncodeFrameAndVerifyLastName("fake-encoder"); + // Duration passed, expect fallback. + clock_.AdvanceTime(rtc::TimeDelta::FromMilliseconds(kMinLowDurationMs)); + EncodeFrameAndVerifyLastName("libvpx"); +} + +TEST_F(ForcedFallbackTestEnabled, NoFallbackIfNotAtLowLimit) { + // Bitrate above low threshold. + SetRateAllocation(kLowKbps + 1); + EncodeFrameAndVerifyLastName("fake-encoder"); + // Duration passed, expect no fallback. + clock_.AdvanceTime(rtc::TimeDelta::FromMilliseconds(kMinLowDurationMs)); + EncodeFrameAndVerifyLastName("fake-encoder"); +} + +TEST_F(ForcedFallbackTestEnabled, NoFallbackIfResolutionIsTooLarge) { + // Resolution above max pixels. + codec_.width += 1; + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + fallback_wrapper_.InitEncode(&codec_, kNumCores, kMaxPayloadSize)); + // Bitrate at low threshold. + SetRateAllocation(kLowKbps); + EncodeFrameAndVerifyLastName("fake-encoder"); + // Duration passed, expect no fallback. + clock_.AdvanceTime(rtc::TimeDelta::FromMilliseconds(kMinLowDurationMs)); + EncodeFrameAndVerifyLastName("fake-encoder"); +} + +TEST_F(ForcedFallbackTestEnabled, FallbackIfMinDurationPassed) { + // Bitrate at low threshold. + SetRateAllocation(kLowKbps); + EncodeFrameAndVerifyLastName("fake-encoder"); + // Duration not passed, expect no fallback. + clock_.AdvanceTime(rtc::TimeDelta::FromMilliseconds(kMinLowDurationMs - 1)); + EncodeFrameAndVerifyLastName("fake-encoder"); + // Duration passed, expect fallback. + clock_.AdvanceTime(rtc::TimeDelta::FromMilliseconds(1)); + EncodeFrameAndVerifyLastName("libvpx"); +} + +TEST_F(ForcedFallbackTestEnabled, FallbackStartTimeResetIfAboveLowLimit) { + // Bitrate at low threshold, start time set. + SetRateAllocation(kLowKbps); + EncodeFrameAndVerifyLastName("fake-encoder"); + // Duration not passed, expect no fallback. + clock_.AdvanceTime(rtc::TimeDelta::FromMilliseconds(kMinLowDurationMs - 1)); + EncodeFrameAndVerifyLastName("fake-encoder"); + + // Bitrate above low threshold, start time reset. + SetRateAllocation(kLowKbps + 1); + clock_.AdvanceTime(rtc::TimeDelta::FromMilliseconds(1)); + EncodeFrameAndVerifyLastName("fake-encoder"); + + // Bitrate at low threshold, start time set. + SetRateAllocation(kLowKbps); + EncodeFrameAndVerifyLastName("fake-encoder"); + // Duration not passed, expect no fallback. + clock_.AdvanceTime(rtc::TimeDelta::FromMilliseconds(kMinLowDurationMs - 1)); + EncodeFrameAndVerifyLastName("fake-encoder"); + // Duration passed, expect fallback. + clock_.AdvanceTime(rtc::TimeDelta::FromMilliseconds(1)); + EncodeFrameAndVerifyLastName("libvpx"); +} + +TEST_F(ForcedFallbackTestEnabled, FallbackEndsIfAtHighLimit) { + // Bitrate at low threshold. + SetRateAllocation(kLowKbps); + EncodeFrameAndVerifyLastName("fake-encoder"); + // Duration passed, expect fallback. + clock_.AdvanceTime(rtc::TimeDelta::FromMilliseconds(kMinLowDurationMs)); + EncodeFrameAndVerifyLastName("libvpx"); + // Bitrate below high threshold, expect fallback. + SetRateAllocation(kHighKbps - 1); + EncodeFrameAndVerifyLastName("libvpx"); + // Bitrate at high threshold, expect fallback ended. + SetRateAllocation(kHighKbps); + EncodeFrameAndVerifyLastName("fake-encoder"); +} + +TEST_F(ForcedFallbackTestEnabled, MultipleStartEndFallback) { + const int kNumRuns = 5; + for (int i = 0; i < kNumRuns; ++i) { + // Bitrate at low threshold. + SetRateAllocation(kLowKbps); + EncodeFrameAndVerifyLastName("fake-encoder"); + // Duration passed, expect fallback. + clock_.AdvanceTime(rtc::TimeDelta::FromMilliseconds(kMinLowDurationMs)); + EncodeFrameAndVerifyLastName("libvpx"); + // Bitrate at high threshold, expect fallback ended. + SetRateAllocation(kHighKbps); + EncodeFrameAndVerifyLastName("fake-encoder"); + } +} + +TEST_F(ForcedFallbackTestEnabled, DropsFirstNonNativeFrameAfterFallbackEnds) { + fake_encoder_.supports_native_handle_ = true; + + // Bitrate at low threshold. + SetRateAllocation(kLowKbps); + EncodeFrameAndVerifyLastName("fake-encoder"); + // Duration passed, expect fallback. + clock_.AdvanceTime(rtc::TimeDelta::FromMilliseconds(kMinLowDurationMs)); + EncodeFrameAndVerifyLastName("libvpx"); + // Bitrate at high threshold, fallback should be ended but first non-native + // frame dropped (i.e. frame not encoded). + SetRateAllocation(kHighKbps); + EncodeFrameAndVerifyLastName("libvpx", WEBRTC_VIDEO_CODEC_ERROR); + // Next frame should be encoded. + EncodeFrameAndVerifyLastName("fake-encoder"); +} + +TEST_F(ForcedFallbackTestEnabled, FallbackIsKeptWhenInitEncodeIsCalled) { + // Bitrate below low threshold. + SetRateAllocation(kLowKbps - 1); + EncodeFrameAndVerifyLastName("fake-encoder"); + // Duration passed, expect fallback. + clock_.AdvanceTime(rtc::TimeDelta::FromMilliseconds(kMinLowDurationMs)); + EncodeFrameAndVerifyLastName("libvpx"); + + // Re-initialize encoder, still expect fallback. + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + fallback_wrapper_.InitEncode(&codec_, kNumCores, kMaxPayloadSize)); + EXPECT_EQ(1, fake_encoder_.init_encode_count_); // No change. + SetRateAllocation(kLowKbps); + EncodeFrameAndVerifyLastName("libvpx"); +} + +TEST_F(ForcedFallbackTestEnabled, FallbackIsEndedWhenResolutionIsTooLarge) { + // Bitrate below low threshold. + SetRateAllocation(kLowKbps - 1); + EncodeFrameAndVerifyLastName("fake-encoder"); + // Duration passed, expect fallback. + clock_.AdvanceTime(rtc::TimeDelta::FromMilliseconds(kMinLowDurationMs)); + EncodeFrameAndVerifyLastName("libvpx"); + + // Re-initialize encoder with a larger resolution, expect no fallback. + codec_.width += 1; + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + fallback_wrapper_.InitEncode(&codec_, kNumCores, kMaxPayloadSize)); + EXPECT_EQ(2, fake_encoder_.init_encode_count_); + SetRateAllocation(kLowKbps); + EncodeFrameAndVerifyLastName("fake-encoder"); +} + +TEST_F(ForcedFallbackTestEnabled, FallbackIsEndedForNonValidSettings) { + // Bitrate below low threshold. + SetRateAllocation(kLowKbps - 1); + EncodeFrameAndVerifyLastName("fake-encoder"); + // Duration passed, expect fallback. + clock_.AdvanceTime(rtc::TimeDelta::FromMilliseconds(kMinLowDurationMs)); + EncodeFrameAndVerifyLastName("libvpx"); + + // Re-initialize encoder with invalid setting, expect no fallback. + codec_.VP8()->numberOfTemporalLayers = 2; + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + fallback_wrapper_.InitEncode(&codec_, kNumCores, kMaxPayloadSize)); + EXPECT_EQ(2, fake_encoder_.init_encode_count_); + SetRateAllocation(kLowKbps); + EncodeFrameAndVerifyLastName("fake-encoder"); + + // Re-initialize encoder with valid setting but fallback disabled from now on. + codec_.VP8()->numberOfTemporalLayers = 1; + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + fallback_wrapper_.InitEncode(&codec_, kNumCores, kMaxPayloadSize)); + EXPECT_EQ(3, fake_encoder_.init_encode_count_); + // Bitrate at low threshold. + SetRateAllocation(kLowKbps); + EncodeFrameAndVerifyLastName("fake-encoder"); + // Duration passed, expect no fallback. + clock_.AdvanceTime(rtc::TimeDelta::FromMilliseconds(kMinLowDurationMs)); + EncodeFrameAndVerifyLastName("fake-encoder"); +} + } // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc index ce7faf9f98..979c64e243 100644 --- a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc +++ b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc @@ -19,6 +19,7 @@ #include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h" #include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/rtc_base/checks.h" +#include "webrtc/system_wrappers/include/field_trial.h" #include "vpx/vpx_encoder.h" #include "vpx/vp8cx.h" @@ -248,6 +249,16 @@ std::vector GetTemporalPattern(size_t num_layers) { TemporalLayers::kNone, TemporalLayers::kNone, TemporalLayers::kNone)}; } +// Temporary fix for forced SW fallback. +// For VP8 SW codec, |TemporalLayers| is created and reported to +// SimulcastRateAllocator::OnTemporalLayersCreated but not for VP8 HW. +// Causes an issue when going from forced SW -> HW as |TemporalLayers| is not +// deregistred when deleted by SW codec (tl factory might not exist, owned by +// SimulcastRateAllocator). +bool ExcludeOnTemporalLayersCreated(int num_temporal_layers) { + return webrtc::field_trial::IsEnabled("WebRTC-VP8-Forced-Fallback-Encoder") && + num_temporal_layers == 1; +} } // namespace DefaultTemporalLayers::DefaultTemporalLayers(int number_of_temporal_layers, @@ -372,7 +383,7 @@ TemporalLayers* TemporalLayersFactory::Create( uint8_t initial_tl0_pic_idx) const { TemporalLayers* tl = new DefaultTemporalLayers(temporal_layers, initial_tl0_pic_idx); - if (listener_) + if (listener_ && !ExcludeOnTemporalLayersCreated(temporal_layers)) listener_->OnTemporalLayersCreated(simulcast_id, tl); return tl; }