Allow software fallback on lowest simulcast stream for temporal support

Bug: webrtc:11324
Change-Id: Ie505be0cda74c0444065d86c3727671c62bd4842
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/167527
Reviewed-by: Sebastian Jansson <srte@webrtc.org>
Reviewed-by: Evan Shrubsole <eshr@google.com>
Reviewed-by: Åsa Persson <asapersson@webrtc.org>
Commit-Queue: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30437}

api/test/mock_video_encoder.h

@@ -51,6 +51,10 @@ class MockVideoEncoder : public VideoEncoder {
   MOCK_METHOD0(Release, int32_t());
   MOCK_METHOD0(Reset, int32_t());
   MOCK_METHOD1(SetRates, void(const RateControlParameters& parameters));
+  MOCK_METHOD1(OnPacketLossRateUpdate, void(float packet_loss_rate));
+  MOCK_METHOD1(OnRttUpdate, void(int64_t rtt_ms));
+  MOCK_METHOD1(OnLossNotification,
+               void(const LossNotification& loss_notification));
   MOCK_CONST_METHOD0(GetEncoderInfo, EncoderInfo(void));
 };
 

api/video_codecs/BUILD.gn

@@ -137,6 +137,7 @@ rtc_library("rtc_software_fallback_wrappers") {
     "../../media:rtc_h264_profile_id",
     "../../media:rtc_media_base",
     "../../modules/video_coding:video_codec_interface",
+    "../../modules/video_coding:video_coding_utility",
     "../../rtc_base:checks",
     "../../rtc_base:rtc_base_approved",
     "../../rtc_base/system:rtc_export",

api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc

@@ -40,6 +40,7 @@
 #include "test/gtest.h"
 
 namespace webrtc {
+using ::testing::_;
 using ::testing::Return;
 
 namespace {
@@ -76,6 +77,17 @@ VideoEncoder::EncoderInfo GetEncoderInfoWithInternalSource(
   info.has_internal_source = internal_source;
   return info;
 }
+
+class FakeEncodedImageCallback : public EncodedImageCallback {
+ public:
+  Result OnEncodedImage(const EncodedImage& encoded_image,
+                        const CodecSpecificInfo* codec_specific_info,
+                        const RTPFragmentationHeader* fragmentation) override {
+    ++callback_count_;
+    return Result(Result::OK, callback_count_);
+  }
+  int callback_count_ = 0;
+};
 }  // namespace
 
 class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test {
@@ -86,9 +98,11 @@ class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test {
       const std::string& field_trials)
       : override_field_trials_(field_trials),
         fake_encoder_(new CountingFakeEncoder()),
+        wrapper_initialized_(false),
         fallback_wrapper_(CreateVideoEncoderSoftwareFallbackWrapper(
             std::unique_ptr<VideoEncoder>(VP8Encoder::Create()),
-            std::unique_ptr<VideoEncoder>(fake_encoder_))) {}
+            std::unique_ptr<VideoEncoder>(fake_encoder_),
+            false)) {}
 
   class CountingFakeEncoder : public VideoEncoder {
    public:
@@ -125,9 +139,7 @@ class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test {
       return WEBRTC_VIDEO_CODEC_OK;
     }
 
-    void SetRates(const RateControlParameters& parameters) override {
+    void SetRates(const RateControlParameters& parameters) override {}
-      ++set_rates_count_;
-    }
 
     EncoderInfo GetEncoderInfo() const override {
       ++supports_native_handle_count_;
@@ -144,23 +156,11 @@ class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test {
     int encode_count_ = 0;
     EncodedImageCallback* encode_complete_callback_ = nullptr;
     int release_count_ = 0;
-    int set_rates_count_ = 0;
     mutable int supports_native_handle_count_ = 0;
     bool supports_native_handle_ = false;
   };
 
-  class FakeEncodedImageCallback : public EncodedImageCallback {
+  void InitEncode();
-   public:
-    Result OnEncodedImage(
-        const EncodedImage& encoded_image,
-        const CodecSpecificInfo* codec_specific_info,
-        const RTPFragmentationHeader* fragmentation) override {
-      ++callback_count_;
-      return Result(Result::OK, callback_count_);
-    }
-    int callback_count_ = 0;
-  };
-
   void UtilizeFallbackEncoder();
   void FallbackFromEncodeRequest();
   void EncodeFrame();
@@ -174,6 +174,7 @@ class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test {
   FakeEncodedImageCallback callback_;
   // |fake_encoder_| is owned and released by |fallback_wrapper_|.
   CountingFakeEncoder* fake_encoder_;
+  bool wrapper_initialized_;
   std::unique_ptr<VideoEncoder> fallback_wrapper_;
   VideoCodec codec_ = {};
   std::unique_ptr<VideoFrame> frame_;
@@ -199,9 +200,42 @@ void VideoEncoderSoftwareFallbackWrapperTest::EncodeFrame(int expected_ret) {
   EXPECT_EQ(expected_ret, fallback_wrapper_->Encode(*frame_, &types));
 }
 
+void VideoEncoderSoftwareFallbackWrapperTest::InitEncode() {
+  if (!wrapper_initialized_) {
+    fallback_wrapper_->RegisterEncodeCompleteCallback(&callback_);
+    EXPECT_EQ(&callback_, fake_encoder_->encode_complete_callback_);
+  }
+
+  // Register fake encoder as main.
+  codec_.codecType = kVideoCodecVP8;
+  codec_.maxFramerate = kFramerate;
+  codec_.width = kWidth;
+  codec_.height = kHeight;
+  codec_.VP8()->numberOfTemporalLayers = 1;
+  rate_allocator_.reset(new SimulcastRateAllocator(codec_));
+
+  if (wrapper_initialized_) {
+    fallback_wrapper_->Release();
+  }
+
+  fake_encoder_->init_encode_return_code_ = WEBRTC_VIDEO_CODEC_OK;
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+            fallback_wrapper_->InitEncode(&codec_, kSettings));
+
+  if (!wrapper_initialized_) {
+    fallback_wrapper_->SetRates(VideoEncoder::RateControlParameters(
+        rate_allocator_->Allocate(
+            VideoBitrateAllocationParameters(300000, kFramerate)),
+        kFramerate));
+  }
+  wrapper_initialized_ = true;
+}
+
 void VideoEncoderSoftwareFallbackWrapperTest::UtilizeFallbackEncoder() {
-  fallback_wrapper_->RegisterEncodeCompleteCallback(&callback_);
+  if (!wrapper_initialized_) {
-  EXPECT_EQ(&callback_, fake_encoder_->encode_complete_callback_);
+    fallback_wrapper_->RegisterEncodeCompleteCallback(&callback_);
+    EXPECT_EQ(&callback_, fake_encoder_->encode_complete_callback_);
+  }
 
   // Register with failing fake encoder. Should succeed with VP8 fallback.
   codec_.codecType = kVideoCodecVP8;
@@ -211,6 +245,10 @@ void VideoEncoderSoftwareFallbackWrapperTest::UtilizeFallbackEncoder() {
   codec_.VP8()->numberOfTemporalLayers = 1;
   rate_allocator_.reset(new SimulcastRateAllocator(codec_));
 
+  if (wrapper_initialized_) {
+    fallback_wrapper_->Release();
+  }
+
   fake_encoder_->init_encode_return_code_ = WEBRTC_VIDEO_CODEC_ERROR;
   EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
             fallback_wrapper_->InitEncode(&codec_, kSettings));
@@ -234,6 +272,9 @@ void VideoEncoderSoftwareFallbackWrapperTest::FallbackFromEncodeRequest() {
   codec_.height = kHeight;
   codec_.VP8()->numberOfTemporalLayers = 1;
   rate_allocator_.reset(new SimulcastRateAllocator(codec_));
+  if (wrapper_initialized_) {
+    fallback_wrapper_->Release();
+  }
   fallback_wrapper_->InitEncode(&codec_, kSettings);
   fallback_wrapper_->SetRates(VideoEncoder::RateControlParameters(
       rate_allocator_->Allocate(
@@ -272,11 +313,24 @@ TEST_F(VideoEncoderSoftwareFallbackWrapperTest, CanUtilizeFallbackEncoder) {
 
 TEST_F(VideoEncoderSoftwareFallbackWrapperTest,
        InternalEncoderReleasedDuringFallback) {
+  EXPECT_EQ(0, fake_encoder_->init_encode_count_);
   EXPECT_EQ(0, fake_encoder_->release_count_);
+
+  InitEncode();
+
+  EXPECT_EQ(1, fake_encoder_->init_encode_count_);
+  EXPECT_EQ(0, fake_encoder_->release_count_);
+
   UtilizeFallbackEncoder();
+
+  // One successful InitEncode(), one failed.
+  EXPECT_EQ(2, fake_encoder_->init_encode_count_);
   EXPECT_EQ(1, fake_encoder_->release_count_);
+
   EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Release());
+
   // No extra release when the fallback is released.
+  EXPECT_EQ(2, fake_encoder_->init_encode_count_);
   EXPECT_EQ(1, fake_encoder_->release_count_);
 }
 
@@ -292,29 +346,30 @@ TEST_F(VideoEncoderSoftwareFallbackWrapperTest,
 
 TEST_F(VideoEncoderSoftwareFallbackWrapperTest,
        CanRegisterCallbackWhileUsingFallbackEncoder) {
+  InitEncode();
+  EXPECT_EQ(&callback_, fake_encoder_->encode_complete_callback_);
+
   UtilizeFallbackEncoder();
-  // Registering an encode-complete callback should still work when fallback
+
-  // encoder is being used.
+  // Registering an encode-complete callback will now pass to the fallback
+  // instead of the main encoder.
   FakeEncodedImageCallback callback2;
   fallback_wrapper_->RegisterEncodeCompleteCallback(&callback2);
-  EXPECT_EQ(&callback2, fake_encoder_->encode_complete_callback_);
+  EXPECT_EQ(&callback_, fake_encoder_->encode_complete_callback_);
 
   // Encoding a frame using the fallback should arrive at the new callback.
   std::vector<VideoFrameType> types(1, VideoFrameType::kVideoFrameKey);
   frame_->set_timestamp(frame_->timestamp() + 1000);
   EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Encode(*frame_, &types));
+  EXPECT_EQ(callback2.callback_count_, 1);
 
-  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Release());
+  // Re-initialize to use the main encoder, the new callback should be in use.
-}
+  InitEncode();
+  EXPECT_EQ(&callback2, fake_encoder_->encode_complete_callback_);
 
-TEST_F(VideoEncoderSoftwareFallbackWrapperTest,
+  frame_->set_timestamp(frame_->timestamp() + 2000);
-       SetRatesForwardedDuringFallback) {
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Encode(*frame_, &types));
-  UtilizeFallbackEncoder();
+  EXPECT_EQ(callback2.callback_count_, 2);
-  EXPECT_EQ(1, fake_encoder_->set_rates_count_);
-  fallback_wrapper_->SetRates(
-      VideoEncoder::RateControlParameters(VideoBitrateAllocation(), 1));
-  EXPECT_EQ(2, fake_encoder_->set_rates_count_);
-  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Release());
 }
 
 TEST_F(VideoEncoderSoftwareFallbackWrapperTest,
@@ -372,11 +427,12 @@ class ForcedFallbackTest : public VideoEncoderSoftwareFallbackWrapperTest {
   }
 
   void TearDown() override {
-    EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Release());
+    if (wrapper_initialized_) {
+      EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Release());
+    }
   }
 
   void ConfigureVp8Codec() {
-    fallback_wrapper_->RegisterEncodeCompleteCallback(&callback_);
     codec_.codecType = kVideoCodecVP8;
     codec_.maxFramerate = kFramerate;
     codec_.width = kWidth;
@@ -390,8 +446,13 @@ class ForcedFallbackTest : public VideoEncoderSoftwareFallbackWrapperTest {
   void InitEncode(int width, int height) {
     codec_.width = width;
     codec_.height = height;
+    if (wrapper_initialized_) {
+      fallback_wrapper_->Release();
+    }
     EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
               fallback_wrapper_->InitEncode(&codec_, kSettings));
+    fallback_wrapper_->RegisterEncodeCompleteCallback(&callback_);
+    wrapper_initialized_ = true;
     SetRateAllocation(kBitrateKbps);
   }
 
@@ -494,11 +555,11 @@ TEST_F(ForcedFallbackTestEnabled, FallbackIsEndedForNonValidSettings) {
   EXPECT_EQ(1, fake_encoder_->init_encode_count_);
   EncodeFrameAndVerifyLastName("fake-encoder");
 
-  // Re-initialize encoder with valid setting but fallback disabled from now on.
+  // Re-initialize encoder with valid setting.
   codec_.VP8()->numberOfTemporalLayers = 1;
   InitEncode(kWidth, kHeight);
-  EXPECT_EQ(2, fake_encoder_->init_encode_count_);
+  EXPECT_EQ(1, fake_encoder_->init_encode_count_);
-  EncodeFrameAndVerifyLastName("fake-encoder");
+  EncodeFrameAndVerifyLastName("libvpx");
 }
 
 TEST_F(ForcedFallbackTestEnabled, MultipleStartEndFallback) {
@@ -689,4 +750,247 @@ TEST(SoftwareFallbackEncoderTest, ReportsInternalSource) {
   EXPECT_FALSE(wrapper->GetEncoderInfo().has_internal_source);
 }
 
+class PreferTemporalLayersFallbackTest : public ::testing::Test {
+ public:
+  PreferTemporalLayersFallbackTest() {}
+  void SetUp() override {
+    sw_ = new ::testing::NiceMock<MockVideoEncoder>();
+    sw_info_.implementation_name = "sw";
+    EXPECT_CALL(*sw_, GetEncoderInfo).WillRepeatedly([&]() {
+      return sw_info_;
+    });
+    EXPECT_CALL(*sw_, InitEncode(_, _, _))
+        .WillRepeatedly(Return(WEBRTC_VIDEO_CODEC_OK));
+
+    hw_ = new ::testing::NiceMock<MockVideoEncoder>();
+    hw_info_.implementation_name = "hw";
+    EXPECT_CALL(*hw_, GetEncoderInfo()).WillRepeatedly([&]() {
+      return hw_info_;
+    });
+    EXPECT_CALL(*hw_, InitEncode(_, _, _))
+        .WillRepeatedly(Return(WEBRTC_VIDEO_CODEC_OK));
+
+    wrapper_ = CreateVideoEncoderSoftwareFallbackWrapper(
+        std::unique_ptr<VideoEncoder>(sw_), std::unique_ptr<VideoEncoder>(hw_),
+        /*prefer_temporal_support=*/true);
+
+    codec_settings.codecType = kVideoCodecVP8;
+    codec_settings.maxFramerate = kFramerate;
+    codec_settings.width = kWidth;
+    codec_settings.height = kHeight;
+    codec_settings.numberOfSimulcastStreams = 1;
+    codec_settings.VP8()->numberOfTemporalLayers = 1;
+  }
+
+ protected:
+  void SetSupportsLayers(VideoEncoder::EncoderInfo* info, bool tl_enabled) {
+    info->fps_allocation[0].clear();
+    int num_layers = 1;
+    if (tl_enabled) {
+      num_layers = codec_settings.VP8()->numberOfTemporalLayers;
+    }
+    for (int i = 0; i < num_layers; ++i) {
+      info->fps_allocation[0].push_back(
+          VideoEncoder::EncoderInfo::kMaxFramerateFraction >>
+          (num_layers - i - 1));
+    }
+  }
+
+  VideoCodec codec_settings;
+  ::testing::NiceMock<MockVideoEncoder>* sw_;
+  ::testing::NiceMock<MockVideoEncoder>* hw_;
+  VideoEncoder::EncoderInfo sw_info_;
+  VideoEncoder::EncoderInfo hw_info_;
+  std::unique_ptr<VideoEncoder> wrapper_;
+};
+
+TEST_F(PreferTemporalLayersFallbackTest, UsesMainWhenLayersNotUsed) {
+  codec_settings.VP8()->numberOfTemporalLayers = 1;
+  SetSupportsLayers(&hw_info_, true);
+  SetSupportsLayers(&sw_info_, true);
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+            wrapper_->InitEncode(&codec_settings, kSettings));
+  EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "hw");
+}
+
+TEST_F(PreferTemporalLayersFallbackTest, UsesMainWhenLayersSupported) {
+  codec_settings.VP8()->numberOfTemporalLayers = 2;
+  SetSupportsLayers(&hw_info_, true);
+  SetSupportsLayers(&sw_info_, true);
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+            wrapper_->InitEncode(&codec_settings, kSettings));
+  EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "hw");
+}
+
+TEST_F(PreferTemporalLayersFallbackTest,
+       UsesFallbackWhenLayersNotSupportedOnMain) {
+  codec_settings.VP8()->numberOfTemporalLayers = 2;
+  SetSupportsLayers(&hw_info_, false);
+  SetSupportsLayers(&sw_info_, true);
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+            wrapper_->InitEncode(&codec_settings, kSettings));
+  EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "sw");
+}
+
+TEST_F(PreferTemporalLayersFallbackTest, UsesMainWhenNeitherSupportsTemporal) {
+  codec_settings.VP8()->numberOfTemporalLayers = 2;
+  SetSupportsLayers(&hw_info_, false);
+  SetSupportsLayers(&sw_info_, false);
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+            wrapper_->InitEncode(&codec_settings, kSettings));
+  EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "hw");
+}
+
+TEST_F(PreferTemporalLayersFallbackTest, PrimesEncoderOnSwitch) {
+  codec_settings.VP8()->numberOfTemporalLayers = 2;
+  // Both support temporal layers, will use main one.
+  SetSupportsLayers(&hw_info_, true);
+  SetSupportsLayers(&sw_info_, true);
+
+  // On first InitEncode most params have no state and will not be
+  // called to update.
+  EXPECT_CALL(*hw_, RegisterEncodeCompleteCallback).Times(0);
+  EXPECT_CALL(*sw_, RegisterEncodeCompleteCallback).Times(0);
+
+  EXPECT_CALL(*hw_, SetFecControllerOverride).Times(0);
+  EXPECT_CALL(*sw_, SetFecControllerOverride).Times(0);
+
+  EXPECT_CALL(*hw_, SetRates).Times(0);
+  EXPECT_CALL(*hw_, SetRates).Times(0);
+
+  EXPECT_CALL(*hw_, OnPacketLossRateUpdate).Times(0);
+  EXPECT_CALL(*sw_, OnPacketLossRateUpdate).Times(0);
+
+  EXPECT_CALL(*hw_, OnRttUpdate).Times(0);
+  EXPECT_CALL(*sw_, OnRttUpdate).Times(0);
+
+  EXPECT_CALL(*hw_, OnLossNotification).Times(0);
+  EXPECT_CALL(*sw_, OnLossNotification).Times(0);
+
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+            wrapper_->InitEncode(&codec_settings, kSettings));
+  EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "hw");
+
+  FakeEncodedImageCallback callback1;
+  class DummyFecControllerOverride : public FecControllerOverride {
+   public:
+    void SetFecAllowed(bool fec_allowed) override {}
+  };
+  DummyFecControllerOverride fec_controller_override1;
+  VideoEncoder::RateControlParameters rate_params1;
+  float packet_loss1 = 0.1;
+  int64_t rtt1 = 1;
+  VideoEncoder::LossNotification lntf1;
+
+  EXPECT_CALL(*hw_, RegisterEncodeCompleteCallback(&callback1));
+  EXPECT_CALL(*sw_, RegisterEncodeCompleteCallback).Times(0);
+  wrapper_->RegisterEncodeCompleteCallback(&callback1);
+
+  EXPECT_CALL(*hw_, SetFecControllerOverride(&fec_controller_override1));
+  EXPECT_CALL(*sw_, SetFecControllerOverride).Times(0);
+  wrapper_->SetFecControllerOverride(&fec_controller_override1);
+
+  EXPECT_CALL(*hw_, SetRates(rate_params1));
+  EXPECT_CALL(*sw_, SetRates).Times(0);
+  wrapper_->SetRates(rate_params1);
+
+  EXPECT_CALL(*hw_, OnPacketLossRateUpdate(packet_loss1));
+  EXPECT_CALL(*sw_, OnPacketLossRateUpdate).Times(0);
+  wrapper_->OnPacketLossRateUpdate(packet_loss1);
+
+  EXPECT_CALL(*hw_, OnRttUpdate(rtt1));
+  EXPECT_CALL(*sw_, OnRttUpdate).Times(0);
+  wrapper_->OnRttUpdate(rtt1);
+
+  EXPECT_CALL(*hw_, OnLossNotification).Times(1);
+  EXPECT_CALL(*sw_, OnLossNotification).Times(0);
+  wrapper_->OnLossNotification(lntf1);
+
+  // Release and re-init, with fallback to software. This should trigger
+  // the software encoder to be primed with the current state.
+  wrapper_->Release();
+  EXPECT_CALL(*sw_, RegisterEncodeCompleteCallback(&callback1));
+  EXPECT_CALL(*hw_, RegisterEncodeCompleteCallback).Times(0);
+
+  EXPECT_CALL(*sw_, SetFecControllerOverride(&fec_controller_override1));
+  EXPECT_CALL(*hw_, SetFecControllerOverride).Times(0);
+
+  // Rate control parameters are cleared on InitEncode.
+  EXPECT_CALL(*sw_, SetRates).Times(0);
+  EXPECT_CALL(*hw_, SetRates).Times(0);
+
+  EXPECT_CALL(*sw_, OnPacketLossRateUpdate(packet_loss1));
+  EXPECT_CALL(*hw_, OnPacketLossRateUpdate).Times(0);
+
+  EXPECT_CALL(*sw_, OnRttUpdate(rtt1));
+  EXPECT_CALL(*hw_, OnRttUpdate).Times(0);
+
+  EXPECT_CALL(*sw_, OnLossNotification).Times(1);
+  EXPECT_CALL(*hw_, OnLossNotification).Times(0);
+
+  SetSupportsLayers(&hw_info_, false);
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+            wrapper_->InitEncode(&codec_settings, kSettings));
+  EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "sw");
+
+  // Update with all-new params for the software encoder.
+  FakeEncodedImageCallback callback2;
+  DummyFecControllerOverride fec_controller_override2;
+  VideoEncoder::RateControlParameters rate_params2;
+  float packet_loss2 = 0.2;
+  int64_t rtt2 = 2;
+  VideoEncoder::LossNotification lntf2;
+
+  EXPECT_CALL(*sw_, RegisterEncodeCompleteCallback(&callback2));
+  EXPECT_CALL(*hw_, RegisterEncodeCompleteCallback).Times(0);
+  wrapper_->RegisterEncodeCompleteCallback(&callback2);
+
+  EXPECT_CALL(*sw_, SetFecControllerOverride(&fec_controller_override2));
+  EXPECT_CALL(*hw_, SetFecControllerOverride).Times(0);
+  wrapper_->SetFecControllerOverride(&fec_controller_override2);
+
+  EXPECT_CALL(*sw_, SetRates(rate_params2));
+  EXPECT_CALL(*hw_, SetRates).Times(0);
+  wrapper_->SetRates(rate_params2);
+
+  EXPECT_CALL(*sw_, OnPacketLossRateUpdate(packet_loss2));
+  EXPECT_CALL(*hw_, OnPacketLossRateUpdate).Times(0);
+  wrapper_->OnPacketLossRateUpdate(packet_loss2);
+
+  EXPECT_CALL(*sw_, OnRttUpdate(rtt2));
+  EXPECT_CALL(*hw_, OnRttUpdate).Times(0);
+  wrapper_->OnRttUpdate(rtt2);
+
+  EXPECT_CALL(*sw_, OnLossNotification).Times(1);
+  EXPECT_CALL(*hw_, OnLossNotification).Times(0);
+  wrapper_->OnLossNotification(lntf2);
+
+  // Release and re-init, back to main encoder. This should trigger
+  // the main encoder to be primed with the current state.
+  wrapper_->Release();
+  EXPECT_CALL(*hw_, RegisterEncodeCompleteCallback(&callback2));
+  EXPECT_CALL(*sw_, RegisterEncodeCompleteCallback).Times(0);
+
+  EXPECT_CALL(*hw_, SetFecControllerOverride(&fec_controller_override2));
+  EXPECT_CALL(*sw_, SetFecControllerOverride).Times(0);
+
+  // Rate control parameters are cleared on InitEncode.
+  EXPECT_CALL(*sw_, SetRates).Times(0);
+  EXPECT_CALL(*hw_, SetRates).Times(0);
+
+  EXPECT_CALL(*hw_, OnPacketLossRateUpdate(packet_loss2));
+  EXPECT_CALL(*sw_, OnPacketLossRateUpdate).Times(0);
+
+  EXPECT_CALL(*hw_, OnRttUpdate(rtt2));
+  EXPECT_CALL(*sw_, OnRttUpdate).Times(0);
+
+  EXPECT_CALL(*hw_, OnLossNotification).Times(1);
+  EXPECT_CALL(*sw_, OnLossNotification).Times(0);
+
+  SetSupportsLayers(&hw_info_, true);
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+            wrapper_->InitEncode(&codec_settings, kSettings));
+  EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "hw");
+}
+
 }  // namespace webrtc

api/video_codecs/video_encoder_software_fallback_wrapper.cc

@@ -15,7 +15,6 @@
 #include <cstdio>
 #include <memory>
 #include <string>
-#include <utility>
 #include <vector>
 
 #include "absl/types/optional.h"
@@ -25,6 +24,7 @@
 #include "api/video_codecs/video_codec.h"
 #include "api/video_codecs/video_encoder.h"
 #include "modules/video_coding/include/video_error_codes.h"
+#include "modules/video_coding/utility/simulcast_utility.h"
 #include "rtc_base/checks.h"
 #include "rtc_base/logging.h"
 #include "system_wrappers/include/field_trial.h"
@@ -33,52 +33,89 @@ namespace webrtc {
 
 namespace {
 
+// If forced fallback is allowed, either:
+//
+// 1) The forced fallback is requested if the resolution is less than or equal
+//    to |max_pixels_|. The resolution is allowed to be scaled down to
+//    |min_pixels_|.
+//
+// 2) The forced fallback is requested if temporal support is preferred and the
+//    SW fallback supports temporal layers while the HW encoder does not.
+
+struct ForcedFallbackParams {
+ public:
+  bool SupportsResolutionBasedSwitch(const VideoCodec& codec) const {
+    return enable_resolution_based_switch &&
+           codec.codecType == kVideoCodecVP8 &&
+           codec.numberOfSimulcastStreams <= 1 &&
+           codec.VP8().numberOfTemporalLayers == 1 &&
+           codec.width * codec.height <= max_pixels;
+  }
+
+  bool SupportsTemporalBasedSwitch(const VideoCodec& codec) const {
+    return enable_temporal_based_switch &&
+           SimulcastUtility::NumberOfTemporalLayers(codec, 0) > 1;
+  }
+
+  bool enable_temporal_based_switch = false;
+  bool enable_resolution_based_switch = false;
+  int min_pixels = 320 * 180;
+  int max_pixels = 320 * 240;
+};
+
 const char kVp8ForceFallbackEncoderFieldTrial[] =
     "WebRTC-VP8-Forced-Fallback-Encoder-v2";
 
-bool EnableForcedFallback() {
+absl::optional<ForcedFallbackParams> ParseFallbackParamsFromFieldTrials(
-  return field_trial::IsEnabled(kVp8ForceFallbackEncoderFieldTrial);
+    const VideoEncoder& main_encoder) {
-}
+  const std::string field_trial =
-
-bool IsForcedFallbackPossible(const VideoCodec& codec_settings) {
-  return codec_settings.codecType == kVideoCodecVP8 &&
-         codec_settings.numberOfSimulcastStreams <= 1 &&
-         codec_settings.VP8().numberOfTemporalLayers == 1;
-}
-
-void GetForcedFallbackParamsFromFieldTrialGroup(int* param_min_pixels,
-                                                int* param_max_pixels,
-                                                int minimum_max_pixels) {
-  RTC_DCHECK(param_min_pixels);
-  RTC_DCHECK(param_max_pixels);
-  std::string group =
       webrtc::field_trial::FindFullName(kVp8ForceFallbackEncoderFieldTrial);
-  if (group.empty())
+  if (field_trial.find("Enabled") != 0) {
-    return;
+    return absl::nullopt;
+  }
 
-  int min_pixels;
+  int max_pixels_lower_bound =
-  int max_pixels;
+      main_encoder.GetEncoderInfo().scaling_settings.min_pixels_per_frame - 1;
-  int min_bps;
+
-  if (sscanf(group.c_str(), "Enabled-%d,%d,%d", &min_pixels, &max_pixels,
+  ForcedFallbackParams params;
-             &min_bps) != 3) {
+  params.enable_resolution_based_switch = true;
+
+  int min_bps = 0;
+  if (sscanf(field_trial.c_str(), "Enabled-%d,%d,%d", &params.min_pixels,
+             &params.max_pixels, &min_bps) != 3) {
     RTC_LOG(LS_WARNING)
         << "Invalid number of forced fallback parameters provided.";
-    return;
+    return absl::nullopt;
-  }
+  } else if (params.min_pixels <= 0 ||
-  if (min_pixels <= 0 || max_pixels < minimum_max_pixels ||
+             params.max_pixels < max_pixels_lower_bound ||
-      max_pixels < min_pixels || min_bps <= 0) {
+             params.max_pixels < params.min_pixels || min_bps <= 0) {
     RTC_LOG(LS_WARNING) << "Invalid forced fallback parameter value provided.";
-    return;
+    return absl::nullopt;
   }
-  *param_min_pixels = min_pixels;
+
-  *param_max_pixels = max_pixels;
+  return params;
+}
+
+absl::optional<ForcedFallbackParams> GetForcedFallbackParams(
+    bool prefer_temporal_support,
+    const VideoEncoder& main_encoder) {
+  absl::optional<ForcedFallbackParams> params =
+      ParseFallbackParamsFromFieldTrials(main_encoder);
+  if (prefer_temporal_support) {
+    if (!params.has_value()) {
+      params.emplace();
+    }
+    params->enable_temporal_based_switch = prefer_temporal_support;
+  }
+  return params;
 }
 
 class VideoEncoderSoftwareFallbackWrapper final : public VideoEncoder {
  public:
   VideoEncoderSoftwareFallbackWrapper(
       std::unique_ptr<webrtc::VideoEncoder> sw_encoder,
-      std::unique_ptr<webrtc::VideoEncoder> hw_encoder);
+      std::unique_ptr<webrtc::VideoEncoder> hw_encoder,
+      bool prefer_temporal_support);
   ~VideoEncoderSoftwareFallbackWrapper() override;
 
   void SetFecControllerOverride(
@@ -106,28 +143,28 @@ class VideoEncoderSoftwareFallbackWrapper final : public VideoEncoder {
   EncoderInfo GetEncoderInfo() const override;
 
  private:
-  bool InitFallbackEncoder();
+  bool InitFallbackEncoder(bool is_forced);
-
-  // If |forced_fallback_possible_| is true:
-  // The forced fallback is requested if the resolution is less than or equal to
-  // |max_pixels_|. The resolution is allowed to be scaled down to
-  // |min_pixels_|.
-  class ForcedFallbackParams {
-   public:
-    bool IsValid(const VideoCodec& codec) const {
-      return codec.width * codec.height <= max_pixels_;
-    }
-
-    bool active_ = false;
-    int min_pixels_ = 320 * 180;
-    int max_pixels_ = 320 * 240;
-  };
-
   bool TryInitForcedFallbackEncoder();
-  bool TryReInitForcedFallbackEncoder();
+  bool IsFallbackActive() const;
-  void ValidateSettingsForForcedFallback();
+
-  bool IsForcedFallbackActive() const;
+  VideoEncoder* current_encoder() {
-  void MaybeModifyCodecForFallback();
+    switch (encoder_state_) {
+      case EncoderState::kUninitialized:
+        RTC_LOG(LS_WARNING)
+            << "Trying to access encoder in uninitialized fallback wrapper.";
+        // Return main encoder to preserve previous behavior.
+        ABSL_FALLTHROUGH_INTENDED;
+      case EncoderState::kMainEncoderUsed:
+        return encoder_.get();
+      case EncoderState::kFallbackDueToFailure:
+      case EncoderState::kForcedFallback:
+        return fallback_encoder_.get();
+    }
+  }
+
+  // Updates encoder with last observed parameters, such as callbacks, rates,
+  // etc.
+  void PrimeEncoder(VideoEncoder* encoder) const;
 
   // Settings used in the last InitEncode call and used if a dynamic fallback to
   // software is required.
@@ -137,65 +174,95 @@ class VideoEncoderSoftwareFallbackWrapper final : public VideoEncoder {
   // The last rate control settings, if set.
   absl::optional<RateControlParameters> rate_control_parameters_;
 
-  // The last channel parameters set, and a flag for noting they are set.
+  // The last channel parameters set.
-  bool channel_parameters_set_;
+  absl::optional<float> packet_loss_;
-  uint32_t packet_loss_;
+  absl::optional<int64_t> rtt_;
-  int64_t rtt_;
+  FecControllerOverride* fec_controller_override_;
+  absl::optional<LossNotification> loss_notification_;
 
-  bool use_fallback_encoder_;
+  enum class EncoderState {
+    kUninitialized,
+    kMainEncoderUsed,
+    kFallbackDueToFailure,
+    kForcedFallback
+  };
+
+  EncoderState encoder_state_;
   const std::unique_ptr<webrtc::VideoEncoder> encoder_;
-
   const std::unique_ptr<webrtc::VideoEncoder> fallback_encoder_;
+
   EncodedImageCallback* callback_;
 
-  bool forced_fallback_possible_;
+  const absl::optional<ForcedFallbackParams> fallback_params_;
-  ForcedFallbackParams forced_fallback_;
 };
 
 VideoEncoderSoftwareFallbackWrapper::VideoEncoderSoftwareFallbackWrapper(
     std::unique_ptr<webrtc::VideoEncoder> sw_encoder,
-    std::unique_ptr<webrtc::VideoEncoder> hw_encoder)
+    std::unique_ptr<webrtc::VideoEncoder> hw_encoder,
-    : channel_parameters_set_(false),
+    bool prefer_temporal_support)
-      packet_loss_(0),
+    : fec_controller_override_(nullptr),
-      rtt_(0),
+      encoder_state_(EncoderState::kUninitialized),
-      use_fallback_encoder_(false),
       encoder_(std::move(hw_encoder)),
       fallback_encoder_(std::move(sw_encoder)),
       callback_(nullptr),
-      forced_fallback_possible_(EnableForcedFallback()) {
+      fallback_params_(
+          GetForcedFallbackParams(prefer_temporal_support, *encoder_)) {
   RTC_DCHECK(fallback_encoder_);
-  if (forced_fallback_possible_) {
-    GetForcedFallbackParamsFromFieldTrialGroup(
-        &forced_fallback_.min_pixels_, &forced_fallback_.max_pixels_,
-        encoder_->GetEncoderInfo().scaling_settings.min_pixels_per_frame -
-            1);  // No HW below.
-  }
 }
+
 VideoEncoderSoftwareFallbackWrapper::~VideoEncoderSoftwareFallbackWrapper() =
     default;
 
-bool VideoEncoderSoftwareFallbackWrapper::InitFallbackEncoder() {
+void VideoEncoderSoftwareFallbackWrapper::PrimeEncoder(
+    VideoEncoder* encoder) const {
+  RTC_DCHECK(encoder);
+  // Replay callback, rates, and channel parameters.
+  if (callback_) {
+    encoder->RegisterEncodeCompleteCallback(callback_);
+  }
+  if (rate_control_parameters_) {
+    encoder->SetRates(*rate_control_parameters_);
+  }
+  if (rtt_.has_value()) {
+    encoder->OnRttUpdate(rtt_.value());
+  }
+  if (packet_loss_.has_value()) {
+    encoder->OnPacketLossRateUpdate(packet_loss_.value());
+  }
+  if (fec_controller_override_) {
+    encoder->SetFecControllerOverride(fec_controller_override_);
+  }
+  if (loss_notification_.has_value()) {
+    encoder->OnLossNotification(loss_notification_.value());
+  }
+}
+
+bool VideoEncoderSoftwareFallbackWrapper::InitFallbackEncoder(bool is_forced) {
   RTC_LOG(LS_WARNING) << "Encoder falling back to software encoding.";
 
   RTC_DCHECK(encoder_settings_.has_value());
   const int ret = fallback_encoder_->InitEncode(&codec_settings_,
                                                 encoder_settings_.value());
-  use_fallback_encoder_ = (ret == WEBRTC_VIDEO_CODEC_OK);
+
-  if (!use_fallback_encoder_) {
+  if (ret != WEBRTC_VIDEO_CODEC_OK) {
     RTC_LOG(LS_ERROR) << "Failed to initialize software-encoder fallback.";
     fallback_encoder_->Release();
     return false;
   }
-  // Replay callback, rates, and channel parameters.
-  if (callback_)
-    fallback_encoder_->RegisterEncodeCompleteCallback(callback_);
-  if (rate_control_parameters_)
-    fallback_encoder_->SetRates(*rate_control_parameters_);
 
-  // Since we're switching to the fallback encoder, Release the real encoder. It
+  if (encoder_state_ == EncoderState::kMainEncoderUsed) {
-  // may be re-initialized via InitEncode later, and it will continue to get
+    // Since we're switching to the fallback encoder, Release the real encoder.
-  // Set calls for rates and channel parameters in the meantime.
+    // It may be re-initialized via InitEncode later, and it will continue to
-  encoder_->Release();
+    // get Set calls for rates and channel parameters in the meantime.
+    encoder_->Release();
+  }
+
+  if (is_forced) {
+    encoder_state_ = EncoderState::kForcedFallback;
+  } else {
+    encoder_state_ = EncoderState::kFallbackDueToFailure;
+  }
+
   return true;
 }
 
@@ -204,8 +271,9 @@ void VideoEncoderSoftwareFallbackWrapper::SetFecControllerOverride(
   // It is important that only one of those would ever interact with the
   // |fec_controller_override| at a given time. This is the responsibility
   // of |this| to maintain.
-  encoder_->SetFecControllerOverride(fec_controller_override);
+
-  fallback_encoder_->SetFecControllerOverride(fec_controller_override);
+  fec_controller_override_ = fec_controller_override;
+  current_encoder()->SetFecControllerOverride(fec_controller_override);
 }
 
 int32_t VideoEncoderSoftwareFallbackWrapper::InitEncode(
@@ -217,93 +285,94 @@ int32_t VideoEncoderSoftwareFallbackWrapper::InitEncode(
   encoder_settings_ = settings;
   // Clear stored rate/channel parameters.
   rate_control_parameters_ = absl::nullopt;
-  ValidateSettingsForForcedFallback();
 
-  // Try to reinit forced software codec if it is in use.
+  RTC_DCHECK_EQ(encoder_state_, EncoderState::kUninitialized)
-  if (TryReInitForcedFallbackEncoder()) {
+      << "InitEncode() should never be called on an active instance!";
-    return WEBRTC_VIDEO_CODEC_OK;
+
-  }
   // Try to init forced software codec if it should be used.
   if (TryInitForcedFallbackEncoder()) {
+    PrimeEncoder(current_encoder());
     return WEBRTC_VIDEO_CODEC_OK;
   }
-  forced_fallback_.active_ = false;
 
   int32_t ret = encoder_->InitEncode(codec_settings, settings);
   if (ret == WEBRTC_VIDEO_CODEC_OK) {
-    if (use_fallback_encoder_) {
+    encoder_state_ = EncoderState::kMainEncoderUsed;
-      RTC_LOG(LS_WARNING)
+    PrimeEncoder(current_encoder());
-          << "InitEncode OK, no longer using the software fallback encoder.";
-      fallback_encoder_->Release();
-      use_fallback_encoder_ = false;
-    }
-    if (callback_)
-      encoder_->RegisterEncodeCompleteCallback(callback_);
     return ret;
   }
+
   // Try to instantiate software codec.
-  if (InitFallbackEncoder()) {
+  if (InitFallbackEncoder(/*is_forced=*/false)) {
+    PrimeEncoder(current_encoder());
     return WEBRTC_VIDEO_CODEC_OK;
   }
-  // Software encoder failed, use original return code.
+
+  // Software encoder failed too, use original return code.
+  encoder_state_ = EncoderState::kUninitialized;
   return ret;
 }
 
 int32_t VideoEncoderSoftwareFallbackWrapper::RegisterEncodeCompleteCallback(
     EncodedImageCallback* callback) {
   callback_ = callback;
-  int32_t ret = encoder_->RegisterEncodeCompleteCallback(callback);
+  return current_encoder()->RegisterEncodeCompleteCallback(callback);
-  if (use_fallback_encoder_)
-    return fallback_encoder_->RegisterEncodeCompleteCallback(callback);
-  return ret;
 }
 
 int32_t VideoEncoderSoftwareFallbackWrapper::Release() {
-  return use_fallback_encoder_ ? fallback_encoder_->Release()
+  if (encoder_state_ == EncoderState::kUninitialized) {
-                               : encoder_->Release();
+    return WEBRTC_VIDEO_CODEC_OK;
+  }
+  int32_t ret = current_encoder()->Release();
+  encoder_state_ = EncoderState::kUninitialized;
+  return ret;
 }
 
 int32_t VideoEncoderSoftwareFallbackWrapper::Encode(
     const VideoFrame& frame,
     const std::vector<VideoFrameType>* frame_types) {
-  if (use_fallback_encoder_)
+  switch (encoder_state_) {
-    return fallback_encoder_->Encode(frame, frame_types);
+    case EncoderState::kUninitialized:
-  int32_t ret = encoder_->Encode(frame, frame_types);
+      return WEBRTC_VIDEO_CODEC_ERROR;
-  // If requested, try a software fallback.
+    case EncoderState::kMainEncoderUsed: {
-  bool fallback_requested = (ret == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE);
+      int32_t ret = encoder_->Encode(frame, frame_types);
-  if (fallback_requested && InitFallbackEncoder()) {
+      // If requested, try a software fallback.
-    // Start using the fallback with this frame.
+      bool fallback_requested = (ret == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE);
-    return fallback_encoder_->Encode(frame, frame_types);
+      if (fallback_requested && InitFallbackEncoder(/*is_forced=*/false)) {
+        // Start using the fallback with this frame.
+        PrimeEncoder(current_encoder());
+        return fallback_encoder_->Encode(frame, frame_types);
+      }
+      // Fallback encoder failed too, return original error code.
+      return ret;
+    }
+    case EncoderState::kFallbackDueToFailure:
+    case EncoderState::kForcedFallback:
+      return fallback_encoder_->Encode(frame, frame_types);
   }
-  return ret;
 }
 
 void VideoEncoderSoftwareFallbackWrapper::SetRates(
     const RateControlParameters& parameters) {
   rate_control_parameters_ = parameters;
-  encoder_->SetRates(parameters);
+  return current_encoder()->SetRates(parameters);
-  if (use_fallback_encoder_)
-    fallback_encoder_->SetRates(parameters);
 }
 
 void VideoEncoderSoftwareFallbackWrapper::OnPacketLossRateUpdate(
     float packet_loss_rate) {
-  VideoEncoder* encoder =
+  packet_loss_ = packet_loss_rate;
-      use_fallback_encoder_ ? fallback_encoder_.get() : encoder_.get();
+  current_encoder()->OnPacketLossRateUpdate(packet_loss_rate);
-  encoder->OnPacketLossRateUpdate(packet_loss_rate);
 }
 
 void VideoEncoderSoftwareFallbackWrapper::OnRttUpdate(int64_t rtt_ms) {
-  VideoEncoder* encoder =
+  rtt_ = rtt_ms;
-      use_fallback_encoder_ ? fallback_encoder_.get() : encoder_.get();
+  current_encoder()->OnRttUpdate(rtt_ms);
-  encoder->OnRttUpdate(rtt_ms);
 }
 
 void VideoEncoderSoftwareFallbackWrapper::OnLossNotification(
     const LossNotification& loss_notification) {
-  VideoEncoder* encoder =
+  loss_notification_ = loss_notification;
-      use_fallback_encoder_ ? fallback_encoder_.get() : encoder_.get();
+  current_encoder()->OnLossNotification(loss_notification);
-  encoder->OnLossNotification(loss_notification);
 }
 
 VideoEncoder::EncoderInfo VideoEncoderSoftwareFallbackWrapper::GetEncoderInfo()
@@ -312,17 +381,17 @@ VideoEncoder::EncoderInfo VideoEncoderSoftwareFallbackWrapper::GetEncoderInfo()
   EncoderInfo default_encoder_info = encoder_->GetEncoderInfo();
 
   EncoderInfo info =
-      use_fallback_encoder_ ? fallback_encoder_info : default_encoder_info;
+      IsFallbackActive() ? fallback_encoder_info : default_encoder_info;
 
-  if (forced_fallback_possible_) {
+  if (fallback_params_.has_value()) {
-    const auto settings = forced_fallback_.active_
+    const auto settings = (encoder_state_ == EncoderState::kForcedFallback)
                               ? fallback_encoder_info.scaling_settings
                               : default_encoder_info.scaling_settings;
     info.scaling_settings =
         settings.thresholds
             ? VideoEncoder::ScalingSettings(settings.thresholds->low,
                                             settings.thresholds->high,
-                                            forced_fallback_.min_pixels_)
+                                            fallback_params_->min_pixels)
             : VideoEncoder::ScalingSettings::kOff;
   } else {
     info.scaling_settings = default_encoder_info.scaling_settings;
@@ -331,72 +400,82 @@ VideoEncoder::EncoderInfo VideoEncoderSoftwareFallbackWrapper::GetEncoderInfo()
   return info;
 }
 
-bool VideoEncoderSoftwareFallbackWrapper::IsForcedFallbackActive() const {
+bool VideoEncoderSoftwareFallbackWrapper::IsFallbackActive() const {
-  return (forced_fallback_possible_ && use_fallback_encoder_ &&
+  return encoder_state_ == EncoderState::kForcedFallback ||
-          forced_fallback_.active_);
+         encoder_state_ == EncoderState::kFallbackDueToFailure;
 }
 
 bool VideoEncoderSoftwareFallbackWrapper::TryInitForcedFallbackEncoder() {
-  if (!forced_fallback_possible_ || use_fallback_encoder_) {
+  if (!fallback_params_) {
-    return false;
-  }
-  // Fallback not active.
-  if (!forced_fallback_.IsValid(codec_settings_)) {
-    return false;
-  }
-  // Settings valid, try to instantiate software codec.
-  RTC_LOG(LS_INFO) << "Request forced SW encoder fallback: "
-                   << codec_settings_.width << "x" << codec_settings_.height;
-  if (!InitFallbackEncoder()) {
-    return false;
-  }
-  forced_fallback_.active_ = true;
-  return true;
-}
-
-bool VideoEncoderSoftwareFallbackWrapper::TryReInitForcedFallbackEncoder() {
-  if (!IsForcedFallbackActive()) {
     return false;
   }
 
-  // Forced fallback active.
+  RTC_DCHECK_EQ(encoder_state_, EncoderState::kUninitialized);
-  if (!forced_fallback_.IsValid(codec_settings_)) {
+
-    RTC_LOG(LS_INFO) << "Stop forced SW encoder fallback, max pixels exceeded.";
+  if (fallback_params_->SupportsResolutionBasedSwitch(codec_settings_)) {
-    return false;
+    // Settings valid, try to instantiate software codec.
+    RTC_LOG(LS_INFO) << "Request forced SW encoder fallback: "
+                     << codec_settings_.width << "x" << codec_settings_.height;
+    return InitFallbackEncoder(/*is_forced=*/true);
   }
 
-  // Settings valid, reinitialize the forced fallback encoder.
+  if (fallback_params_->SupportsTemporalBasedSwitch(codec_settings_)) {
-  RTC_DCHECK(encoder_settings_.has_value());
+    // First init main encoder to see if that supports temporal layers.
-  if (fallback_encoder_->InitEncode(&codec_settings_,
+    if (encoder_->InitEncode(&codec_settings_, encoder_settings_.value()) ==
-                                    encoder_settings_.value()) !=
+        WEBRTC_VIDEO_CODEC_OK) {
-      WEBRTC_VIDEO_CODEC_OK) {
+      encoder_state_ = EncoderState::kMainEncoderUsed;
-    RTC_LOG(LS_ERROR) << "Failed to init forced SW encoder fallback.";
+    }
-    return false;
+
-  }
+    if (encoder_state_ == EncoderState::kMainEncoderUsed &&
-  return true;
+        encoder_->GetEncoderInfo().fps_allocation[0].size() > 1) {
-}
+      // Primary encoder already supports temporal layers, use that instead.
-
+      return true;
-void VideoEncoderSoftwareFallbackWrapper::ValidateSettingsForForcedFallback() {
+    }
-  if (!forced_fallback_possible_)
+
-    return;
+    // Try to initialize fallback and check if it supports temporal layers.
-
+    if (fallback_encoder_->InitEncode(&codec_settings_,
-  if (!IsForcedFallbackPossible(codec_settings_)) {
+                                      encoder_settings_.value()) ==
-    if (IsForcedFallbackActive()) {
+        WEBRTC_VIDEO_CODEC_OK) {
-      fallback_encoder_->Release();
+      if (fallback_encoder_->GetEncoderInfo().fps_allocation[0].size() > 1) {
-      use_fallback_encoder_ = false;
+        // Fallback encoder available and supports temporal layers, use it!
+        if (encoder_state_ == EncoderState::kMainEncoderUsed) {
+          // Main encoder initialized but does not support temporal layers,
+          // release it again.
+          encoder_->Release();
+        }
+        encoder_state_ = EncoderState::kForcedFallback;
+        RTC_LOG(LS_INFO)
+            << "Forced switch to SW encoder due to temporal support.";
+        return true;
+      } else {
+        // Fallback encoder intialization succeeded, but it does not support
+        // temporal layers either - release it.
+        fallback_encoder_->Release();
+      }
+    }
+
+    if (encoder_state_ == EncoderState::kMainEncoderUsed) {
+      // Main encoder already initialized - make use of it.
+      RTC_LOG(LS_INFO)
+          << "Cannot fall back for temporal support since fallback that "
+             "supports is not available. Using main encoder instead.";
+      return true;
     }
-    RTC_LOG(LS_INFO) << "Disable forced_fallback_possible_ due to settings.";
-    forced_fallback_possible_ = false;
   }
+
+  // Neither forced fallback mode supported.
+  return false;
 }
 
 }  // namespace
 
 std::unique_ptr<VideoEncoder> CreateVideoEncoderSoftwareFallbackWrapper(
     std::unique_ptr<VideoEncoder> sw_fallback_encoder,
-    std::unique_ptr<VideoEncoder> hw_encoder) {
+    std::unique_ptr<VideoEncoder> hw_encoder,
+    bool prefer_temporal_support) {
   return std::make_unique<VideoEncoderSoftwareFallbackWrapper>(
-      std::move(sw_fallback_encoder), std::move(hw_encoder));
+      std::move(sw_fallback_encoder), std::move(hw_encoder),
+      prefer_temporal_support);
 }
 
 }  // namespace webrtc

api/video_codecs/video_encoder_software_fallback_wrapper.h

@@ -12,6 +12,7 @@
 #define API_VIDEO_CODECS_VIDEO_ENCODER_SOFTWARE_FALLBACK_WRAPPER_H_
 
 #include <memory>
+#include <utility>
 
 #include "api/video_codecs/video_encoder.h"
 #include "rtc_base/system/rtc_export.h"
@@ -21,10 +22,25 @@ namespace webrtc {
 // Used to wrap external VideoEncoders to provide a fallback option on
 // software encoding when a hardware encoder fails to encode a stream due to
 // hardware restrictions, such as max resolution.
+// |bool prefer_temporal_support| indicates that if the software fallback
+// encoder supports temporal layers but the hardware encoder does not, a
+// fallback should be forced even if the encoder otherwise works.
 RTC_EXPORT std::unique_ptr<VideoEncoder>
 CreateVideoEncoderSoftwareFallbackWrapper(
     std::unique_ptr<VideoEncoder> sw_fallback_encoder,
-    std::unique_ptr<VideoEncoder> hw_encoder);
+    std::unique_ptr<VideoEncoder> hw_encoder,
+    bool prefer_temporal_support);
+
+// Default fallback for call-sites not yet updated with
+// |prefer_temporal_support|.
+// TODO(sprang): Remove when usage is gone.
+RTC_EXPORT inline std::unique_ptr<VideoEncoder>
+CreateVideoEncoderSoftwareFallbackWrapper(
+    std::unique_ptr<VideoEncoder> sw_fallback_encoder,
+    std::unique_ptr<VideoEncoder> hw_encoder) {
+  return CreateVideoEncoderSoftwareFallbackWrapper(
+      std::move(sw_fallback_encoder), std::move(hw_encoder), false);
+}
 
 }  // namespace webrtc
 

media/engine/simulcast_encoder_adapter.cc

@@ -150,7 +150,9 @@ SimulcastEncoderAdapter::SimulcastEncoderAdapter(
       encoded_complete_callback_(nullptr),
       experimental_boosted_screenshare_qp_(GetScreenshareBoostedQpValue()),
       boost_base_layer_quality_(RateControlSettings::ParseFromFieldTrials()
-                                    .Vp8BoostBaseLayerQuality()) {
+                                    .Vp8BoostBaseLayerQuality()),
+      prefer_temporal_support_on_base_layer_(field_trial::IsEnabled(
+          "WebRTC-Video-PreferTemporalSupportOnBaseLayer")) {
   RTC_DCHECK(primary_factory);
 
   // The adapter is typically created on the worker thread, but operated on
@@ -259,7 +261,9 @@ int SimulcastEncoderAdapter::InitEncode(
       if (fallback_encoder_factory_ != nullptr) {
         encoder = CreateVideoEncoderSoftwareFallbackWrapper(
             fallback_encoder_factory_->CreateVideoEncoder(format),
-            std::move(encoder));
+            std::move(encoder),
+            i == lowest_resolution_stream_index &&
+                prefer_temporal_support_on_base_layer_);
       }
     }
 

media/engine/simulcast_encoder_adapter.h

@@ -133,6 +133,7 @@ class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder {
 
   const absl::optional<unsigned int> experimental_boosted_screenshare_qp_;
   const bool boost_base_layer_quality_;
+  const bool prefer_temporal_support_on_base_layer_;
 };
 
 }  // namespace webrtc