Add ability to specify if rate controller of video encoder is trusted.

If rate controller is trusted, we disable the frame dropper in the
media optimization module.

This is a re-land of
https://webrtc-review.googlesource.com/c/src/+/105020

Bug: webrtc:9890
Change-Id: I418e47a43a1a98cb2fd5295c03360b954f2288f2
Reviewed-on: https://webrtc-review.googlesource.com/c/109141
Commit-Queue: Erik Språng <sprang@webrtc.org>
Reviewed-by: Niels Moller <nisse@webrtc.org>
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#25570}

api/video_codecs/test/BUILD.gn

@@ -21,6 +21,7 @@ if (rtc_include_tests) {
       "..:builtin_video_encoder_factory",
       "..:rtc_software_fallback_wrappers",
       "..:video_codecs_api",
+      "../..:mock_video_encoder",
       "../../../modules/video_coding:video_codec_interface",
       "../../../modules/video_coding:video_coding_utility",
       "../../../modules/video_coding:webrtc_vp8",

api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc

@@ -10,6 +10,7 @@
 
 #include <utility>
 
+#include "api/test/mock_video_encoder.h"
 #include "api/video/i420_buffer.h"
 #include "api/video/video_bitrate_allocation.h"
 #include "api/video_codecs/video_encoder_software_fallback_wrapper.h"
@@ -21,9 +22,12 @@
 #include "rtc_base/checks.h"
 #include "rtc_base/fakeclock.h"
 #include "test/field_trial.h"
+#include "test/gmock.h"
 #include "test/gtest.h"
 
 namespace webrtc {
+using ::testing::Return;
+
 namespace {
 const int kWidth = 320;
 const int kHeight = 240;
@@ -33,6 +37,12 @@ const size_t kMaxPayloadSize = 800;
 const int kDefaultMinPixelsPerFrame = 320 * 180;
 const int kLowThreshold = 10;
 const int kHighThreshold = 20;
+
+VideoEncoder::EncoderInfo GetEncoderInfo(bool trusted_rate_controller) {
+  VideoEncoder::EncoderInfo info;
+  info.has_trusted_rate_controller = trusted_rate_controller;
+  return info;
+}
 }  // namespace
 
 class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test {
@@ -512,4 +522,73 @@ TEST_F(ForcedFallbackTestEnabled, ScalingDisabledIfResizeOff) {
   EXPECT_FALSE(settings.thresholds.has_value());
 }
 
+TEST(SoftwareFallbackEncoderTest, BothRateControllersNotTrusted) {
+  auto* sw_encoder = new testing::NiceMock<MockVideoEncoder>();
+  auto* hw_encoder = new testing::NiceMock<MockVideoEncoder>();
+
+  EXPECT_CALL(*sw_encoder, GetEncoderInfo())
+      .WillRepeatedly(Return(GetEncoderInfo(false)));
+  EXPECT_CALL(*hw_encoder, GetEncoderInfo())
+      .WillRepeatedly(Return(GetEncoderInfo(false)));
+
+  std::unique_ptr<VideoEncoder> wrapper =
+      CreateVideoEncoderSoftwareFallbackWrapper(
+          std::unique_ptr<VideoEncoder>(sw_encoder),
+          std::unique_ptr<VideoEncoder>(hw_encoder));
+  EXPECT_FALSE(wrapper->GetEncoderInfo().has_trusted_rate_controller);
+}
+
+TEST(SoftwareFallbackEncoderTest, SwRateControllerTrusted) {
+  auto* sw_encoder = new testing::NiceMock<MockVideoEncoder>();
+  auto* hw_encoder = new testing::NiceMock<MockVideoEncoder>();
+  EXPECT_CALL(*sw_encoder, GetEncoderInfo())
+      .WillRepeatedly(Return(GetEncoderInfo(true)));
+  EXPECT_CALL(*hw_encoder, GetEncoderInfo())
+      .WillRepeatedly(Return(GetEncoderInfo(false)));
+
+  std::unique_ptr<VideoEncoder> wrapper =
+      CreateVideoEncoderSoftwareFallbackWrapper(
+          std::unique_ptr<VideoEncoder>(sw_encoder),
+          std::unique_ptr<VideoEncoder>(hw_encoder));
+  EXPECT_FALSE(wrapper->GetEncoderInfo().has_trusted_rate_controller);
+}
+
+TEST(SoftwareFallbackEncoderTest, HwRateControllerTrusted) {
+  auto* sw_encoder = new testing::NiceMock<MockVideoEncoder>();
+  auto* hw_encoder = new testing::NiceMock<MockVideoEncoder>();
+  EXPECT_CALL(*sw_encoder, GetEncoderInfo())
+      .WillRepeatedly(Return(GetEncoderInfo(false)));
+  EXPECT_CALL(*hw_encoder, GetEncoderInfo())
+      .WillRepeatedly(Return(GetEncoderInfo(true)));
+
+  std::unique_ptr<VideoEncoder> wrapper =
+      CreateVideoEncoderSoftwareFallbackWrapper(
+          std::unique_ptr<VideoEncoder>(sw_encoder),
+          std::unique_ptr<VideoEncoder>(hw_encoder));
+  EXPECT_TRUE(wrapper->GetEncoderInfo().has_trusted_rate_controller);
+
+  // Trigger fallback to software.
+  EXPECT_CALL(*hw_encoder, Encode)
+      .WillOnce(Return(WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE));
+  VideoFrame frame = VideoFrame::Builder().build();
+  wrapper->Encode(frame, nullptr, nullptr);
+
+  EXPECT_FALSE(wrapper->GetEncoderInfo().has_trusted_rate_controller);
+}
+
+TEST(SoftwareFallbackEncoderTest, BothRateControllersTrusted) {
+  auto* sw_encoder = new testing::NiceMock<MockVideoEncoder>();
+  auto* hw_encoder = new testing::NiceMock<MockVideoEncoder>();
+  EXPECT_CALL(*sw_encoder, GetEncoderInfo())
+      .WillRepeatedly(Return(GetEncoderInfo(true)));
+  EXPECT_CALL(*hw_encoder, GetEncoderInfo())
+      .WillRepeatedly(Return(GetEncoderInfo(true)));
+
+  std::unique_ptr<VideoEncoder> wrapper =
+      CreateVideoEncoderSoftwareFallbackWrapper(
+          std::unique_ptr<VideoEncoder>(sw_encoder),
+          std::unique_ptr<VideoEncoder>(hw_encoder));
+  EXPECT_TRUE(wrapper->GetEncoderInfo().has_trusted_rate_controller);
+}
+
 }  // namespace webrtc

api/video_codecs/video_encoder.cc

@@ -86,7 +86,8 @@ constexpr VideoEncoder::ScalingSettings::KOff
 VideoEncoder::EncoderInfo::EncoderInfo()
     : scaling_settings(VideoEncoder::ScalingSettings::kOff),
       supports_native_handle(false),
-      implementation_name("unknown") {}
+      implementation_name("unknown"),
+      has_trusted_rate_controller(false) {}
 
 VideoEncoder::EncoderInfo::~EncoderInfo() = default;
 

api/video_codecs/video_encoder.h

@@ -132,6 +132,19 @@ class RTC_EXPORT VideoEncoder {
 
     // The name of this particular encoder implementation, e.g. "libvpx".
     std::string implementation_name;
+
+    // If this field is true, the encoder rate controller must perform
+    // well even in difficult situations, and produce close to the specified
+    // target bitrate seen over a reasonable time window, drop frames if
+    // necessary in order to keep the rate correct, and react quickly to
+    // changing bitrate targets. If this method returns true, we disable the
+    // frame dropper in the media optimization module and rely entirely on the
+    // encoder to produce media at a bitrate that closely matches the target.
+    // Any overshooting may result in delay buildup. If this method returns
+    // false (default behavior), the media opt frame dropper will drop input
+    // frames if it suspect encoder misbehavior. Misbehavior is common,
+    // especially in hardware codecs. Disable media opt at your own risk.
+    bool has_trusted_rate_controller;
   };
 
   static VideoCodecVP8 GetDefaultVp8Settings();
@@ -220,6 +233,10 @@ class RTC_EXPORT VideoEncoder {
   virtual bool SupportsNativeHandle() const;
   virtual const char* ImplementationName() const;
 
+  // Returns meta-data about the encoder, such as implementation name.
+  // The output of this method may change during runtime. For instance if a
+  // hardware encoder fails, it may fall back to doing software encoding using
+  // an implementation with different characteristics.
   virtual EncoderInfo GetEncoderInfo() const;
 };
 }  // namespace webrtc