[Overuse] Setting encoder configurations through the interface.

This squashes together several input signals that were spread out
through several calls into a single method and calling place:
SetEncoderSettings(), invoked from ReconfigureEncoder(). This is added
to the abstract interface.

This makes the following methods obsolete which are removed:
- SetEncoder(): The VideoEncoder was only used for GetEncoderInfo();
  the VideoEncoder::EncoderInfo is now part of the EncoderSettings.
- SetEncoderConfig(): The VideoEncoderConfig is part of
  EncoderSettings. The config is used for its codec_type and
  content_type enums.
- SetCodecMaxFrameRate(): The max frame rate was the same as
  VideoCodec::maxFramerate. VideoCodec is now part of EncoderSettings.

There may be some overlap in information between EncoderConfig and
VideoCodec, but that is outside the scope of this CL, which only makes
sure to bundle encoder settings-like information into one input signal.

Bug: webrtc:11222
Change-Id: I67c49c49c0a859cb7d5051939a461593c695a789
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/166602
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Evan Shrubsole <eshr@google.com>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Henrik Boström <hbos@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30332}

api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc

@@ -48,7 +48,6 @@ const int kHeight = 240;
 const int kNumCores = 2;
 const uint32_t kFramerate = 30;
 const size_t kMaxPayloadSize = 800;
-const int kDefaultMinPixelsPerFrame = 320 * 180;
 const int kLowThreshold = 10;
 const int kHighThreshold = 20;
 

api/video_codecs/video_encoder.h

@@ -34,6 +34,8 @@ class RTPFragmentationHeader;
 // TODO(pbos): Expose these through a public (root) header or change these APIs.
 struct CodecSpecificInfo;
 
+constexpr int kDefaultMinPixelsPerFrame = 320 * 180;
+
 class EncodedImageCallback {
  public:
   virtual ~EncodedImageCallback() {}
@@ -115,7 +117,7 @@ class RTC_EXPORT VideoEncoder {
     // TODO(kthelgason): Lower this limit when better testing
     // on MediaCodec and fallback implementations are in place.
     // See https://bugs.chromium.org/p/webrtc/issues/detail?id=7206
-    int min_pixels_per_frame = 320 * 180;
+    int min_pixels_per_frame = kDefaultMinPixelsPerFrame;
 
    private:
     // Private constructor; to get an object without thresholds, use

call/adaptation/BUILD.gn

@@ -25,6 +25,7 @@ rtc_library("resource_adaptation") {
   ]
   deps = [
     "../../api:rtp_parameters",
+    "../../api/video_codecs:video_codecs_api",
     "../../rtc_base:checks",
     "../../rtc_base:rtc_base_approved",
     "//third_party/abseil-cpp/absl/types:optional",

call/adaptation/resource_adaptation_module_interface.cc

@@ -10,8 +10,29 @@
 
 #include "call/adaptation/resource_adaptation_module_interface.h"
 
+#include <utility>
+
 namespace webrtc {
 
+EncoderSettings::EncoderSettings(VideoEncoder::EncoderInfo encoder_info,
+                                 VideoEncoderConfig encoder_config,
+                                 VideoCodec video_codec)
+    : encoder_info_(std::move(encoder_info)),
+      encoder_config_(std::move(encoder_config)),
+      video_codec_(std::move(video_codec)) {}
+
+const VideoEncoder::EncoderInfo& EncoderSettings::encoder_info() const {
+  return encoder_info_;
+}
+
+const VideoEncoderConfig& EncoderSettings::encoder_config() const {
+  return encoder_config_;
+}
+
+const VideoCodec& EncoderSettings::video_codec() const {
+  return video_codec_;
+}
+
 ResourceAdaptationModuleListener::~ResourceAdaptationModuleListener() {}
 
 ResourceAdaptationModuleInterface::~ResourceAdaptationModuleInterface() {}

call/adaptation/resource_adaptation_module_interface.h

@@ -12,10 +12,32 @@
 #define CALL_ADAPTATION_RESOURCE_ADAPTATION_MODULE_INTERFACE_H_
 
 #include "api/rtp_parameters.h"
+#include "api/video_codecs/video_encoder.h"
+#include "api/video_codecs/video_encoder_config.h"
 #include "call/adaptation/video_source_restrictions.h"
 
 namespace webrtc {
 
+// Information about an encoder available when reconfiguring the encoder.
+class EncoderSettings {
+ public:
+  EncoderSettings(VideoEncoder::EncoderInfo encoder_info,
+                  VideoEncoderConfig encoder_config,
+                  VideoCodec video_codec);
+
+  // Encoder capabilities, implementation info, etc.
+  const VideoEncoder::EncoderInfo& encoder_info() const;
+  // Configuration parameters, ultimately coming from the API and negotiation.
+  const VideoEncoderConfig& encoder_config() const;
+  // Lower level config, heavily based on the VideoEncoderConfig.
+  const VideoCodec& video_codec() const;
+
+ private:
+  VideoEncoder::EncoderInfo encoder_info_;
+  VideoEncoderConfig encoder_config_;
+  VideoCodec video_codec_;
+};
+
 // The listener is responsible for carrying out the reconfiguration of the video
 // source such that the VideoSourceRestrictions are fulfilled.
 class ResourceAdaptationModuleListener {
@@ -61,6 +83,7 @@ class ResourceAdaptationModuleInterface {
   virtual void SetHasInputVideo(bool has_input_video) = 0;
   virtual void SetDegradationPreference(
       DegradationPreference degradation_preference) = 0;
+  virtual void SetEncoderSettings(EncoderSettings encoder_settings) = 0;
   // Removes all restrictions; the module will need to adapt all over again.
   // TODO(hbos): It's not clear why anybody should be able to tell the module to
   // reset like this; can we get rid of this method?

modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc

@@ -50,7 +50,6 @@ constexpr int64_t kTestNtpTimeMs = 456;
 constexpr int64_t kInitialTimestampMs = 789;
 constexpr int kNumCores = 1;
 constexpr size_t kMaxPayloadSize = 1440;
-constexpr int kDefaultMinPixelsPerFrame = 320 * 180;
 constexpr int kWidth = 172;
 constexpr int kHeight = 144;
 constexpr float kFramerateFps = 30;

video/overuse_frame_detector_resource_adaptation_module.cc

@@ -356,12 +356,10 @@ OveruseFrameDetectorResourceAdaptationModule::
       source_restrictor_(std::make_unique<VideoSourceRestrictor>()),
       overuse_detector_(std::move(overuse_detector)),
       overuse_detector_is_started_(false),
-      codec_max_frame_rate_(absl::nullopt),
       target_frame_rate_(absl::nullopt),
       encoder_start_bitrate_bps_(0),
       is_quality_scaler_enabled_(false),
-      encoder_config_(),
+      encoder_settings_(absl::nullopt),
-      encoder_(nullptr),
       encoder_stats_observer_(encoder_stats_observer) {
   RTC_DCHECK(adaptation_listener_);
   RTC_DCHECK(video_stream_encoder_);
@@ -372,14 +370,9 @@ OveruseFrameDetectorResourceAdaptationModule::
 OveruseFrameDetectorResourceAdaptationModule::
     ~OveruseFrameDetectorResourceAdaptationModule() {}
 
-void OveruseFrameDetectorResourceAdaptationModule::SetEncoder(
-    VideoEncoder* encoder) {
-  encoder_ = encoder;
-}
-
 void OveruseFrameDetectorResourceAdaptationModule::StartResourceAdaptation(
     ResourceAdaptationModuleListener* adaptation_listener) {
-  RTC_DCHECK(encoder_);
+  RTC_DCHECK(encoder_settings_.has_value());
   RTC_DCHECK(!overuse_detector_is_started_);
   // TODO(hbos): When AdaptUp() and AdaptDown() are no longer invoked outside
   // the interval between StartCheckForOveruse() and StopCheckForOveruse(),
@@ -427,6 +420,12 @@ void OveruseFrameDetectorResourceAdaptationModule::SetDegradationPreference(
   MaybeUpdateVideoSourceRestrictions();
 }
 
+void OveruseFrameDetectorResourceAdaptationModule::SetEncoderSettings(
+    EncoderSettings encoder_settings) {
+  encoder_settings_ = std::move(encoder_settings);
+  MaybeUpdateTargetFrameRate();
+}
+
 void OveruseFrameDetectorResourceAdaptationModule::
     ResetVideoSourceRestrictions() {
   last_adaptation_request_.reset();
@@ -473,19 +472,6 @@ void OveruseFrameDetectorResourceAdaptationModule::SetLastFramePixelCount(
   last_frame_pixel_count_ = last_frame_pixel_count;
 }
 
-void OveruseFrameDetectorResourceAdaptationModule::SetEncoderConfig(
-    VideoEncoderConfig encoder_config) {
-  encoder_config_ = std::move(encoder_config);
-}
-
-void OveruseFrameDetectorResourceAdaptationModule::SetCodecMaxFrameRate(
-    absl::optional<double> codec_max_frame_rate) {
-  RTC_DCHECK(!codec_max_frame_rate.has_value() ||
-             codec_max_frame_rate.value() > 0.0);
-  codec_max_frame_rate_ = codec_max_frame_rate;
-  MaybeUpdateTargetFrameRate();
-}
-
 void OveruseFrameDetectorResourceAdaptationModule::SetEncoderStartBitrateBps(
     uint32_t encoder_start_bitrate_bps) {
   encoder_start_bitrate_bps_ = encoder_start_bitrate_bps;
@@ -528,13 +514,13 @@ void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) {
     case DegradationPreference::BALANCED: {
       // Check if quality should be increased based on bitrate.
       if (reason == kQuality &&
-          !balanced_settings_.CanAdaptUp(encoder_config_.codec_type,
+          !balanced_settings_.CanAdaptUp(GetVideoCodecTypeOrGeneric(),
                                          *last_frame_pixel_count_,
                                          encoder_start_bitrate_bps_)) {
         return;
       }
       // Try scale up framerate, if higher.
-      int fps = balanced_settings_.MaxFps(encoder_config_.codec_type,
+      int fps = balanced_settings_.MaxFps(GetVideoCodecTypeOrGeneric(),
                                           *last_frame_pixel_count_);
       if (source_restrictor_->IncreaseFramerate(fps)) {
         GetAdaptCounter().DecrementFramerate(reason, fps);
@@ -550,7 +536,7 @@ void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) {
       // Check if resolution should be increased based on bitrate.
       if (reason == kQuality &&
           !balanced_settings_.CanAdaptUpResolution(
-              encoder_config_.codec_type, *last_frame_pixel_count_,
+              GetVideoCodecTypeOrGeneric(), *last_frame_pixel_count_,
               encoder_start_bitrate_bps_)) {
         return;
       }
@@ -654,7 +640,7 @@ bool OveruseFrameDetectorResourceAdaptationModule::AdaptDown(
   switch (EffectiveDegradataionPreference()) {
     case DegradationPreference::BALANCED: {
       // Try scale down framerate, if lower.
-      int fps = balanced_settings_.MinFps(encoder_config_.codec_type,
+      int fps = balanced_settings_.MinFps(GetVideoCodecTypeOrGeneric(),
                                           *last_frame_pixel_count_);
       if (source_restrictor_->RestrictFramerate(fps)) {
         GetAdaptCounter().IncrementFramerate(reason);
@@ -677,7 +663,10 @@ bool OveruseFrameDetectorResourceAdaptationModule::AdaptDown(
       bool min_pixels_reached = false;
       if (!source_restrictor_->RequestResolutionLowerThan(
               adaptation_request.input_pixel_count_,
-              encoder_->GetEncoderInfo().scaling_settings.min_pixels_per_frame,
+              encoder_settings_.has_value()
+                  ? encoder_settings_->encoder_info()
+                        .scaling_settings.min_pixels_per_frame
+                  : kDefaultMinPixelsPerFrame,
               &min_pixels_reached)) {
         if (min_pixels_reached)
           encoder_stats_observer_->OnMinPixelLimitReached();
@@ -712,6 +701,14 @@ bool OveruseFrameDetectorResourceAdaptationModule::AdaptDown(
   return did_adapt;
 }
 
+VideoCodecType
+OveruseFrameDetectorResourceAdaptationModule::GetVideoCodecTypeOrGeneric()
+    const {
+  return encoder_settings_.has_value()
+             ? encoder_settings_->encoder_config().codec_type
+             : kVideoCodecGeneric;
+}
+
 void OveruseFrameDetectorResourceAdaptationModule::
     MaybeUpdateVideoSourceRestrictions() {
   VideoSourceRestrictions new_restrictions = ApplyDegradationPreference(
@@ -726,6 +723,11 @@ void OveruseFrameDetectorResourceAdaptationModule::
 
 void OveruseFrameDetectorResourceAdaptationModule::
     MaybeUpdateTargetFrameRate() {
+  absl::optional<double> codec_max_frame_rate =
+      encoder_settings_.has_value()
+          ? absl::optional<double>(
+                encoder_settings_->video_codec().maxFramerate)
+          : absl::nullopt;
   // The current target framerate is the maximum frame rate as specified by
   // the current codec configuration or any limit imposed by the adaptation
   // module. This is used to make sure overuse detection doesn't needlessly
@@ -735,9 +737,9 @@ void OveruseFrameDetectorResourceAdaptationModule::
                                  degradation_preference_)
           .max_frame_rate();
   if (!target_frame_rate.has_value() ||
-      (codec_max_frame_rate_.has_value() &&
+      (codec_max_frame_rate.has_value() &&
-       codec_max_frame_rate_.value() < target_frame_rate.value())) {
+       codec_max_frame_rate.value() < target_frame_rate.value())) {
-    target_frame_rate = codec_max_frame_rate_;
+    target_frame_rate = codec_max_frame_rate;
   }
   if (target_frame_rate != target_frame_rate_) {
     target_frame_rate_ = target_frame_rate;
@@ -799,7 +801,8 @@ DegradationPreference OveruseFrameDetectorResourceAdaptationModule::
   // Resolution is capped for fullscreen animated content.
   // Adapatation is done only via framerate downgrade.
   // Thus effective degradation preference is MAINTAIN_RESOLUTION.
-  return (encoder_config_.content_type ==
+  return (encoder_settings_.has_value() &&
+          encoder_settings_->encoder_config().content_type ==
               VideoEncoderConfig::ContentType::kScreen &&
           degradation_preference_ == DegradationPreference::BALANCED)
              ? DegradationPreference::MAINTAIN_RESOLUTION
@@ -819,7 +822,7 @@ OveruseFrameDetectorResourceAdaptationModule::GetConstAdaptCounter() {
 absl::optional<VideoEncoder::QpThresholds>
 OveruseFrameDetectorResourceAdaptationModule::GetQpThresholds() const {
   RTC_DCHECK(last_frame_pixel_count_.has_value());
-  return balanced_settings_.GetQpThresholds(encoder_config_.codec_type,
+  return balanced_settings_.GetQpThresholds(GetVideoCodecTypeOrGeneric(),
                                             last_frame_pixel_count_.value());
 }
 
@@ -827,9 +830,11 @@ bool OveruseFrameDetectorResourceAdaptationModule::CanAdaptUpResolution(
     int pixels,
     uint32_t bitrate_bps) const {
   absl::optional<VideoEncoder::ResolutionBitrateLimits> bitrate_limits =
-      GetEncoderBitrateLimits(
+      encoder_settings_.has_value()
-          encoder_->GetEncoderInfo(),
+          ? GetEncoderBitrateLimits(
-          source_restrictor_->GetHigherResolutionThan(pixels));
+                encoder_settings_->encoder_info(),
+                source_restrictor_->GetHigherResolutionThan(pixels))
+          : absl::nullopt;
   if (!bitrate_limits.has_value() || bitrate_bps == 0) {
     return true;  // No limit configured or bitrate provided.
   }

video/overuse_frame_detector_resource_adaptation_module.h

@@ -22,6 +22,7 @@
 #include "api/video/video_frame.h"
 #include "api/video/video_source_interface.h"
 #include "api/video/video_stream_encoder_observer.h"
+#include "api/video_codecs/video_codec.h"
 #include "api/video_codecs/video_encoder.h"
 #include "api/video_codecs/video_encoder_config.h"
 #include "call/adaptation/resource_adaptation_module_interface.h"
@@ -60,12 +61,6 @@ class OveruseFrameDetectorResourceAdaptationModule
       ResourceAdaptationModuleListener* adaptation_listener);
   ~OveruseFrameDetectorResourceAdaptationModule() override;
 
-  // Sets the encoder to reconfigure based on overuse.
-  // TODO(hbos): Don't reconfigure the encoder directly. Instead, define the
-  // output of a resource adaptation module as a struct and let the
-  // VideoStreamEncoder handle the interaction with the actual encoder.
-  void SetEncoder(VideoEncoder* encoder);
-
   DegradationPreference degradation_preference() const {
     return degradation_preference_;
   }
@@ -77,6 +72,7 @@ class OveruseFrameDetectorResourceAdaptationModule
   void SetHasInputVideo(bool has_input_video) override;
   void SetDegradationPreference(
       DegradationPreference degradation_preference) override;
+  void SetEncoderSettings(EncoderSettings encoder_settings) override;
   void ResetVideoSourceRestrictions() override;
 
   // Input to the OveruseFrameDetector, which are required for this module to
@@ -96,8 +92,6 @@ class OveruseFrameDetectorResourceAdaptationModule
   // resource adaptation module. Unify code paths where possible. Do we really
   // need this many public methods?
   void SetLastFramePixelCount(absl::optional<int> last_frame_pixel_count);
-  void SetEncoderConfig(VideoEncoderConfig encoder_config);
-  void SetCodecMaxFrameRate(absl::optional<double> codec_max_frame_rate);
   void SetEncoderStartBitrateBps(uint32_t encoder_start_bitrate_bps);
   // Inform the detector whether or not the quality scaler is enabled. This
   // helps GetActiveCounts() return absl::nullopt when appropriate.
@@ -182,6 +176,8 @@ class OveruseFrameDetectorResourceAdaptationModule
     enum class Mode { kAdaptUp, kAdaptDown } mode_;
   };
 
+  VideoCodecType GetVideoCodecTypeOrGeneric() const;
+
   // Makes |video_source_restrictions_| up-to-date and informs the
   // |adaptation_listener_| if restrictions are changed, allowing the listener
   // to reconfigure the source accordingly.
@@ -218,12 +214,10 @@ class OveruseFrameDetectorResourceAdaptationModule
   const std::unique_ptr<VideoSourceRestrictor> source_restrictor_;
   const std::unique_ptr<OveruseFrameDetector> overuse_detector_;
   bool overuse_detector_is_started_;
-  absl::optional<double> codec_max_frame_rate_;
   absl::optional<double> target_frame_rate_;
   uint32_t encoder_start_bitrate_bps_;
   bool is_quality_scaler_enabled_;
-  VideoEncoderConfig encoder_config_;
+  absl::optional<EncoderSettings> encoder_settings_;
-  VideoEncoder* encoder_;
   VideoStreamEncoderObserver* const encoder_stats_observer_;
 };
 

video/video_stream_encoder.cc

@@ -23,6 +23,7 @@
 #include "api/video/video_bitrate_allocator_factory.h"
 #include "api/video/video_codec_constants.h"
 #include "api/video_codecs/video_encoder.h"
+#include "call/adaptation/resource_adaptation_module_interface.h"
 #include "modules/video_coding/codecs/vp9/svc_rate_allocator.h"
 #include "modules/video_coding/include/video_codec_initializer.h"
 #include "modules/video_coding/utility/default_video_bitrate_allocator.h"
@@ -424,7 +425,6 @@ void VideoStreamEncoder::ConfigureEncoder(VideoEncoderConfig config,
             (!encoder_ || encoder_config_.video_format != config.video_format ||
              max_data_payload_length_ != max_data_payload_length);
         encoder_config_ = std::move(config);
-        resource_adaptation_module_->SetEncoderConfig(encoder_config_.Copy());
         max_data_payload_length_ = max_data_payload_length;
         pending_encoder_reconfiguration_ = true;
 
@@ -498,7 +498,6 @@ void VideoStreamEncoder::ReconfigureEncoder() {
 
     encoder_ = settings_.encoder_factory->CreateVideoEncoder(
         encoder_config_.video_format);
-    resource_adaptation_module_->SetEncoder(encoder_.get());
     // TODO(nisse): What to do if creating the encoder fails? Crash,
     // or just discard incoming frames?
     RTC_CHECK(encoder_);
@@ -588,7 +587,6 @@ void VideoStreamEncoder::ReconfigureEncoder() {
   // Make sure the start bit rate is sane...
   RTC_DCHECK_LE(codec.startBitrate, 1000000);
   max_framerate_ = codec.maxFramerate;
-  resource_adaptation_module_->SetCodecMaxFrameRate(max_framerate_);
 
   // Inform source about max configured framerate.
   int max_framerate = 0;
@@ -632,6 +630,9 @@ void VideoStreamEncoder::ReconfigureEncoder() {
   }
   send_codec_ = codec;
 
+  resource_adaptation_module_->SetEncoderSettings(EncoderSettings(
+      encoder_->GetEncoderInfo(), encoder_config_.Copy(), send_codec_));
+
   encoder_switch_experiment_.SetCodec(send_codec_.codecType);
   quality_rampup_experiment_.SetMaxBitrate(
       last_frame_info_->width * last_frame_info_->height, codec.maxBitrate);
@@ -1180,6 +1181,8 @@ void VideoStreamEncoder::EncodeVideoFrame(const VideoFrame& video_frame,
   }
 
   if (encoder_info_ != info) {
+    resource_adaptation_module_->SetEncoderSettings(EncoderSettings(
+        encoder_->GetEncoderInfo(), encoder_config_.Copy(), send_codec_));
     RTC_LOG(LS_INFO) << "Encoder settings changed from "
                      << encoder_info_.ToString() << " to " << info.ToString();
   }