Move initial frame drop to overuse module

It would be nice for this to stay in video stream encoder,
but this feature is mostly related to quality scaling. Perhaps
something easier to understand is possible in the future.

Bug: webrtc:11222
Change-Id: I71705f33ff94bbcf2fb9b5c94226c8e76dcba94c
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/168051
Commit-Queue: Evan Shrubsole <eshr@google.com>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30446}

call/adaptation/resource_adaptation_module_interface.h

@@ -113,7 +113,13 @@ class ResourceAdaptationModuleInterface {
   // TODO(hbos): If we take frame rate into account perhaps it would be valid to
   // adapt down in frame rate as well.
   virtual void OnFrameDroppedDueToSize() = 0;
-  // 2.ii) An input frame is about to be encoded. It may have been cropped and
+  // 2.ii) If the frame will not be dropped due to size then signal that it may
+  // get encoded. However the frame is not guaranteed to be encoded right away
+  // or ever (for example if encoding is paused).
+  // TODO(eshr): Try replace OnMaybeEncodeFrame and merge behaviour into
+  // EncodeStarted.
+  virtual void OnMaybeEncodeFrame() = 0;
+  // 2.iii) An input frame is about to be encoded. It may have been cropped and
   // have different dimensions than what was observed at OnFrame(). Next
   // up: encoding completes or fails, see OnEncodeCompleted(). There is
   // currently no signal for encode failure.

video/overuse_frame_detector_resource_adaptation_module.cc

@@ -66,6 +66,10 @@ VideoSourceRestrictions ApplyDegradationPreference(
   return source_restrictions;
 }
 
+// The maximum number of frames to drop at beginning of stream
+// to try and achieve desired bitrate.
+const int kMaxInitialFramedrop = 4;
+
 }  // namespace
 
 // VideoSourceRestrictor is responsible for keeping track of current
@@ -361,7 +365,8 @@ OveruseFrameDetectorResourceAdaptationModule::
       target_bitrate_bps_(absl::nullopt),
       quality_scaler_(nullptr),
       encoder_settings_(absl::nullopt),
-      encoder_stats_observer_(encoder_stats_observer) {
+      encoder_stats_observer_(encoder_stats_observer),
+      initial_framedrop_(0) {
   RTC_DCHECK(adaptation_listener_);
   RTC_DCHECK(overuse_detector_);
   RTC_DCHECK(encoder_stats_observer_);
@@ -459,6 +464,7 @@ void OveruseFrameDetectorResourceAdaptationModule::OnFrameDroppedDueToSize() {
           AdaptationObserverInterface::AdaptReason::kQuality) > res_count) {
     encoder_stats_observer_->OnInitialQualityResolutionAdaptDown();
   }
+  ++initial_framedrop_;
 }
 
 void OveruseFrameDetectorResourceAdaptationModule::OnEncodeStarted(
@@ -499,13 +505,29 @@ void OveruseFrameDetectorResourceAdaptationModule::OnFrameDropped(
   }
 }
 
+void OveruseFrameDetectorResourceAdaptationModule::OnMaybeEncodeFrame() {
+  initial_framedrop_ = kMaxInitialFramedrop;
+}
+
+bool OveruseFrameDetectorResourceAdaptationModule::DropInitialFrames() const {
+  return initial_framedrop_ < kMaxInitialFramedrop;
+}
+
+void OveruseFrameDetectorResourceAdaptationModule::ResetInitialFrameDropping() {
+  initial_framedrop_ = 0;
+}
+
 void OveruseFrameDetectorResourceAdaptationModule::UpdateQualityScalerSettings(
     absl::optional<VideoEncoder::QpThresholds> qp_thresholds) {
   if (qp_thresholds.has_value()) {
     quality_scaler_ =
         std::make_unique<QualityScaler>(this, qp_thresholds.value());
+    // Restart frame drops due to size.
+    initial_framedrop_ = 0;
   } else {
     quality_scaler_ = nullptr;
+    // Quality scaling disabled so we shouldn't drop initial frames.
+    initial_framedrop_ = kMaxInitialFramedrop;
   }
 }
 

video/overuse_frame_detector_resource_adaptation_module.h

@@ -79,12 +79,16 @@ class OveruseFrameDetectorResourceAdaptationModule
 
   void OnFrame(const VideoFrame& frame) override;
   void OnFrameDroppedDueToSize() override;
+  void OnMaybeEncodeFrame() override;
   void OnEncodeStarted(const VideoFrame& cropped_frame,
                        int64_t time_when_first_seen_us) override;
   void OnEncodeCompleted(const EncodedImage& encoded_image,
                          int64_t time_sent_in_us,
                          absl::optional<int> encode_duration_us) override;
   void OnFrameDropped(EncodedImageCallback::DropReason reason) override;
+  bool DropInitialFrames() const;
+  // TODO(eshr): Remove once all qp-scaling is in this class.
+  void ResetInitialFrameDropping();
 
   // Use nullopt to disable quality scaling.
   void UpdateQualityScalerSettings(
@@ -210,6 +214,8 @@ class OveruseFrameDetectorResourceAdaptationModule
   std::unique_ptr<QualityScaler> quality_scaler_;
   absl::optional<EncoderSettings> encoder_settings_;
   VideoStreamEncoderObserver* const encoder_stats_observer_;
+  // Counts how many frames we've dropped in the initial framedrop phase.
+  int initial_framedrop_;
 };
 
 }  // namespace webrtc

video/video_stream_encoder.cc

@@ -52,10 +52,6 @@ const int64_t kPendingFrameTimeoutMs = 1000;
 
 constexpr char kFrameDropperFieldTrial[] = "WebRTC-FrameDropper";
 
-// The maximum number of frames to drop at beginning of stream
-// to try and achieve desired bitrate.
-const int kMaxInitialFramedrop = 4;
-
 // Averaging window spanning 90 frames at default 30fps, matching old media
 // optimization module defaults.
 const int64_t kFrameRateAvergingWindowSizeMs = (1000 / 30) * 90;
@@ -258,7 +254,6 @@ VideoStreamEncoder::VideoStreamEncoder(
     TaskQueueFactory* task_queue_factory)
     : shutdown_event_(true /* manual_reset */, false),
       number_of_cores_(number_of_cores),
-      initial_framedrop_(0),
       quality_rampup_done_(false),
       quality_rampup_experiment_(QualityRampupExperiment::ParseSettings()),
       quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()),
@@ -798,11 +793,9 @@ void VideoStreamEncoder::ConfigureQualityScaler(
       resource_adaptation_module_->UpdateQualityScalerSettings(
           experimental_thresholds ? *experimental_thresholds
                                   : *(scaling_settings.thresholds));
-      initial_framedrop_ = 0;
     }
   } else {
     resource_adaptation_module_->UpdateQualityScalerSettings(absl::nullopt);
-    initial_framedrop_ = kMaxInitialFramedrop;
   }
 
   QualityScaler* quality_scaler = resource_adaptation_module_->quality_scaler();
@@ -1112,7 +1105,6 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame,
   if (DropDueToSize(video_frame.size())) {
     RTC_LOG(LS_INFO) << "Dropping frame. Too large for target bitrate.";
     resource_adaptation_module_->OnFrameDroppedDueToSize();
-    ++initial_framedrop_;
     // Storing references to a native buffer risks blocking frame capture.
     if (video_frame.video_frame_buffer()->type() !=
         VideoFrameBuffer::Type::kNative) {
@@ -1126,7 +1118,7 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame,
     }
     return;
   }
-  initial_framedrop_ = kMaxInitialFramedrop;
+  resource_adaptation_module_->OnMaybeEncodeFrame();
 
   if (!quality_rampup_done_ && TryQualityRampup(now_ms) &&
       resource_adaptation_module_->GetConstAdaptCounter().ResolutionCount(
@@ -1608,7 +1600,7 @@ void VideoStreamEncoder::OnBitrateUpdated(DataRate target_bitrate,
       RTC_LOG(LS_INFO) << "Reset initial_framedrop_. Start bitrate: "
                        << set_start_bitrate_bps_
                        << ", target bitrate: " << target_bitrate.bps();
-      initial_framedrop_ = 0;
+      resource_adaptation_module_->ResetInitialFrameDropping();
       has_seen_first_bwe_drop_ = true;
     }
   }
@@ -1648,7 +1640,7 @@ void VideoStreamEncoder::OnBitrateUpdated(DataRate target_bitrate,
 }
 
 bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const {
-  if (initial_framedrop_ >= kMaxInitialFramedrop ||
+  if (!resource_adaptation_module_->DropInitialFrames() ||
       !encoder_target_bitrate_bps_.has_value()) {
     return false;
   }

video/video_stream_encoder.h

@@ -213,8 +213,6 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
   rtc::Event shutdown_event_;
 
   const uint32_t number_of_cores_;
-  // Counts how many frames we've dropped in the initial framedrop phase.
-  int initial_framedrop_;
   bool quality_rampup_done_ RTC_GUARDED_BY(&encoder_queue_);
   QualityRampupExperiment quality_rampup_experiment_
       RTC_GUARDED_BY(&encoder_queue_);