In (new) estimator of encoder cpu load, count max per input frame.

Bug: webrtc:9619
Change-Id: Ifc874fa3bd069e48a10fec57b673546aafd070e3
Reviewed-on: https://webrtc-review.googlesource.com/94143
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Niels Moller <nisse@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#24293}

video/overuse_frame_detector.cc

@@ -243,11 +243,13 @@ class SendProcessingUsage2 : public OveruseFrameDetector::ProcessingUsage {
                      int64_t last_capture_time_us) override {}
 
   absl::optional<int> FrameSent(
-      uint32_t timestamp,
+      uint32_t /* timestamp */,
-      int64_t time_sent_in_us,
+      int64_t /* time_sent_in_us */,
       int64_t capture_time_us,
       absl::optional<int> encode_duration_us) override {
     if (encode_duration_us) {
+      int duration_per_frame_us =
+          DurationPerInputFrame(capture_time_us, *encode_duration_us);
       if (prev_time_us_ != -1) {
         if (capture_time_us < prev_time_us_) {
           // The weighting in AddSample assumes that samples are processed with
@@ -257,7 +259,7 @@ class SendProcessingUsage2 : public OveruseFrameDetector::ProcessingUsage {
           // bit forward in time.
           capture_time_us = prev_time_us_;
         }
-        AddSample(1e-6 * (*encode_duration_us),
+        AddSample(1e-6 * duration_per_frame_us,
                   1e-6 * (capture_time_us - prev_time_us_));
       }
     }
@@ -287,12 +289,43 @@ class SendProcessingUsage2 : public OveruseFrameDetector::ProcessingUsage {
     load_estimate_ = c * encode_time + exp(-e) * load_estimate_;
   }
 
+  int64_t DurationPerInputFrame(int64_t capture_time_us,
+                                int64_t encode_time_us) {
+    // Discard data on old frames; limit 2 seconds.
+    static constexpr int64_t kMaxAge = 2 * rtc::kNumMicrosecsPerSec;
+    for (auto it = max_encode_time_per_input_frame_.begin();
+         it != max_encode_time_per_input_frame_.end() &&
+         it->first < capture_time_us - kMaxAge;) {
+      it = max_encode_time_per_input_frame_.erase(it);
+    }
+
+    std::map<int64_t, int>::iterator it;
+    bool inserted;
+    std::tie(it, inserted) = max_encode_time_per_input_frame_.emplace(
+        capture_time_us, encode_time_us);
+    if (inserted) {
+      // First encoded frame for this input frame.
+      return encode_time_us;
+    }
+    if (encode_time_us <= it->second) {
+      // Shorter encode time than previous frame (unlikely). Count it as being
+      // done in parallel.
+      return 0;
+    }
+    // Record new maximum encode time, and return increase from previous max.
+    int increase = encode_time_us - it->second;
+    it->second = encode_time_us;
+    return increase;
+  }
+
   int Value() override {
     return static_cast<int>(100.0 * load_estimate_ + 0.5);
   }
 
- private:
   const CpuOveruseOptions options_;
+  // Indexed by the capture timestamp, used as frame id.
+  std::map<int64_t, int> max_encode_time_per_input_frame_;
+
   int64_t prev_time_us_ = -1;
   double load_estimate_;
 };

video/overuse_frame_detector_unittest.cc

@@ -108,6 +108,36 @@ class OveruseFrameDetectorTest : public ::testing::Test,
     }
   }
 
+  virtual void InsertAndSendSimulcastFramesWithInterval(
+      int num_frames,
+      int interval_us,
+      int width,
+      int height,
+      // One element per layer
+      rtc::ArrayView<const int> delays_us) {
+    VideoFrame frame(I420Buffer::Create(width, height),
+                     webrtc::kVideoRotation_0, 0);
+    uint32_t timestamp = 0;
+    while (num_frames-- > 0) {
+      frame.set_timestamp(timestamp);
+      int64_t capture_time_us = rtc::TimeMicros();
+      overuse_detector_->FrameCaptured(frame, capture_time_us);
+      int max_delay_us = 0;
+      for (int delay_us : delays_us) {
+        if (delay_us > max_delay_us) {
+          clock_.AdvanceTimeMicros(delay_us - max_delay_us);
+          max_delay_us = delay_us;
+        }
+
+        overuse_detector_->FrameSent(timestamp, rtc::TimeMicros(),
+                                     capture_time_us, delay_us);
+      }
+      overuse_detector_->CheckForOveruse(observer_);
+      clock_.AdvanceTimeMicros(interval_us - max_delay_us);
+      timestamp += interval_us * 90 / 1000;
+    }
+  }
+
   virtual void InsertAndSendFramesWithRandomInterval(int num_frames,
                                                      int min_interval_us,
                                                      int max_interval_us,
@@ -578,6 +608,27 @@ TEST_F(OveruseFrameDetectorTest, NoOveruseForRandomFrameIntervalWithReset) {
   EXPECT_LE(UsagePercent(), InitialUsage() + 5);
 }
 
+// Models simulcast, with multiple encoded frames for each input frame.
+// Load estimate should be based on the maximum encode time per input frame.
+TEST_F(OveruseFrameDetectorTest, NoOveruseForSimulcast) {
+  overuse_detector_->SetOptions(options_);
+  EXPECT_CALL(mock_observer_, AdaptDown(_)).Times(0);
+
+  constexpr int kNumFrames = 500;
+  constexpr int kEncodeTimesUs[] = {
+      10 * rtc::kNumMicrosecsPerMillisec, 8 * rtc::kNumMicrosecsPerMillisec,
+      12 * rtc::kNumMicrosecsPerMillisec,
+  };
+  constexpr int kIntervalUs = 30 * rtc::kNumMicrosecsPerMillisec;
+
+  InsertAndSendSimulcastFramesWithInterval(kNumFrames, kIntervalUs, kWidth,
+                                           kHeight, kEncodeTimesUs);
+
+  // Average usage 40%. 12 ms / 30 ms.
+  EXPECT_GE(UsagePercent(), 35);
+  EXPECT_LE(UsagePercent(), 45);
+}
+
 // Tests using new cpu load estimator
 class OveruseFrameDetectorTest2 : public OveruseFrameDetectorTest {
  protected:
@@ -905,4 +956,25 @@ TEST_F(OveruseFrameDetectorTest2, ToleratesOutOfOrderFrames) {
   EXPECT_GE(UsagePercent(), InitialUsage());
 }
 
+// Models simulcast, with multiple encoded frames for each input frame.
+// Load estimate should be based on the maximum encode time per input frame.
+TEST_F(OveruseFrameDetectorTest2, NoOveruseForSimulcast) {
+  overuse_detector_->SetOptions(options_);
+  EXPECT_CALL(mock_observer_, AdaptDown(_)).Times(0);
+
+  constexpr int kNumFrames = 500;
+  constexpr int kEncodeTimesUs[] = {
+      10 * rtc::kNumMicrosecsPerMillisec, 8 * rtc::kNumMicrosecsPerMillisec,
+      12 * rtc::kNumMicrosecsPerMillisec,
+  };
+  constexpr int kIntervalUs = 30 * rtc::kNumMicrosecsPerMillisec;
+
+  InsertAndSendSimulcastFramesWithInterval(kNumFrames, kIntervalUs, kWidth,
+                                           kHeight, kEncodeTimesUs);
+
+  // Average usage 40%. 12 ms / 30 ms.
+  EXPECT_GE(UsagePercent(), 35);
+  EXPECT_LE(UsagePercent(), 45);
+}
+
 }  // namespace webrtc