Add fieldtrial to enable minimum pacing of video frames

If the RTP header extension playout-delay is used and set
to min=0, max>=0, frames are scheduled to be decoded as
soon as possible. There's a risk that too many frames are
sent to the decoder at once, which may cause problems
further down in the video pipeline.

This CL adds the fieldtrial WebRTC-ZeroPlayoutDelay with
the parameter min_pacing that determines the minimum
pacing interval between two frames scheduled for
decoding.

Bug: None
Change-Id: I471f7718761cfce9789b3aa8adea3e8a16ecb2fd
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/223742
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Johannes Kron <kron@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#34387}

modules/video_coding/frame_buffer2_unittest.cc

@@ -55,8 +55,7 @@ class VCMTimingFake : public VCMTiming {
     return last_ms_;
   }
 
-  int64_t MaxWaitingTime(int64_t render_time_ms,
-                         int64_t now_ms) const override {
+  int64_t MaxWaitingTime(int64_t render_time_ms, int64_t now_ms) override {
     return render_time_ms - now_ms - kDecodeTime;
   }
 

modules/video_coding/timing.cc

@@ -34,9 +34,13 @@ VCMTiming::VCMTiming(Clock* clock)
       prev_frame_timestamp_(0),
       timing_frame_info_(),
       num_decoded_frames_(0),
-      low_latency_renderer_enabled_("enabled", true) {
+      low_latency_renderer_enabled_("enabled", true),
+      zero_playout_delay_min_pacing_("min_pacing", TimeDelta::Millis(0)),
+      earliest_next_decode_start_time_(0) {
   ParseFieldTrial({&low_latency_renderer_enabled_},
                   field_trial::FindFullName("WebRTC-LowLatencyRenderer"));
+  ParseFieldTrial({&zero_playout_delay_min_pacing_},
+                  field_trial::FindFullName("WebRTC-ZeroPlayoutDelay"));
 }
 
 void VCMTiming::Reset() {
@@ -199,14 +203,22 @@ int VCMTiming::RequiredDecodeTimeMs() const {
   return decode_time_ms;
 }
 
-int64_t VCMTiming::MaxWaitingTime(int64_t render_time_ms,
-                                  int64_t now_ms) const {
+int64_t VCMTiming::MaxWaitingTime(int64_t render_time_ms, int64_t now_ms) {
   MutexLock lock(&mutex_);
 
-  const int64_t max_wait_time_ms =
-      render_time_ms - now_ms - RequiredDecodeTimeMs() - render_delay_ms_;
-
+  if (render_time_ms == 0 && zero_playout_delay_min_pacing_->us() > 0) {
+    // |render_time_ms| == 0 indicates that the frame should be decoded and
+    // rendered as soon as possible. However, the decoder can be choked if too
+    // many frames are sent at ones. Therefore, limit the interframe delay to
+    // |zero_playout_delay_min_pacing_|.
+    int64_t max_wait_time_ms = now_ms >= earliest_next_decode_start_time_
+                                   ? 0
+                                   : earliest_next_decode_start_time_ - now_ms;
+    earliest_next_decode_start_time_ =
+        now_ms + max_wait_time_ms + zero_playout_delay_min_pacing_->ms();
     return max_wait_time_ms;
+  }
+  return render_time_ms - now_ms - RequiredDecodeTimeMs() - render_delay_ms_;
 }
 
 int VCMTiming::TargetVideoDelay() const {

modules/video_coding/timing.h

@@ -14,6 +14,7 @@
 #include <memory>
 
 #include "absl/types/optional.h"
+#include "api/units/time_delta.h"
 #include "api/video/video_timing.h"
 #include "modules/video_coding/codec_timer.h"
 #include "rtc_base/experiments/field_trial_parser.h"
@@ -82,7 +83,7 @@ class VCMTiming {
 
   // Returns the maximum time in ms that we can wait for a frame to become
   // complete before we must pass it to the decoder.
-  virtual int64_t MaxWaitingTime(int64_t render_time_ms, int64_t now_ms) const;
+  virtual int64_t MaxWaitingTime(int64_t render_time_ms, int64_t now_ms);
 
   // Returns the current target delay which is required delay + decode time +
   // render delay.
@@ -139,6 +140,15 @@ class VCMTiming {
   FieldTrialParameter<bool> low_latency_renderer_enabled_
       RTC_GUARDED_BY(mutex_);
   absl::optional<int> max_composition_delay_in_frames_ RTC_GUARDED_BY(mutex_);
+  // Set by the field trial WebRTC-ZeroPlayoutDelay. The parameter min_pacing
+  // determines the minimum delay between frames scheduled for decoding that is
+  // used when min playout delay=0 and max playout delay>=0.
+  FieldTrialParameter<TimeDelta> zero_playout_delay_min_pacing_
+      RTC_GUARDED_BY(mutex_);
+  // An estimate of when the last frame is scheduled to be sent to the decoder.
+  // Used only when the RTP header extension playout delay is set to min=0 ms
+  // which is indicated by a render time set to 0.
+  int64_t earliest_next_decode_start_time_ RTC_GUARDED_BY(mutex_);
 };
 }  // namespace webrtc
 

modules/video_coding/timing_unittest.cc

@@ -11,6 +11,7 @@
 #include "modules/video_coding/timing.h"
 
 #include "system_wrappers/include/clock.h"
+#include "test/field_trial.h"
 #include "test/gtest.h"
 
 namespace webrtc {
@@ -18,7 +19,7 @@ namespace {
 const int kFps = 25;
 }  // namespace
 
-TEST(ReceiverTiming, Tests) {
+TEST(ReceiverTimingTest, JitterDelay) {
   SimulatedClock clock(0);
   VCMTiming timing(&clock);
   timing.Reset();
@@ -110,7 +111,7 @@ TEST(ReceiverTiming, Tests) {
   timing.UpdateCurrentDelay(timestamp);
 }
 
-TEST(ReceiverTiming, WrapAround) {
+TEST(ReceiverTimingTest, TimestampWrapAround) {
   SimulatedClock clock(0);
   VCMTiming timing(&clock);
   // Provoke a wrap-around. The fifth frame will have wrapped at 25 fps.
@@ -127,4 +128,89 @@ TEST(ReceiverTiming, WrapAround) {
   }
 }
 
+TEST(ReceiverTimingTest, MaxWaitingTimeIsZeroForZeroRenderTime) {
+  // This is the default path when the RTP playout delay header extension is set
+  // to min==0.
+  constexpr int64_t kStartTimeUs = 3.15e13;  // About one year in us.
+  constexpr int64_t kTimeDeltaMs = 1000.0 / 60.0;
+  constexpr int64_t kZeroRenderTimeMs = 0;
+  SimulatedClock clock(kStartTimeUs);
+  VCMTiming timing(&clock);
+  timing.Reset();
+  for (int i = 0; i < 10; ++i) {
+    clock.AdvanceTimeMilliseconds(kTimeDeltaMs);
+    int64_t now_ms = clock.TimeInMilliseconds();
+    EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 0);
+  }
+  // Another frame submitted at the same time also returns a negative max
+  // waiting time.
+  int64_t now_ms = clock.TimeInMilliseconds();
+  EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 0);
+  // MaxWaitingTime should be less than zero even if there's a burst of frames.
+  EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 0);
+  EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 0);
+  EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 0);
+}
+
+TEST(ReceiverTimingTest, MaxWaitingTimeZeroDelayPacingExperiment) {
+  // The minimum pacing is enabled by a field trial and active if the RTP
+  // playout delay header extension is set to min==0.
+  constexpr int64_t kMinPacingMs = 3;
+  test::ScopedFieldTrials override_field_trials(
+      "WebRTC-ZeroPlayoutDelay/min_pacing:3ms/");
+  constexpr int64_t kStartTimeUs = 3.15e13;  // About one year in us.
+  constexpr int64_t kTimeDeltaMs = 1000.0 / 60.0;
+  constexpr int64_t kZeroRenderTimeMs = 0;
+  SimulatedClock clock(kStartTimeUs);
+  VCMTiming timing(&clock);
+  timing.Reset();
+  // MaxWaitingTime() returns zero for evenly spaced video frames.
+  for (int i = 0; i < 10; ++i) {
+    clock.AdvanceTimeMilliseconds(kTimeDeltaMs);
+    int64_t now_ms = clock.TimeInMilliseconds();
+    EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 0);
+  }
+  // Another frame submitted at the same time is paced according to the field
+  // trial setting.
+  int64_t now_ms = clock.TimeInMilliseconds();
+  EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), kMinPacingMs);
+  // If there's a burst of frames, the min pacing interval is summed.
+  EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 2 * kMinPacingMs);
+  EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 3 * kMinPacingMs);
+  EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 4 * kMinPacingMs);
+  // Allow a few ms to pass, this should be subtracted from the MaxWaitingTime.
+  constexpr int64_t kTwoMs = 2;
+  clock.AdvanceTimeMilliseconds(kTwoMs);
+  now_ms = clock.TimeInMilliseconds();
+  EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms),
+            5 * kMinPacingMs - kTwoMs);
+}
+
+TEST(ReceiverTimingTest, DefaultMaxWaitingTimeUnaffectedByPacingExperiment) {
+  // The minimum pacing is enabled by a field trial but should not have any
+  // effect if render_time_ms is greater than 0;
+  test::ScopedFieldTrials override_field_trials(
+      "WebRTC-ZeroPlayoutDelay/min_pacing:3ms/");
+  constexpr int64_t kStartTimeUs = 3.15e13;  // About one year in us.
+  constexpr int64_t kTimeDeltaMs = 1000.0 / 60.0;
+  SimulatedClock clock(kStartTimeUs);
+  VCMTiming timing(&clock);
+  timing.Reset();
+  clock.AdvanceTimeMilliseconds(kTimeDeltaMs);
+  int64_t now_ms = clock.TimeInMilliseconds();
+  int64_t render_time_ms = now_ms + 30;
+  // Estimate the internal processing delay from the first frame.
+  int64_t estimated_processing_delay =
+      (render_time_ms - now_ms) - timing.MaxWaitingTime(render_time_ms, now_ms);
+  EXPECT_GT(estimated_processing_delay, 0);
+
+  // Any other frame submitted at the same time should be scheduled according to
+  // its render time.
+  for (int i = 0; i < 5; ++i) {
+    render_time_ms += kTimeDeltaMs;
+    EXPECT_EQ(timing.MaxWaitingTime(render_time_ms, now_ms),
+              render_time_ms - now_ms - estimated_processing_delay);
+  }
+}
+
 }  // namespace webrtc