From 82c224851162d7f32ff59706baf09bec2e3a2b93 Mon Sep 17 00:00:00 2001 From: Rezaul Barbhuiya Date: Thu, 5 Aug 2021 17:54:11 -0700 Subject: [PATCH] Calculate pacing delay based on decode start time Schedule the frames to be decoded based on the pacing delay from the last decode scheduled time. In the current implementation, multiple threads and different functions in same thread can call MaxWaitingTime(), thereby increasing the wait time each time the function is called. Instead of returning the wait time for a future frame based on the number of times the function is called, return the wait time only for the next frame to be decoded. Threads can call the function repeatedly to check the waiting time for next frame and wake up and then go back to waiting if an encoded frame is not available. Change-Id: I00886c1619599f94bde5d5eb87405572e435bd73 Bug: chromium:1237402 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/226502 Reviewed-by: Johannes Kron Reviewed-by: Philip Eliasson Commit-Queue: Johannes Kron Cr-Commit-Position: refs/heads/master@{#34660} --- modules/video_coding/frame_buffer2.cc | 2 ++ .../video_coding/frame_buffer2_unittest.cc | 3 ++- modules/video_coding/timing.cc | 19 +++++++++++++------ modules/video_coding/timing.h | 9 ++++++--- modules/video_coding/timing_unittest.cc | 15 ++++++++++----- 5 files changed, 33 insertions(+), 15 deletions(-) diff --git a/modules/video_coding/frame_buffer2.cc b/modules/video_coding/frame_buffer2.cc index 903b9fb623..ba77b72d4f 100644 --- a/modules/video_coding/frame_buffer2.cc +++ b/modules/video_coding/frame_buffer2.cc @@ -110,6 +110,8 @@ void FrameBuffer::StartWaitForNextFrameOnQueue() { if (!frames_to_decode_.empty()) { // We have frames, deliver! frame = absl::WrapUnique(GetNextFrame()); + timing_->SetLastDecodeScheduledTimestamp( + clock_->TimeInMilliseconds()); } else if (clock_->TimeInMilliseconds() < latest_return_time_ms_) { // If there's no frames to decode and there is still time left, it // means that the frame buffer was cleared between creation and diff --git a/modules/video_coding/frame_buffer2_unittest.cc b/modules/video_coding/frame_buffer2_unittest.cc index d37efda57b..68acf813ae 100644 --- a/modules/video_coding/frame_buffer2_unittest.cc +++ b/modules/video_coding/frame_buffer2_unittest.cc @@ -55,7 +55,8 @@ class VCMTimingFake : public VCMTiming { return last_ms_; } - int64_t MaxWaitingTime(int64_t render_time_ms, int64_t now_ms) override { + int64_t MaxWaitingTime(int64_t render_time_ms, + int64_t now_ms) const override { return render_time_ms - now_ms - kDecodeTime; } diff --git a/modules/video_coding/timing.cc b/modules/video_coding/timing.cc index 748013cf47..dc25efe0c7 100644 --- a/modules/video_coding/timing.cc +++ b/modules/video_coding/timing.cc @@ -35,7 +35,7 @@ VCMTiming::VCMTiming(Clock* clock) num_decoded_frames_(0), low_latency_renderer_enabled_("enabled", true), zero_playout_delay_min_pacing_("min_pacing", TimeDelta::Millis(0)), - earliest_next_decode_start_time_(0) { + last_decode_scheduled_ts_(0) { ParseFieldTrial({&low_latency_renderer_enabled_}, field_trial::FindFullName("WebRTC-LowLatencyRenderer")); ParseFieldTrial({&zero_playout_delay_min_pacing_}, @@ -171,6 +171,12 @@ int64_t VCMTiming::RenderTimeMs(uint32_t frame_timestamp, return RenderTimeMsInternal(frame_timestamp, now_ms); } +void VCMTiming::SetLastDecodeScheduledTimestamp( + int64_t last_decode_scheduled_ts) { + MutexLock lock(&mutex_); + last_decode_scheduled_ts_ = last_decode_scheduled_ts; +} + int64_t VCMTiming::RenderTimeMsInternal(uint32_t frame_timestamp, int64_t now_ms) const { constexpr int kLowLatencyRendererMaxPlayoutDelayMs = 500; @@ -202,7 +208,8 @@ int VCMTiming::RequiredDecodeTimeMs() const { return decode_time_ms; } -int64_t VCMTiming::MaxWaitingTime(int64_t render_time_ms, int64_t now_ms) { +int64_t VCMTiming::MaxWaitingTime(int64_t render_time_ms, + int64_t now_ms) const { MutexLock lock(&mutex_); if (render_time_ms == 0 && zero_playout_delay_min_pacing_->us() > 0) { @@ -210,11 +217,11 @@ int64_t VCMTiming::MaxWaitingTime(int64_t render_time_ms, int64_t now_ms) { // rendered as soon as possible. However, the decoder can be choked if too // many frames are sent at ones. Therefore, limit the interframe delay to // |zero_playout_delay_min_pacing_|. - int64_t max_wait_time_ms = now_ms >= earliest_next_decode_start_time_ + int64_t earliest_next_decode_start_time = + last_decode_scheduled_ts_ + zero_playout_delay_min_pacing_->ms(); + int64_t max_wait_time_ms = now_ms >= earliest_next_decode_start_time ? 0 - : earliest_next_decode_start_time_ - now_ms; - earliest_next_decode_start_time_ = - now_ms + max_wait_time_ms + zero_playout_delay_min_pacing_->ms(); + : earliest_next_decode_start_time - now_ms; return max_wait_time_ms; } return render_time_ms - now_ms - RequiredDecodeTimeMs() - render_delay_ms_; diff --git a/modules/video_coding/timing.h b/modules/video_coding/timing.h index 1583082a58..2f3fdfe057 100644 --- a/modules/video_coding/timing.h +++ b/modules/video_coding/timing.h @@ -83,7 +83,7 @@ class VCMTiming { // Returns the maximum time in ms that we can wait for a frame to become // complete before we must pass it to the decoder. - virtual int64_t MaxWaitingTime(int64_t render_time_ms, int64_t now_ms); + virtual int64_t MaxWaitingTime(int64_t render_time_ms, int64_t now_ms) const; // Returns the current target delay which is required delay + decode time + // render delay. @@ -105,6 +105,9 @@ class VCMTiming { absl::optional max_composition_delay_in_frames); absl::optional MaxCompositionDelayInFrames() const; + // Updates the last time a frame was scheduled for decoding. + void SetLastDecodeScheduledTimestamp(int64_t last_decode_scheduled_ts); + enum { kDefaultRenderDelayMs = 10 }; enum { kDelayMaxChangeMsPerS = 100 }; @@ -145,10 +148,10 @@ class VCMTiming { // used when min playout delay=0 and max playout delay>=0. FieldTrialParameter zero_playout_delay_min_pacing_ RTC_GUARDED_BY(mutex_); - // An estimate of when the last frame is scheduled to be sent to the decoder. + // Timestamp at which the last frame was scheduled to be sent to the decoder. // Used only when the RTP header extension playout delay is set to min=0 ms // which is indicated by a render time set to 0. - int64_t earliest_next_decode_start_time_ RTC_GUARDED_BY(mutex_); + int64_t last_decode_scheduled_ts_ RTC_GUARDED_BY(mutex_); }; } // namespace webrtc diff --git a/modules/video_coding/timing_unittest.cc b/modules/video_coding/timing_unittest.cc index be6ac52d92..988e55faab 100644 --- a/modules/video_coding/timing_unittest.cc +++ b/modules/video_coding/timing_unittest.cc @@ -169,21 +169,26 @@ TEST(ReceiverTimingTest, MaxWaitingTimeZeroDelayPacingExperiment) { clock.AdvanceTimeMilliseconds(kTimeDeltaMs); int64_t now_ms = clock.TimeInMilliseconds(); EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 0); + timing.SetLastDecodeScheduledTimestamp(now_ms); } // Another frame submitted at the same time is paced according to the field // trial setting. int64_t now_ms = clock.TimeInMilliseconds(); EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), kMinPacingMs); - // If there's a burst of frames, the min pacing interval is summed. - EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 2 * kMinPacingMs); - EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 3 * kMinPacingMs); - EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 4 * kMinPacingMs); + // If there's a burst of frames, the wait time is calculated based on next + // decode time. + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), kMinPacingMs); + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), kMinPacingMs); // Allow a few ms to pass, this should be subtracted from the MaxWaitingTime. constexpr int64_t kTwoMs = 2; clock.AdvanceTimeMilliseconds(kTwoMs); now_ms = clock.TimeInMilliseconds(); EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), - 5 * kMinPacingMs - kTwoMs); + kMinPacingMs - kTwoMs); + // A frame is decoded at the current time, the wait time should be restored to + // pacing delay. + timing.SetLastDecodeScheduledTimestamp(now_ms); + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), kMinPacingMs); } TEST(ReceiverTimingTest, DefaultMaxWaitingTimeUnaffectedByPacingExperiment) {