From 476f18d2a4b0e12b72894a0ddb7bc77dba3f10ff Mon Sep 17 00:00:00 2001 From: Evan Shrubsole Date: Mon, 15 Aug 2022 15:21:16 +0000 Subject: [PATCH] Rename FrameBufferProxy to VideoStreamBufferController MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This makes the class concrete, and the former FrameBuffer3Proxy is the implementation. Bug: webrtc:14003 Change-Id: Ife825b9f4efc7b79d9be8b4afb03904da819958a Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/265868 Reviewed-by: Philip Eliasson Reviewed-by: Erik Språng Commit-Queue: Evan Shrubsole Cr-Commit-Position: refs/heads/main@{#37793} --- video/BUILD.gn | 12 +- video/frame_buffer_proxy.cc | 509 ------------------ video/frame_buffer_proxy.h | 71 --- video/video_receive_stream2.cc | 26 +- video/video_receive_stream2.h | 4 +- video/video_stream_buffer_controller.cc | 461 ++++++++++++++++ video/video_stream_buffer_controller.h | 135 +++++ ...ideo_stream_buffer_controller_unittest.cc} | 411 +++++++------- 8 files changed, 829 insertions(+), 800 deletions(-) delete mode 100644 video/frame_buffer_proxy.cc delete mode 100644 video/frame_buffer_proxy.h create mode 100644 video/video_stream_buffer_controller.cc create mode 100644 video/video_stream_buffer_controller.h rename video/{frame_buffer_proxy_unittest.cc => video_stream_buffer_controller_unittest.cc} (68%) diff --git a/video/BUILD.gn b/video/BUILD.gn index 5176e13d1c..dbffa2b71a 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -53,12 +53,12 @@ rtc_library("video") { ] deps = [ - ":frame_buffer_proxy", ":frame_cadence_adapter", ":frame_decode_scheduler", ":frame_dumping_decoder", ":unique_timestamp_counter", ":video_receive_stream_timeout_tracker", + ":video_stream_buffer_controller", ":video_stream_encoder_impl", "../api:array_view", "../api:fec_controller_api", @@ -249,10 +249,10 @@ rtc_library("frame_cadence_adapter") { ] } -rtc_library("frame_buffer_proxy") { +rtc_library("video_stream_buffer_controller") { sources = [ - "frame_buffer_proxy.cc", - "frame_buffer_proxy.h", + "video_stream_buffer_controller.cc", + "video_stream_buffer_controller.h", ] deps = [ ":decode_synchronizer", @@ -744,7 +744,6 @@ if (rtc_include_tests) { "end_to_end_tests/ssrc_tests.cc", "end_to_end_tests/stats_tests.cc", "end_to_end_tests/transport_feedback_tests.cc", - "frame_buffer_proxy_unittest.cc", "frame_cadence_adapter_unittest.cc", "frame_decode_timing_unittest.cc", "frame_encode_metadata_writer_unittest.cc", @@ -767,12 +766,12 @@ if (rtc_include_tests) { "video_send_stream_impl_unittest.cc", "video_send_stream_tests.cc", "video_source_sink_controller_unittest.cc", + "video_stream_buffer_controller_unittest.cc", "video_stream_decoder_impl_unittest.cc", "video_stream_encoder_unittest.cc", ] deps = [ ":decode_synchronizer", - ":frame_buffer_proxy", ":frame_cadence_adapter", ":frame_decode_scheduler", ":frame_decode_timing", @@ -781,6 +780,7 @@ if (rtc_include_tests) { ":video", ":video_mocks", ":video_receive_stream_timeout_tracker", + ":video_stream_buffer_controller", ":video_stream_decoder_impl", ":video_stream_encoder_impl", "../api:create_frame_generator", diff --git a/video/frame_buffer_proxy.cc b/video/frame_buffer_proxy.cc deleted file mode 100644 index f8901aed3a..0000000000 --- a/video/frame_buffer_proxy.cc +++ /dev/null @@ -1,509 +0,0 @@ -/* - * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "video/frame_buffer_proxy.h" - -#include -#include -#include - -#include "absl/base/attributes.h" -#include "absl/functional/bind_front.h" -#include "api/sequence_checker.h" -#include "api/task_queue/task_queue_base.h" -#include "api/units/data_size.h" -#include "api/video/encoded_frame.h" -#include "api/video/frame_buffer.h" -#include "api/video/video_content_type.h" -#include "modules/video_coding/frame_helpers.h" -#include "modules/video_coding/timing/inter_frame_delay.h" -#include "modules/video_coding/timing/jitter_estimator.h" -#include "rtc_base/checks.h" -#include "rtc_base/experiments/rtt_mult_experiment.h" -#include "rtc_base/logging.h" -#include "rtc_base/thread_annotations.h" -#include "video/frame_decode_timing.h" -#include "video/task_queue_frame_decode_scheduler.h" -#include "video/video_receive_stream_timeout_tracker.h" - -namespace webrtc { - -namespace { - -// Max number of frames the buffer will hold. -static constexpr size_t kMaxFramesBuffered = 800; -// Max number of decoded frame info that will be saved. -static constexpr int kMaxFramesHistory = 1 << 13; - -// Default value for the maximum decode queue size that is used when the -// low-latency renderer is used. -static constexpr size_t kZeroPlayoutDelayDefaultMaxDecodeQueueSize = 8; - -struct FrameMetadata { - explicit FrameMetadata(const EncodedFrame& frame) - : is_last_spatial_layer(frame.is_last_spatial_layer), - is_keyframe(frame.is_keyframe()), - size(frame.size()), - contentType(frame.contentType()), - delayed_by_retransmission(frame.delayed_by_retransmission()), - rtp_timestamp(frame.Timestamp()), - receive_time(frame.ReceivedTimestamp()) {} - - const bool is_last_spatial_layer; - const bool is_keyframe; - const size_t size; - const VideoContentType contentType; - const bool delayed_by_retransmission; - const uint32_t rtp_timestamp; - const absl::optional receive_time; -}; - -Timestamp ReceiveTime(const EncodedFrame& frame) { - absl::optional ts = frame.ReceivedTimestamp(); - RTC_DCHECK(ts.has_value()) << "Received frame must have a timestamp set!"; - return *ts; -} - -// Encapsulates use of the new frame buffer for use in -// VideoReceiveStreamInterface. This behaves the same as the FrameBuffer2Proxy -// but uses frame_buffer instead. Responsibilities from frame_buffer2, like -// stats, jitter and frame timing accounting are moved into this pro -class FrameBuffer3Proxy : public FrameBufferProxy { - public: - FrameBuffer3Proxy( - Clock* clock, - TaskQueueBase* worker_queue, - VCMTiming* timing, - VCMReceiveStatisticsCallback* stats_proxy, - TaskQueueBase* decode_queue, - FrameSchedulingReceiver* receiver, - TimeDelta max_wait_for_keyframe, - TimeDelta max_wait_for_frame, - std::unique_ptr frame_decode_scheduler, - const FieldTrialsView& field_trials) - : field_trials_(field_trials), - clock_(clock), - worker_queue_(worker_queue), - decode_queue_(decode_queue), - stats_proxy_(stats_proxy), - receiver_(receiver), - timing_(timing), - frame_decode_scheduler_(std::move(frame_decode_scheduler)), - jitter_estimator_(clock_, field_trials), - buffer_(std::make_unique(kMaxFramesBuffered, - kMaxFramesHistory, - field_trials)), - decode_timing_(clock_, timing_), - timeout_tracker_(clock_, - worker_queue_, - VideoReceiveStreamTimeoutTracker::Timeouts{ - .max_wait_for_keyframe = max_wait_for_keyframe, - .max_wait_for_frame = max_wait_for_frame}, - absl::bind_front(&FrameBuffer3Proxy::OnTimeout, this)), - zero_playout_delay_max_decode_queue_size_( - "max_decode_queue_size", - kZeroPlayoutDelayDefaultMaxDecodeQueueSize) { - RTC_DCHECK(decode_queue_); - RTC_DCHECK(stats_proxy_); - RTC_DCHECK(receiver_); - RTC_DCHECK(timing_); - RTC_DCHECK(worker_queue_); - RTC_DCHECK(clock_); - RTC_DCHECK(frame_decode_scheduler_); - RTC_LOG(LS_WARNING) << "Using FrameBuffer3"; - - ParseFieldTrial({&zero_playout_delay_max_decode_queue_size_}, - field_trials.Lookup("WebRTC-ZeroPlayoutDelay")); - } - - // FrameBufferProxy implementation. - void StopOnWorker() override { - RTC_DCHECK_RUN_ON(&worker_sequence_checker_); - frame_decode_scheduler_->Stop(); - timeout_tracker_.Stop(); - decoder_ready_for_new_frame_ = false; - decode_queue_->PostTask([this] { - RTC_DCHECK_RUN_ON(decode_queue_); - decode_safety_->SetNotAlive(); - }); - } - - void SetProtectionMode(VCMVideoProtection protection_mode) override { - RTC_DCHECK_RUN_ON(&worker_sequence_checker_); - protection_mode_ = kProtectionNackFEC; - } - - void Clear() override { - RTC_DCHECK_RUN_ON(&worker_sequence_checker_); - stats_proxy_->OnDroppedFrames(buffer_->CurrentSize()); - buffer_ = std::make_unique(kMaxFramesBuffered, - kMaxFramesHistory, field_trials_); - frame_decode_scheduler_->CancelOutstanding(); - } - - absl::optional InsertFrame( - std::unique_ptr frame) override { - RTC_DCHECK_RUN_ON(&worker_sequence_checker_); - FrameMetadata metadata(*frame); - int complete_units = buffer_->GetTotalNumberOfContinuousTemporalUnits(); - if (buffer_->InsertFrame(std::move(frame))) { - RTC_DCHECK(metadata.receive_time) << "Frame receive time must be set!"; - if (!metadata.delayed_by_retransmission && metadata.receive_time) - timing_->IncomingTimestamp(metadata.rtp_timestamp, - *metadata.receive_time); - if (complete_units < buffer_->GetTotalNumberOfContinuousTemporalUnits()) { - stats_proxy_->OnCompleteFrame(metadata.is_keyframe, metadata.size, - metadata.contentType); - MaybeScheduleFrameForRelease(); - } - } - - return buffer_->LastContinuousFrameId(); - } - - void UpdateRtt(int64_t max_rtt_ms) override { - RTC_DCHECK_RUN_ON(&worker_sequence_checker_); - jitter_estimator_.UpdateRtt(TimeDelta::Millis(max_rtt_ms)); - } - - void SetMaxWaits(TimeDelta max_wait_for_keyframe, - TimeDelta max_wait_for_frame) override { - RTC_DCHECK_RUN_ON(&worker_sequence_checker_); - timeout_tracker_.SetTimeouts( - {.max_wait_for_keyframe = max_wait_for_keyframe, - .max_wait_for_frame = max_wait_for_frame}); - } - - void StartNextDecode(bool keyframe_required) override { - if (!worker_queue_->IsCurrent()) { - worker_queue_->PostTask(SafeTask( - worker_safety_.flag(), - [this, keyframe_required] { StartNextDecode(keyframe_required); })); - return; - } - - RTC_DCHECK_RUN_ON(&worker_sequence_checker_); - if (!timeout_tracker_.Running()) - timeout_tracker_.Start(keyframe_required); - keyframe_required_ = keyframe_required; - if (keyframe_required_) { - timeout_tracker_.SetWaitingForKeyframe(); - } - decoder_ready_for_new_frame_ = true; - MaybeScheduleFrameForRelease(); - } - - int Size() override { - RTC_DCHECK_RUN_ON(&worker_sequence_checker_); - return buffer_->CurrentSize(); - } - - void OnFrameReady( - absl::InlinedVector, 4> frames, - Timestamp render_time) { - RTC_DCHECK_RUN_ON(&worker_sequence_checker_); - RTC_DCHECK(!frames.empty()); - - timeout_tracker_.OnEncodedFrameReleased(); - - Timestamp now = clock_->CurrentTime(); - bool superframe_delayed_by_retransmission = false; - DataSize superframe_size = DataSize::Zero(); - const EncodedFrame& first_frame = *frames.front(); - Timestamp receive_time = ReceiveTime(first_frame); - - if (first_frame.is_keyframe()) - keyframe_required_ = false; - - // Gracefully handle bad RTP timestamps and render time issues. - if (FrameHasBadRenderTiming(render_time, now, - timing_->TargetVideoDelay())) { - jitter_estimator_.Reset(); - timing_->Reset(); - render_time = timing_->RenderTime(first_frame.Timestamp(), now); - } - - for (std::unique_ptr& frame : frames) { - frame->SetRenderTime(render_time.ms()); - - superframe_delayed_by_retransmission |= - frame->delayed_by_retransmission(); - receive_time = std::max(receive_time, ReceiveTime(*frame)); - superframe_size += DataSize::Bytes(frame->size()); - } - - if (!superframe_delayed_by_retransmission) { - auto frame_delay = inter_frame_delay_.CalculateDelay( - first_frame.Timestamp(), receive_time); - if (frame_delay) { - jitter_estimator_.UpdateEstimate(*frame_delay, superframe_size); - } - - float rtt_mult = protection_mode_ == kProtectionNackFEC ? 0.0 : 1.0; - absl::optional rtt_mult_add_cap_ms = absl::nullopt; - if (rtt_mult_settings_.has_value()) { - rtt_mult = rtt_mult_settings_->rtt_mult_setting; - rtt_mult_add_cap_ms = - TimeDelta::Millis(rtt_mult_settings_->rtt_mult_add_cap_ms); - } - timing_->SetJitterDelay( - jitter_estimator_.GetJitterEstimate(rtt_mult, rtt_mult_add_cap_ms)); - timing_->UpdateCurrentDelay(render_time, now); - } else if (RttMultExperiment::RttMultEnabled()) { - jitter_estimator_.FrameNacked(); - } - - // Update stats. - UpdateDroppedFrames(); - UpdateJitterDelay(); - UpdateTimingFrameInfo(); - - std::unique_ptr frame = - CombineAndDeleteFrames(std::move(frames)); - - timing_->SetLastDecodeScheduledTimestamp(now); - - decoder_ready_for_new_frame_ = false; - // VideoReceiveStream2 wants frames on the decoder thread. - decode_queue_->PostTask( - SafeTask(decode_safety_, [this, frame = std::move(frame)]() mutable { - RTC_DCHECK_RUN_ON(decode_queue_); - receiver_->OnEncodedFrame(std::move(frame)); - })); - } - - void OnTimeout(TimeDelta delay) { - RTC_DCHECK_RUN_ON(&worker_sequence_checker_); - // If the stream is paused then ignore the timeout. - if (!decoder_ready_for_new_frame_) { - timeout_tracker_.Stop(); - return; - } - decode_queue_->PostTask(SafeTask(decode_safety_, [this, delay]() { - RTC_DCHECK_RUN_ON(decode_queue_); - receiver_->OnDecodableFrameTimeout(delay); - })); - // Stop sending timeouts until receive starts waiting for a new frame. - timeout_tracker_.Stop(); - decoder_ready_for_new_frame_ = false; - } - - private: - void FrameReadyForDecode(uint32_t rtp_timestamp, Timestamp render_time) { - RTC_DCHECK_RUN_ON(&worker_sequence_checker_); - auto frames = buffer_->ExtractNextDecodableTemporalUnit(); - RTC_DCHECK(frames[0]->Timestamp() == rtp_timestamp) - << "Frame buffer's next decodable frame was not the one sent for " - "extraction rtp=" - << rtp_timestamp << " extracted rtp=" << frames[0]->Timestamp(); - OnFrameReady(std::move(frames), render_time); - } - - void UpdateDroppedFrames() RTC_RUN_ON(&worker_sequence_checker_) { - const int dropped_frames = buffer_->GetTotalNumberOfDroppedFrames() - - frames_dropped_before_last_new_frame_; - if (dropped_frames > 0) - stats_proxy_->OnDroppedFrames(dropped_frames); - frames_dropped_before_last_new_frame_ = - buffer_->GetTotalNumberOfDroppedFrames(); - } - - void UpdateJitterDelay() { - auto timings = timing_->GetTimings(); - if (timings.num_decoded_frames) { - stats_proxy_->OnFrameBufferTimingsUpdated( - timings.max_decode_duration.ms(), timings.current_delay.ms(), - timings.target_delay.ms(), timings.jitter_buffer_delay.ms(), - timings.min_playout_delay.ms(), timings.render_delay.ms()); - } - } - - void UpdateTimingFrameInfo() { - absl::optional info = timing_->GetTimingFrameInfo(); - if (info) - stats_proxy_->OnTimingFrameInfoUpdated(*info); - } - - bool IsTooManyFramesQueued() const RTC_RUN_ON(&worker_sequence_checker_) { - return buffer_->CurrentSize() > zero_playout_delay_max_decode_queue_size_; - } - - void ForceKeyFrameReleaseImmediately() RTC_RUN_ON(&worker_sequence_checker_) { - RTC_DCHECK(keyframe_required_); - // Iterate through the frame buffer until there is a complete keyframe and - // release this right away. - while (buffer_->DecodableTemporalUnitsInfo()) { - auto next_frame = buffer_->ExtractNextDecodableTemporalUnit(); - if (next_frame.empty()) { - RTC_DCHECK_NOTREACHED() - << "Frame buffer should always return at least 1 frame."; - continue; - } - // Found keyframe - decode right away. - if (next_frame.front()->is_keyframe()) { - auto render_time = timing_->RenderTime(next_frame.front()->Timestamp(), - clock_->CurrentTime()); - OnFrameReady(std::move(next_frame), render_time); - return; - } - } - } - - void MaybeScheduleFrameForRelease() RTC_RUN_ON(&worker_sequence_checker_) { - auto decodable_tu_info = buffer_->DecodableTemporalUnitsInfo(); - if (!decoder_ready_for_new_frame_ || !decodable_tu_info) { - return; - } - - if (keyframe_required_) { - return ForceKeyFrameReleaseImmediately(); - } - - // If already scheduled then abort. - if (frame_decode_scheduler_->ScheduledRtpTimestamp() == - decodable_tu_info->next_rtp_timestamp) { - return; - } - - TimeDelta max_wait = timeout_tracker_.TimeUntilTimeout(); - // Ensures the frame is scheduled for decode before the stream times out. - // This is otherwise a race condition. - max_wait = std::max(max_wait - TimeDelta::Millis(1), TimeDelta::Zero()); - absl::optional schedule; - while (decodable_tu_info) { - schedule = decode_timing_.OnFrameBufferUpdated( - decodable_tu_info->next_rtp_timestamp, - decodable_tu_info->last_rtp_timestamp, max_wait, - IsTooManyFramesQueued()); - if (schedule) { - // Don't schedule if already waiting for the same frame. - if (frame_decode_scheduler_->ScheduledRtpTimestamp() != - decodable_tu_info->next_rtp_timestamp) { - frame_decode_scheduler_->CancelOutstanding(); - frame_decode_scheduler_->ScheduleFrame( - decodable_tu_info->next_rtp_timestamp, *schedule, - absl::bind_front(&FrameBuffer3Proxy::FrameReadyForDecode, this)); - } - return; - } - // If no schedule for current rtp, drop and try again. - buffer_->DropNextDecodableTemporalUnit(); - decodable_tu_info = buffer_->DecodableTemporalUnitsInfo(); - } - } - - RTC_NO_UNIQUE_ADDRESS SequenceChecker worker_sequence_checker_; - const FieldTrialsView& field_trials_; - const absl::optional rtt_mult_settings_ = - RttMultExperiment::GetRttMultValue(); - Clock* const clock_; - TaskQueueBase* const worker_queue_; - TaskQueueBase* const decode_queue_; - VCMReceiveStatisticsCallback* const stats_proxy_; - FrameSchedulingReceiver* const receiver_ RTC_PT_GUARDED_BY(decode_queue_); - VCMTiming* const timing_; - const std::unique_ptr frame_decode_scheduler_ - RTC_GUARDED_BY(&worker_sequence_checker_); - - JitterEstimator jitter_estimator_ RTC_GUARDED_BY(&worker_sequence_checker_); - InterFrameDelay inter_frame_delay_ RTC_GUARDED_BY(&worker_sequence_checker_); - bool keyframe_required_ RTC_GUARDED_BY(&worker_sequence_checker_) = false; - std::unique_ptr buffer_ - RTC_GUARDED_BY(&worker_sequence_checker_); - FrameDecodeTiming decode_timing_ RTC_GUARDED_BY(&worker_sequence_checker_); - VideoReceiveStreamTimeoutTracker timeout_tracker_ - RTC_GUARDED_BY(&worker_sequence_checker_); - int frames_dropped_before_last_new_frame_ - RTC_GUARDED_BY(&worker_sequence_checker_) = 0; - VCMVideoProtection protection_mode_ - RTC_GUARDED_BY(&worker_sequence_checker_) = kProtectionNack; - - // This flag guards frames from queuing in front of the decoder. Without this - // guard, encoded frames will not wait for the decoder to finish decoding a - // frame and just queue up, meaning frames will not be dropped or - // fast-forwarded when the decoder is slow or hangs. - bool decoder_ready_for_new_frame_ RTC_GUARDED_BY(&worker_sequence_checker_) = - false; - - // Maximum number of frames in the decode queue to allow pacing. If the - // queue grows beyond the max limit, pacing will be disabled and frames will - // be pushed to the decoder as soon as possible. This only has an effect - // when the low-latency rendering path is active, which is indicated by - // the frame's render time == 0. - FieldTrialParameter zero_playout_delay_max_decode_queue_size_; - - rtc::scoped_refptr decode_safety_ = - PendingTaskSafetyFlag::CreateDetached(); - ScopedTaskSafety worker_safety_; -}; - -enum class FrameBufferArm { - kFrameBuffer3, - kSyncDecode, -}; - -constexpr const char* kFrameBufferFieldTrial = "WebRTC-FrameBuffer3"; - -FrameBufferArm ParseFrameBufferFieldTrial(const FieldTrialsView& field_trials) { - webrtc::FieldTrialEnum arm( - "arm", FrameBufferArm::kFrameBuffer3, - { - {"FrameBuffer3", FrameBufferArm::kFrameBuffer3}, - {"SyncDecoding", FrameBufferArm::kSyncDecode}, - }); - ParseFieldTrial({&arm}, field_trials.Lookup(kFrameBufferFieldTrial)); - return arm.Get(); -} - -} // namespace - -std::unique_ptr FrameBufferProxy::CreateFromFieldTrial( - Clock* clock, - TaskQueueBase* worker_queue, - VCMTiming* timing, - VCMReceiveStatisticsCallback* stats_proxy, - TaskQueueBase* decode_queue, - FrameSchedulingReceiver* receiver, - TimeDelta max_wait_for_keyframe, - TimeDelta max_wait_for_frame, - DecodeSynchronizer* decode_sync, - const FieldTrialsView& field_trials) { - switch (ParseFrameBufferFieldTrial(field_trials)) { - case FrameBufferArm::kSyncDecode: { - std::unique_ptr scheduler; - if (decode_sync) { - scheduler = decode_sync->CreateSynchronizedFrameScheduler(); - } else { - RTC_LOG(LS_ERROR) << "In FrameBuffer with sync decode trial, but " - "no DecodeSynchronizer was present!"; - // Crash in debug, but in production use the task queue scheduler. - RTC_DCHECK_NOTREACHED(); - scheduler = std::make_unique( - clock, worker_queue); - } - return std::make_unique( - clock, worker_queue, timing, stats_proxy, decode_queue, receiver, - max_wait_for_keyframe, max_wait_for_frame, std::move(scheduler), - field_trials); - } - case FrameBufferArm::kFrameBuffer3: - ABSL_FALLTHROUGH_INTENDED; - default: { - auto scheduler = - std::make_unique(clock, worker_queue); - return std::make_unique( - clock, worker_queue, timing, stats_proxy, decode_queue, receiver, - max_wait_for_keyframe, max_wait_for_frame, std::move(scheduler), - field_trials); - } - } -} - -} // namespace webrtc diff --git a/video/frame_buffer_proxy.h b/video/frame_buffer_proxy.h deleted file mode 100644 index a616d491d0..0000000000 --- a/video/frame_buffer_proxy.h +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef VIDEO_FRAME_BUFFER_PROXY_H_ -#define VIDEO_FRAME_BUFFER_PROXY_H_ - -#include - -#include "api/field_trials_view.h" -#include "api/metronome/metronome.h" -#include "api/task_queue/task_queue_base.h" -#include "api/video/encoded_frame.h" -#include "modules/video_coding/include/video_coding_defines.h" -#include "modules/video_coding/timing/timing.h" -#include "system_wrappers/include/clock.h" -#include "video/decode_synchronizer.h" - -namespace webrtc { - -class FrameSchedulingReceiver { - public: - virtual ~FrameSchedulingReceiver() = default; - - virtual void OnEncodedFrame(std::unique_ptr frame) = 0; - virtual void OnDecodableFrameTimeout(TimeDelta wait_time) = 0; -}; - -// Temporary class to enable replacement of frame_buffer2 with frame_buffer. -// Once frame_buffer has shown to work with a field trial, frame_buffer2 will -// be removed and this class should be directly integrated into -// video_receive_stream2. bugs.webrtc.org/13343 tracks this integration. -class FrameBufferProxy { - public: - static std::unique_ptr CreateFromFieldTrial( - Clock* clock, - TaskQueueBase* worker_queue, - VCMTiming* timing, - VCMReceiveStatisticsCallback* stats_proxy, - TaskQueueBase* decode_queue, - FrameSchedulingReceiver* receiver, - TimeDelta max_wait_for_keyframe, - TimeDelta max_wait_for_frame, - DecodeSynchronizer* decode_sync, - const FieldTrialsView& field_trials); - virtual ~FrameBufferProxy() = default; - - // Run on the worker thread. - virtual void StopOnWorker() = 0; - virtual void SetProtectionMode(VCMVideoProtection protection_mode) = 0; - virtual void Clear() = 0; - virtual absl::optional InsertFrame( - std::unique_ptr frame) = 0; - virtual void UpdateRtt(int64_t max_rtt_ms) = 0; - virtual int Size() = 0; - virtual void SetMaxWaits(TimeDelta max_wait_for_keyframe, - TimeDelta max_wait_for_frame) = 0; - - // Run on either the worker thread or the decode thread. - virtual void StartNextDecode(bool keyframe_required) = 0; -}; - -} // namespace webrtc - -#endif // VIDEO_FRAME_BUFFER_PROXY_H_ diff --git a/video/video_receive_stream2.cc b/video/video_receive_stream2.cc index 67720898ae..f4a53154b7 100644 --- a/video/video_receive_stream2.cc +++ b/video/video_receive_stream2.cc @@ -261,7 +261,7 @@ VideoReceiveStream2::VideoReceiveStream2( timing_->set_render_delay(TimeDelta::Millis(config_.render_delay_ms)); - frame_buffer_ = FrameBufferProxy::CreateFromFieldTrial( + buffer_ = VideoStreamBufferController::CreateFromFieldTrial( clock_, call_->worker_thread(), timing_.get(), &stats_proxy_, decode_queue_.Get(), this, max_wait_for_keyframe_, max_wait_for_frame_, decode_sync_, call_->trials()); @@ -353,7 +353,7 @@ void VideoReceiveStream2::Start() { rtp_video_stream_receiver_.ulpfec_payload_type() != -1; if (config_.rtp.nack.rtp_history_ms > 0 && protected_by_fec) { - frame_buffer_->SetProtectionMode(kProtectionNackFEC); + buffer_->SetProtectionMode(kProtectionNackFEC); } transport_adapter_.Enable(); @@ -410,7 +410,7 @@ void VideoReceiveStream2::Start() { RTC_DCHECK_RUN_ON(&decode_queue_); decoder_stopped_ = false; }); - frame_buffer_->StartNextDecode(true); + buffer_->StartNextDecode(true); decoder_running_ = true; { @@ -433,7 +433,7 @@ void VideoReceiveStream2::Stop() { stats_proxy_.OnUniqueFramesCounted( rtp_video_stream_receiver_.GetUniqueFramesSeen()); - frame_buffer_->StopOnWorker(); + buffer_->StopOnWorker(); call_stats_->DeregisterStatsObserver(this); if (decoder_running_) { rtc::Event done; @@ -534,9 +534,9 @@ void VideoReceiveStream2::SetNackHistory(TimeDelta history) { config_.rtp.protected_by_flexfec || rtp_video_stream_receiver_.ulpfec_payload_type() != -1; - frame_buffer_->SetProtectionMode(history.ms() > 0 && protected_by_fec - ? kProtectionNackFEC - : kProtectionNack); + buffer_->SetProtectionMode(history.ms() > 0 && protected_by_fec + ? kProtectionNackFEC + : kProtectionNack); rtp_video_stream_receiver_.SetNackHistory(history); TimeDelta max_wait_for_keyframe = DetermineMaxWaitForFrame(history, true); @@ -548,7 +548,7 @@ void VideoReceiveStream2::SetNackHistory(TimeDelta history) { max_wait_for_frame_ = max_wait_for_frame; }); - frame_buffer_->SetMaxWaits(max_wait_for_keyframe, max_wait_for_frame); + buffer_->SetMaxWaits(max_wait_for_keyframe, max_wait_for_frame); } void VideoReceiveStream2::SetProtectionPayloadTypes(int red_payload_type, @@ -739,7 +739,7 @@ void VideoReceiveStream2::OnCompleteFrame(std::unique_ptr frame) { UpdatePlayoutDelays(); } - auto last_continuous_pid = frame_buffer_->InsertFrame(std::move(frame)); + auto last_continuous_pid = buffer_->InsertFrame(std::move(frame)); if (last_continuous_pid.has_value()) { { // TODO(bugs.webrtc.org/11993): Call on the network thread. @@ -752,7 +752,7 @@ void VideoReceiveStream2::OnCompleteFrame(std::unique_ptr frame) { void VideoReceiveStream2::OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms) { RTC_DCHECK_RUN_ON(&worker_sequence_checker_); // TODO(bugs.webrtc.org/13757): Replace with TimeDelta. - frame_buffer_->UpdateRtt(max_rtt_ms); + buffer_->UpdateRtt(max_rtt_ms); rtp_video_stream_receiver_.UpdateRtt(max_rtt_ms); stats_proxy_.OnRttUpdate(avg_rtt_ms); } @@ -803,7 +803,7 @@ void VideoReceiveStream2::OnEncodedFrame(std::unique_ptr frame) { if (decoder_stopped_) return; HandleEncodedFrame(std::move(frame)); - frame_buffer_->StartNextDecode(keyframe_required_); + buffer_->StartNextDecode(keyframe_required_); } void VideoReceiveStream2::OnDecodableFrameTimeout(TimeDelta wait_time) { @@ -818,7 +818,7 @@ void VideoReceiveStream2::OnDecodableFrameTimeout(TimeDelta wait_time) { decode_queue_.PostTask([this] { RTC_DCHECK_RUN_ON(&decode_queue_); - frame_buffer_->StartNextDecode(keyframe_required_); + buffer_->StartNextDecode(keyframe_required_); }); })); } @@ -1059,7 +1059,7 @@ void VideoReceiveStream2::UpdatePlayoutDelays() const { std::lrint(*frame_maximum_playout_delay_ * kFrameRate); // Subtract frames in buffer. max_composition_delay_in_frames = - std::max(max_composition_delay_in_frames - frame_buffer_->Size(), 0); + std::max(max_composition_delay_in_frames - buffer_->Size(), 0); timing_->SetMaxCompositionDelayInFrames(max_composition_delay_in_frames); } } diff --git a/video/video_receive_stream2.h b/video/video_receive_stream2.h index a14638c075..4420798218 100644 --- a/video/video_receive_stream2.h +++ b/video/video_receive_stream2.h @@ -34,11 +34,11 @@ #include "rtc_base/task_queue.h" #include "rtc_base/thread_annotations.h" #include "system_wrappers/include/clock.h" -#include "video/frame_buffer_proxy.h" #include "video/receive_statistics_proxy2.h" #include "video/rtp_streams_synchronizer2.h" #include "video/rtp_video_stream_receiver2.h" #include "video/transport_adapter.h" +#include "video/video_stream_buffer_controller.h" #include "video/video_stream_decoder2.h" namespace webrtc { @@ -262,7 +262,7 @@ class VideoReceiveStream2 // moved to the new VideoStreamDecoder. std::vector> video_decoders_; - std::unique_ptr frame_buffer_; + std::unique_ptr buffer_; std::unique_ptr media_receiver_ RTC_GUARDED_BY(packet_sequence_checker_); diff --git a/video/video_stream_buffer_controller.cc b/video/video_stream_buffer_controller.cc new file mode 100644 index 0000000000..e642578c29 --- /dev/null +++ b/video/video_stream_buffer_controller.cc @@ -0,0 +1,461 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/video_stream_buffer_controller.h" + +#include +#include +#include + +#include "absl/base/attributes.h" +#include "absl/functional/bind_front.h" +#include "api/sequence_checker.h" +#include "api/task_queue/task_queue_base.h" +#include "api/units/data_size.h" +#include "api/video/encoded_frame.h" +#include "api/video/frame_buffer.h" +#include "api/video/video_content_type.h" +#include "modules/video_coding/frame_helpers.h" +#include "modules/video_coding/timing/inter_frame_delay.h" +#include "modules/video_coding/timing/jitter_estimator.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/thread_annotations.h" +#include "video/frame_decode_timing.h" +#include "video/task_queue_frame_decode_scheduler.h" +#include "video/video_receive_stream_timeout_tracker.h" + +namespace webrtc { + +namespace { + +// Max number of frames the buffer will hold. +static constexpr size_t kMaxFramesBuffered = 800; +// Max number of decoded frame info that will be saved. +static constexpr int kMaxFramesHistory = 1 << 13; + +// Default value for the maximum decode queue size that is used when the +// low-latency renderer is used. +static constexpr size_t kZeroPlayoutDelayDefaultMaxDecodeQueueSize = 8; + +struct FrameMetadata { + explicit FrameMetadata(const EncodedFrame& frame) + : is_last_spatial_layer(frame.is_last_spatial_layer), + is_keyframe(frame.is_keyframe()), + size(frame.size()), + contentType(frame.contentType()), + delayed_by_retransmission(frame.delayed_by_retransmission()), + rtp_timestamp(frame.Timestamp()), + receive_time(frame.ReceivedTimestamp()) {} + + const bool is_last_spatial_layer; + const bool is_keyframe; + const size_t size; + const VideoContentType contentType; + const bool delayed_by_retransmission; + const uint32_t rtp_timestamp; + const absl::optional receive_time; +}; + +Timestamp ReceiveTime(const EncodedFrame& frame) { + absl::optional ts = frame.ReceivedTimestamp(); + RTC_DCHECK(ts.has_value()) << "Received frame must have a timestamp set!"; + return *ts; +} + +enum class FrameBufferArm { + kFrameBuffer3, + kSyncDecode, +}; + +constexpr const char* kFrameBufferFieldTrial = "WebRTC-FrameBuffer3"; + +FrameBufferArm ParseFrameBufferFieldTrial(const FieldTrialsView& field_trials) { + webrtc::FieldTrialEnum arm( + "arm", FrameBufferArm::kFrameBuffer3, + { + {"FrameBuffer3", FrameBufferArm::kFrameBuffer3}, + {"SyncDecoding", FrameBufferArm::kSyncDecode}, + }); + ParseFieldTrial({&arm}, field_trials.Lookup(kFrameBufferFieldTrial)); + return arm.Get(); +} + +} // namespace + +std::unique_ptr +VideoStreamBufferController::CreateFromFieldTrial( + Clock* clock, + TaskQueueBase* worker_queue, + VCMTiming* timing, + VCMReceiveStatisticsCallback* stats_proxy, + TaskQueueBase* decode_queue, + FrameSchedulingReceiver* receiver, + TimeDelta max_wait_for_keyframe, + TimeDelta max_wait_for_frame, + DecodeSynchronizer* decode_sync, + const FieldTrialsView& field_trials) { + switch (ParseFrameBufferFieldTrial(field_trials)) { + case FrameBufferArm::kSyncDecode: { + std::unique_ptr scheduler; + if (decode_sync) { + scheduler = decode_sync->CreateSynchronizedFrameScheduler(); + } else { + RTC_LOG(LS_ERROR) << "In FrameBuffer with sync decode trial, but " + "no DecodeSynchronizer was present!"; + // Crash in debug, but in production use the task queue scheduler. + RTC_DCHECK_NOTREACHED(); + scheduler = std::make_unique( + clock, worker_queue); + } + return std::make_unique( + clock, worker_queue, timing, stats_proxy, decode_queue, receiver, + max_wait_for_keyframe, max_wait_for_frame, std::move(scheduler), + field_trials); + } + case FrameBufferArm::kFrameBuffer3: + ABSL_FALLTHROUGH_INTENDED; + default: { + auto scheduler = + std::make_unique(clock, worker_queue); + return std::make_unique( + clock, worker_queue, timing, stats_proxy, decode_queue, receiver, + max_wait_for_keyframe, max_wait_for_frame, std::move(scheduler), + field_trials); + } + } +} + +VideoStreamBufferController::VideoStreamBufferController( + Clock* clock, + TaskQueueBase* worker_queue, + VCMTiming* timing, + VCMReceiveStatisticsCallback* stats_proxy, + TaskQueueBase* decode_queue, + FrameSchedulingReceiver* receiver, + TimeDelta max_wait_for_keyframe, + TimeDelta max_wait_for_frame, + std::unique_ptr frame_decode_scheduler, + const FieldTrialsView& field_trials) + : field_trials_(field_trials), + clock_(clock), + worker_queue_(worker_queue), + decode_queue_(decode_queue), + stats_proxy_(stats_proxy), + receiver_(receiver), + timing_(timing), + frame_decode_scheduler_(std::move(frame_decode_scheduler)), + jitter_estimator_(clock_, field_trials), + buffer_(std::make_unique(kMaxFramesBuffered, + kMaxFramesHistory, + field_trials)), + decode_timing_(clock_, timing_), + timeout_tracker_( + clock_, + worker_queue_, + VideoReceiveStreamTimeoutTracker::Timeouts{ + .max_wait_for_keyframe = max_wait_for_keyframe, + .max_wait_for_frame = max_wait_for_frame}, + absl::bind_front(&VideoStreamBufferController::OnTimeout, this)), + zero_playout_delay_max_decode_queue_size_( + "max_decode_queue_size", + kZeroPlayoutDelayDefaultMaxDecodeQueueSize) { + RTC_DCHECK(decode_queue_); + RTC_DCHECK(stats_proxy_); + RTC_DCHECK(receiver_); + RTC_DCHECK(timing_); + RTC_DCHECK(worker_queue_); + RTC_DCHECK(clock_); + RTC_DCHECK(frame_decode_scheduler_); + RTC_LOG(LS_WARNING) << "Using FrameBuffer3"; + + ParseFieldTrial({&zero_playout_delay_max_decode_queue_size_}, + field_trials.Lookup("WebRTC-ZeroPlayoutDelay")); +} + +void VideoStreamBufferController::StopOnWorker() { + RTC_DCHECK_RUN_ON(&worker_sequence_checker_); + frame_decode_scheduler_->Stop(); + timeout_tracker_.Stop(); + decoder_ready_for_new_frame_ = false; + decode_queue_->PostTask([this] { + RTC_DCHECK_RUN_ON(decode_queue_); + decode_safety_->SetNotAlive(); + }); +} + +void VideoStreamBufferController::SetProtectionMode( + VCMVideoProtection protection_mode) { + RTC_DCHECK_RUN_ON(&worker_sequence_checker_); + protection_mode_ = protection_mode; +} + +void VideoStreamBufferController::Clear() { + RTC_DCHECK_RUN_ON(&worker_sequence_checker_); + stats_proxy_->OnDroppedFrames(buffer_->CurrentSize()); + buffer_ = std::make_unique(kMaxFramesBuffered, kMaxFramesHistory, + field_trials_); + frame_decode_scheduler_->CancelOutstanding(); +} + +absl::optional VideoStreamBufferController::InsertFrame( + std::unique_ptr frame) { + RTC_DCHECK_RUN_ON(&worker_sequence_checker_); + FrameMetadata metadata(*frame); + int complete_units = buffer_->GetTotalNumberOfContinuousTemporalUnits(); + if (buffer_->InsertFrame(std::move(frame))) { + RTC_DCHECK(metadata.receive_time) << "Frame receive time must be set!"; + if (!metadata.delayed_by_retransmission && metadata.receive_time) + timing_->IncomingTimestamp(metadata.rtp_timestamp, + *metadata.receive_time); + if (complete_units < buffer_->GetTotalNumberOfContinuousTemporalUnits()) { + stats_proxy_->OnCompleteFrame(metadata.is_keyframe, metadata.size, + metadata.contentType); + MaybeScheduleFrameForRelease(); + } + } + + return buffer_->LastContinuousFrameId(); +} + +void VideoStreamBufferController::UpdateRtt(int64_t max_rtt_ms) { + RTC_DCHECK_RUN_ON(&worker_sequence_checker_); + jitter_estimator_.UpdateRtt(TimeDelta::Millis(max_rtt_ms)); +} + +void VideoStreamBufferController::SetMaxWaits(TimeDelta max_wait_for_keyframe, + TimeDelta max_wait_for_frame) { + RTC_DCHECK_RUN_ON(&worker_sequence_checker_); + timeout_tracker_.SetTimeouts({.max_wait_for_keyframe = max_wait_for_keyframe, + .max_wait_for_frame = max_wait_for_frame}); +} + +void VideoStreamBufferController::StartNextDecode(bool keyframe_required) { + if (!worker_queue_->IsCurrent()) { + worker_queue_->PostTask(SafeTask( + worker_safety_.flag(), + [this, keyframe_required] { StartNextDecode(keyframe_required); })); + return; + } + + RTC_DCHECK_RUN_ON(&worker_sequence_checker_); + if (!timeout_tracker_.Running()) + timeout_tracker_.Start(keyframe_required); + keyframe_required_ = keyframe_required; + if (keyframe_required_) { + timeout_tracker_.SetWaitingForKeyframe(); + } + decoder_ready_for_new_frame_ = true; + MaybeScheduleFrameForRelease(); +} + +int VideoStreamBufferController::Size() { + RTC_DCHECK_RUN_ON(&worker_sequence_checker_); + return buffer_->CurrentSize(); +} + +void VideoStreamBufferController::OnFrameReady( + absl::InlinedVector, 4> frames, + Timestamp render_time) { + RTC_DCHECK_RUN_ON(&worker_sequence_checker_); + RTC_DCHECK(!frames.empty()); + + timeout_tracker_.OnEncodedFrameReleased(); + + Timestamp now = clock_->CurrentTime(); + bool superframe_delayed_by_retransmission = false; + DataSize superframe_size = DataSize::Zero(); + const EncodedFrame& first_frame = *frames.front(); + Timestamp receive_time = ReceiveTime(first_frame); + + if (first_frame.is_keyframe()) + keyframe_required_ = false; + + // Gracefully handle bad RTP timestamps and render time issues. + if (FrameHasBadRenderTiming(render_time, now, timing_->TargetVideoDelay())) { + jitter_estimator_.Reset(); + timing_->Reset(); + render_time = timing_->RenderTime(first_frame.Timestamp(), now); + } + + for (std::unique_ptr& frame : frames) { + frame->SetRenderTime(render_time.ms()); + + superframe_delayed_by_retransmission |= frame->delayed_by_retransmission(); + receive_time = std::max(receive_time, ReceiveTime(*frame)); + superframe_size += DataSize::Bytes(frame->size()); + } + + if (!superframe_delayed_by_retransmission) { + auto frame_delay = inter_frame_delay_.CalculateDelay( + first_frame.Timestamp(), receive_time); + if (frame_delay) { + jitter_estimator_.UpdateEstimate(*frame_delay, superframe_size); + } + + float rtt_mult = protection_mode_ == kProtectionNackFEC ? 0.0 : 1.0; + absl::optional rtt_mult_add_cap_ms = absl::nullopt; + if (rtt_mult_settings_.has_value()) { + rtt_mult = rtt_mult_settings_->rtt_mult_setting; + rtt_mult_add_cap_ms = + TimeDelta::Millis(rtt_mult_settings_->rtt_mult_add_cap_ms); + } + timing_->SetJitterDelay( + jitter_estimator_.GetJitterEstimate(rtt_mult, rtt_mult_add_cap_ms)); + timing_->UpdateCurrentDelay(render_time, now); + } else if (RttMultExperiment::RttMultEnabled()) { + jitter_estimator_.FrameNacked(); + } + + // Update stats. + UpdateDroppedFrames(); + UpdateJitterDelay(); + UpdateTimingFrameInfo(); + + std::unique_ptr frame = + CombineAndDeleteFrames(std::move(frames)); + + timing_->SetLastDecodeScheduledTimestamp(now); + + decoder_ready_for_new_frame_ = false; + // VideoReceiveStream2 wants frames on the decoder thread. + decode_queue_->PostTask( + SafeTask(decode_safety_, [this, frame = std::move(frame)]() mutable { + RTC_DCHECK_RUN_ON(decode_queue_); + receiver_->OnEncodedFrame(std::move(frame)); + })); +} + +void VideoStreamBufferController::OnTimeout(TimeDelta delay) { + RTC_DCHECK_RUN_ON(&worker_sequence_checker_); + // If the stream is paused then ignore the timeout. + if (!decoder_ready_for_new_frame_) { + timeout_tracker_.Stop(); + return; + } + decode_queue_->PostTask(SafeTask(decode_safety_, [this, delay]() { + RTC_DCHECK_RUN_ON(decode_queue_); + receiver_->OnDecodableFrameTimeout(delay); + })); + // Stop sending timeouts until receive starts waiting for a new frame. + timeout_tracker_.Stop(); + decoder_ready_for_new_frame_ = false; +} + +void VideoStreamBufferController::FrameReadyForDecode(uint32_t rtp_timestamp, + Timestamp render_time) { + RTC_DCHECK_RUN_ON(&worker_sequence_checker_); + auto frames = buffer_->ExtractNextDecodableTemporalUnit(); + RTC_DCHECK(frames[0]->Timestamp() == rtp_timestamp) + << "Frame buffer's next decodable frame was not the one sent for " + "extraction rtp=" + << rtp_timestamp << " extracted rtp=" << frames[0]->Timestamp(); + OnFrameReady(std::move(frames), render_time); +} + +void VideoStreamBufferController::UpdateDroppedFrames() + RTC_RUN_ON(&worker_sequence_checker_) { + const int dropped_frames = buffer_->GetTotalNumberOfDroppedFrames() - + frames_dropped_before_last_new_frame_; + if (dropped_frames > 0) + stats_proxy_->OnDroppedFrames(dropped_frames); + frames_dropped_before_last_new_frame_ = + buffer_->GetTotalNumberOfDroppedFrames(); +} + +void VideoStreamBufferController::UpdateJitterDelay() { + auto timings = timing_->GetTimings(); + if (timings.num_decoded_frames) { + stats_proxy_->OnFrameBufferTimingsUpdated( + timings.max_decode_duration.ms(), timings.current_delay.ms(), + timings.target_delay.ms(), timings.jitter_buffer_delay.ms(), + timings.min_playout_delay.ms(), timings.render_delay.ms()); + } +} + +void VideoStreamBufferController::UpdateTimingFrameInfo() { + absl::optional info = timing_->GetTimingFrameInfo(); + if (info) + stats_proxy_->OnTimingFrameInfoUpdated(*info); +} + +bool VideoStreamBufferController::IsTooManyFramesQueued() const + RTC_RUN_ON(&worker_sequence_checker_) { + return buffer_->CurrentSize() > zero_playout_delay_max_decode_queue_size_; +} + +void VideoStreamBufferController::ForceKeyFrameReleaseImmediately() + RTC_RUN_ON(&worker_sequence_checker_) { + RTC_DCHECK(keyframe_required_); + // Iterate through the frame buffer until there is a complete keyframe and + // release this right away. + while (buffer_->DecodableTemporalUnitsInfo()) { + auto next_frame = buffer_->ExtractNextDecodableTemporalUnit(); + if (next_frame.empty()) { + RTC_DCHECK_NOTREACHED() + << "Frame buffer should always return at least 1 frame."; + continue; + } + // Found keyframe - decode right away. + if (next_frame.front()->is_keyframe()) { + auto render_time = timing_->RenderTime(next_frame.front()->Timestamp(), + clock_->CurrentTime()); + OnFrameReady(std::move(next_frame), render_time); + return; + } + } +} + +void VideoStreamBufferController::MaybeScheduleFrameForRelease() + RTC_RUN_ON(&worker_sequence_checker_) { + auto decodable_tu_info = buffer_->DecodableTemporalUnitsInfo(); + if (!decoder_ready_for_new_frame_ || !decodable_tu_info) { + return; + } + + if (keyframe_required_) { + return ForceKeyFrameReleaseImmediately(); + } + + // If already scheduled then abort. + if (frame_decode_scheduler_->ScheduledRtpTimestamp() == + decodable_tu_info->next_rtp_timestamp) { + return; + } + + TimeDelta max_wait = timeout_tracker_.TimeUntilTimeout(); + // Ensures the frame is scheduled for decode before the stream times out. + // This is otherwise a race condition. + max_wait = std::max(max_wait - TimeDelta::Millis(1), TimeDelta::Zero()); + absl::optional schedule; + while (decodable_tu_info) { + schedule = decode_timing_.OnFrameBufferUpdated( + decodable_tu_info->next_rtp_timestamp, + decodable_tu_info->last_rtp_timestamp, max_wait, + IsTooManyFramesQueued()); + if (schedule) { + // Don't schedule if already waiting for the same frame. + if (frame_decode_scheduler_->ScheduledRtpTimestamp() != + decodable_tu_info->next_rtp_timestamp) { + frame_decode_scheduler_->CancelOutstanding(); + frame_decode_scheduler_->ScheduleFrame( + decodable_tu_info->next_rtp_timestamp, *schedule, + absl::bind_front(&VideoStreamBufferController::FrameReadyForDecode, + this)); + } + return; + } + // If no schedule for current rtp, drop and try again. + buffer_->DropNextDecodableTemporalUnit(); + decodable_tu_info = buffer_->DecodableTemporalUnitsInfo(); + } +} + +} // namespace webrtc diff --git a/video/video_stream_buffer_controller.h b/video/video_stream_buffer_controller.h new file mode 100644 index 0000000000..f592c8c582 --- /dev/null +++ b/video/video_stream_buffer_controller.h @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_VIDEO_STREAM_BUFFER_CONTROLLER_H_ +#define VIDEO_VIDEO_STREAM_BUFFER_CONTROLLER_H_ + +#include + +#include "api/field_trials_view.h" +#include "api/task_queue/task_queue_base.h" +#include "api/video/encoded_frame.h" +#include "api/video/frame_buffer.h" +#include "modules/video_coding/include/video_coding_defines.h" +#include "modules/video_coding/timing/inter_frame_delay.h" +#include "modules/video_coding/timing/jitter_estimator.h" +#include "modules/video_coding/timing/timing.h" +#include "rtc_base/experiments/rtt_mult_experiment.h" +#include "system_wrappers/include/clock.h" +#include "video/decode_synchronizer.h" +#include "video/video_receive_stream_timeout_tracker.h" + +namespace webrtc { + +class FrameSchedulingReceiver { + public: + virtual ~FrameSchedulingReceiver() = default; + + virtual void OnEncodedFrame(std::unique_ptr frame) = 0; + virtual void OnDecodableFrameTimeout(TimeDelta wait_time) = 0; +}; + +class VideoStreamBufferController { + public: + static std::unique_ptr CreateFromFieldTrial( + Clock* clock, + TaskQueueBase* worker_queue, + VCMTiming* timing, + VCMReceiveStatisticsCallback* stats_proxy, + TaskQueueBase* decode_queue, + FrameSchedulingReceiver* receiver, + TimeDelta max_wait_for_keyframe, + TimeDelta max_wait_for_frame, + DecodeSynchronizer* decode_sync, + const FieldTrialsView& field_trials); + virtual ~VideoStreamBufferController() = default; + VideoStreamBufferController( + Clock* clock, + TaskQueueBase* worker_queue, + VCMTiming* timing, + VCMReceiveStatisticsCallback* stats_proxy, + TaskQueueBase* decode_queue, + FrameSchedulingReceiver* receiver, + TimeDelta max_wait_for_keyframe, + TimeDelta max_wait_for_frame, + std::unique_ptr frame_decode_scheduler, + const FieldTrialsView& field_trials); + + void StopOnWorker(); + void SetProtectionMode(VCMVideoProtection protection_mode); + void Clear(); + absl::optional InsertFrame(std::unique_ptr frame); + void UpdateRtt(int64_t max_rtt_ms); + void SetMaxWaits(TimeDelta max_wait_for_keyframe, + TimeDelta max_wait_for_frame); + void StartNextDecode(bool keyframe_required); + int Size(); + + private: + void OnFrameReady( + absl::InlinedVector, 4> frames, + Timestamp render_time); + void OnTimeout(TimeDelta delay); + void FrameReadyForDecode(uint32_t rtp_timestamp, Timestamp render_time); + void UpdateDroppedFrames() RTC_RUN_ON(&worker_sequence_checker_); + void UpdateJitterDelay(); + void UpdateTimingFrameInfo(); + bool IsTooManyFramesQueued() const RTC_RUN_ON(&worker_sequence_checker_); + void ForceKeyFrameReleaseImmediately() RTC_RUN_ON(&worker_sequence_checker_); + void MaybeScheduleFrameForRelease() RTC_RUN_ON(&worker_sequence_checker_); + + RTC_NO_UNIQUE_ADDRESS SequenceChecker worker_sequence_checker_; + const FieldTrialsView& field_trials_; + const absl::optional rtt_mult_settings_ = + RttMultExperiment::GetRttMultValue(); + Clock* const clock_; + TaskQueueBase* const worker_queue_; + TaskQueueBase* const decode_queue_; + VCMReceiveStatisticsCallback* const stats_proxy_; + FrameSchedulingReceiver* const receiver_ RTC_PT_GUARDED_BY(decode_queue_); + VCMTiming* const timing_; + const std::unique_ptr frame_decode_scheduler_ + RTC_GUARDED_BY(&worker_sequence_checker_); + + JitterEstimator jitter_estimator_ RTC_GUARDED_BY(&worker_sequence_checker_); + InterFrameDelay inter_frame_delay_ RTC_GUARDED_BY(&worker_sequence_checker_); + bool keyframe_required_ RTC_GUARDED_BY(&worker_sequence_checker_) = false; + std::unique_ptr buffer_ + RTC_GUARDED_BY(&worker_sequence_checker_); + FrameDecodeTiming decode_timing_ RTC_GUARDED_BY(&worker_sequence_checker_); + VideoReceiveStreamTimeoutTracker timeout_tracker_ + RTC_GUARDED_BY(&worker_sequence_checker_); + int frames_dropped_before_last_new_frame_ + RTC_GUARDED_BY(&worker_sequence_checker_) = 0; + VCMVideoProtection protection_mode_ + RTC_GUARDED_BY(&worker_sequence_checker_) = kProtectionNack; + + // This flag guards frames from queuing in front of the decoder. Without this + // guard, encoded frames will not wait for the decoder to finish decoding a + // frame and just queue up, meaning frames will not be dropped or + // fast-forwarded when the decoder is slow or hangs. + bool decoder_ready_for_new_frame_ RTC_GUARDED_BY(&worker_sequence_checker_) = + false; + + // Maximum number of frames in the decode queue to allow pacing. If the + // queue grows beyond the max limit, pacing will be disabled and frames will + // be pushed to the decoder as soon as possible. This only has an effect + // when the low-latency rendering path is active, which is indicated by + // the frame's render time == 0. + FieldTrialParameter zero_playout_delay_max_decode_queue_size_; + + rtc::scoped_refptr decode_safety_ = + PendingTaskSafetyFlag::CreateDetached(); + ScopedTaskSafety worker_safety_; +}; + +} // namespace webrtc + +#endif // VIDEO_VIDEO_STREAM_BUFFER_CONTROLLER_H_ diff --git a/video/frame_buffer_proxy_unittest.cc b/video/video_stream_buffer_controller_unittest.cc similarity index 68% rename from video/frame_buffer_proxy_unittest.cc rename to video/video_stream_buffer_controller_unittest.cc index 3295fac16a..81dae65210 100644 --- a/video/frame_buffer_proxy_unittest.cc +++ b/video/video_stream_buffer_controller_unittest.cc @@ -8,7 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "video/frame_buffer_proxy.h" +#include "video/video_stream_buffer_controller.h" #include @@ -107,11 +107,11 @@ class VCMReceiveStatisticsCallbackMock : public VCMReceiveStatisticsCallback { constexpr auto kMaxWaitForKeyframe = TimeDelta::Millis(500); constexpr auto kMaxWaitForFrame = TimeDelta::Millis(1500); -class FrameBufferProxyFixture +class VideoStreamBufferControllerFixture : public ::testing::WithParamInterface, public FrameSchedulingReceiver { public: - FrameBufferProxyFixture() + VideoStreamBufferControllerFixture() : field_trials_(GetParam()), time_controller_(kClockStart), clock_(time_controller_.GetClock()), @@ -122,16 +122,17 @@ class FrameBufferProxyFixture TimeDelta::Millis(16)), decode_sync_(clock_, &fake_metronome_, run_loop_.task_queue()), timing_(clock_, field_trials_), - proxy_(FrameBufferProxy::CreateFromFieldTrial(clock_, - run_loop_.task_queue(), - &timing_, - &stats_callback_, - decode_queue_.Get(), - this, - kMaxWaitForKeyframe, - kMaxWaitForFrame, - &decode_sync_, - field_trials_)) { + buffer_(VideoStreamBufferController::CreateFromFieldTrial( + clock_, + run_loop_.task_queue(), + &timing_, + &stats_callback_, + decode_queue_.Get(), + this, + kMaxWaitForKeyframe, + kMaxWaitForFrame, + &decode_sync_, + field_trials_)) { // Avoid starting with negative render times. timing_.set_min_playout_delay(TimeDelta::Millis(10)); @@ -140,9 +141,9 @@ class FrameBufferProxyFixture [this](auto num_dropped) { dropped_frames_ += num_dropped; }); } - ~FrameBufferProxyFixture() override { - if (proxy_) { - proxy_->StopOnWorker(); + ~VideoStreamBufferControllerFixture() override { + if (buffer_) { + buffer_->StopOnWorker(); } fake_metronome_.Stop(); time_controller_.AdvanceTime(TimeDelta::Zero()); @@ -187,13 +188,13 @@ class FrameBufferProxyFixture void StartNextDecode() { ResetLastResult(); - proxy_->StartNextDecode(false); + buffer_->StartNextDecode(false); time_controller_.AdvanceTime(TimeDelta::Zero()); } void StartNextDecodeForceKeyframe() { ResetLastResult(); - proxy_->StartNextDecode(true); + buffer_->StartNextDecode(true); time_controller_.AdvanceTime(TimeDelta::Zero()); } @@ -212,7 +213,7 @@ class FrameBufferProxyFixture VCMTiming timing_; ::testing::NiceMock stats_callback_; - std::unique_ptr proxy_; + std::unique_ptr buffer_; private: void SetWaitResult(WaitResult result) { @@ -228,10 +229,12 @@ class FrameBufferProxyFixture absl::optional wait_result_; }; -class FrameBufferProxyTest : public ::testing::Test, - public FrameBufferProxyFixture {}; +class VideoStreamBufferControllerTest + : public ::testing::Test, + public VideoStreamBufferControllerFixture {}; -TEST_P(FrameBufferProxyTest, InitialTimeoutAfterKeyframeTimeoutPeriod) { +TEST_P(VideoStreamBufferControllerTest, + InitialTimeoutAfterKeyframeTimeoutPeriod) { StartNextDecodeForceKeyframe(); // No frame inserted. Timeout expected. EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForKeyframe), TimedOut()); @@ -245,22 +248,23 @@ TEST_P(FrameBufferProxyTest, InitialTimeoutAfterKeyframeTimeoutPeriod) { EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForKeyframe), TimedOut()); } -TEST_P(FrameBufferProxyTest, KeyFramesAreScheduled) { +TEST_P(VideoStreamBufferControllerTest, KeyFramesAreScheduled) { StartNextDecodeForceKeyframe(); time_controller_.AdvanceTime(TimeDelta::Millis(50)); auto frame = test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build(); - proxy_->InsertFrame(std::move(frame)); + buffer_->InsertFrame(std::move(frame)); EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); } -TEST_P(FrameBufferProxyTest, DeltaFrameTimeoutAfterKeyframeExtracted) { +TEST_P(VideoStreamBufferControllerTest, + DeltaFrameTimeoutAfterKeyframeExtracted) { StartNextDecodeForceKeyframe(); time_controller_.AdvanceTime(TimeDelta::Millis(50)); auto frame = test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build(); - proxy_->InsertFrame(std::move(frame)); + buffer_->InsertFrame(std::move(frame)); EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForKeyframe), Frame(test::WithId(0))); @@ -275,45 +279,45 @@ TEST_P(FrameBufferProxyTest, DeltaFrameTimeoutAfterKeyframeExtracted) { } } -TEST_P(FrameBufferProxyTest, DependantFramesAreScheduled) { +TEST_P(VideoStreamBufferControllerTest, DependantFramesAreScheduled) { StartNextDecodeForceKeyframe(); - proxy_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); + buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); StartNextDecode(); time_controller_.AdvanceTime(kFps30Delay); - proxy_->InsertFrame(test::FakeFrameBuilder() - .Id(1) - .Time(kFps30Rtp) - .AsLast() - .Refs({0}) - .Build()); + buffer_->InsertFrame(test::FakeFrameBuilder() + .Id(1) + .Time(kFps30Rtp) + .AsLast() + .Refs({0}) + .Build()); EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(1))); } -TEST_P(FrameBufferProxyTest, SpatialLayersAreScheduled) { +TEST_P(VideoStreamBufferControllerTest, SpatialLayersAreScheduled) { StartNextDecodeForceKeyframe(); - proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp( + buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( test::FakeFrameBuilder().Id(0).SpatialLayer(0).Time(0).Build())); - proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp( + buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( test::FakeFrameBuilder().Id(1).SpatialLayer(1).Time(0).Build())); - proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp( + buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( test::FakeFrameBuilder().Id(2).SpatialLayer(2).Time(0).AsLast().Build())); EXPECT_THAT( WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(AllOf(test::WithId(0), test::FrameWithSize(3 * kFrameSize)))); - proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp( + buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( test::FakeFrameBuilder().Id(3).Time(kFps30Rtp).SpatialLayer(0).Build())); - proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp( + buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( test::FakeFrameBuilder().Id(4).Time(kFps30Rtp).SpatialLayer(1).Build())); - proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder() - .Id(5) - .Time(kFps30Rtp) - .SpatialLayer(2) - .AsLast() - .Build())); + buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder() + .Id(5) + .Time(kFps30Rtp) + .SpatialLayer(2) + .AsLast() + .Build())); StartNextDecode(); EXPECT_THAT( @@ -321,41 +325,42 @@ TEST_P(FrameBufferProxyTest, SpatialLayersAreScheduled) { Frame(AllOf(test::WithId(3), test::FrameWithSize(3 * kFrameSize)))); } -TEST_P(FrameBufferProxyTest, OutstandingFrameTasksAreCancelledAfterDeletion) { +TEST_P(VideoStreamBufferControllerTest, + OutstandingFrameTasksAreCancelledAfterDeletion) { StartNextDecodeForceKeyframe(); - proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp( + buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build())); // Get keyframe. Delta frame should now be scheduled. EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); StartNextDecode(); - proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder() - .Id(1) - .Time(kFps30Rtp) - .AsLast() - .Refs({0}) - .Build())); - proxy_->StopOnWorker(); + buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder() + .Id(1) + .Time(kFps30Rtp) + .AsLast() + .Refs({0}) + .Build())); + buffer_->StopOnWorker(); // Wait for 2x max wait time. Since we stopped, this should cause no timeouts // or frame-ready callbacks. EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame * 2), Eq(absl::nullopt)); } -TEST_P(FrameBufferProxyTest, FramesWaitForDecoderToComplete) { +TEST_P(VideoStreamBufferControllerTest, FramesWaitForDecoderToComplete) { StartNextDecodeForceKeyframe(); // Start with a keyframe. - proxy_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); + buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); ResetLastResult(); // Insert a delta frame. - proxy_->InsertFrame(test::FakeFrameBuilder() - .Id(1) - .Time(kFps30Rtp) - .AsLast() - .Refs({0}) - .Build()); + buffer_->InsertFrame(test::FakeFrameBuilder() + .Id(1) + .Time(kFps30Rtp) + .AsLast() + .Refs({0}) + .Build()); // Advancing time should not result in a frame since the scheduler has not // been signalled that we are ready. @@ -365,12 +370,12 @@ TEST_P(FrameBufferProxyTest, FramesWaitForDecoderToComplete) { EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(1))); } -TEST_P(FrameBufferProxyTest, LateFrameDropped) { +TEST_P(VideoStreamBufferControllerTest, LateFrameDropped) { StartNextDecodeForceKeyframe(); // F1 // / // F0 --> F2 - proxy_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); + buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); // Start with a keyframe. EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); @@ -378,50 +383,50 @@ TEST_P(FrameBufferProxyTest, LateFrameDropped) { // Simulate late F1 which arrives after F2. time_controller_.AdvanceTime(kFps30Delay * 2); - proxy_->InsertFrame(test::FakeFrameBuilder() - .Id(2) - .Time(2 * kFps30Rtp) - .AsLast() - .Refs({0}) - .Build()); + buffer_->InsertFrame(test::FakeFrameBuilder() + .Id(2) + .Time(2 * kFps30Rtp) + .AsLast() + .Refs({0}) + .Build()); EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(2))); StartNextDecode(); - proxy_->InsertFrame(test::FakeFrameBuilder() - .Id(1) - .Time(1 * kFps30Rtp) - .AsLast() - .Refs({0}) - .Build()); + buffer_->InsertFrame(test::FakeFrameBuilder() + .Id(1) + .Time(1 * kFps30Rtp) + .AsLast() + .Refs({0}) + .Build()); // Confirm frame 1 is never scheduled by timing out. EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), TimedOut()); } -TEST_P(FrameBufferProxyTest, FramesFastForwardOnSystemHalt) { +TEST_P(VideoStreamBufferControllerTest, FramesFastForwardOnSystemHalt) { StartNextDecodeForceKeyframe(); // F1 // / // F0 --> F2 - proxy_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); + buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); // Start with a keyframe. EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); time_controller_.AdvanceTime(kFps30Delay); - proxy_->InsertFrame(test::FakeFrameBuilder() - .Id(1) - .Time(kFps30Rtp) - .AsLast() - .Refs({0}) - .Build()); + buffer_->InsertFrame(test::FakeFrameBuilder() + .Id(1) + .Time(kFps30Rtp) + .AsLast() + .Refs({0}) + .Build()); time_controller_.AdvanceTime(kFps30Delay); - proxy_->InsertFrame(test::FakeFrameBuilder() - .Id(2) - .Time(2 * kFps30Rtp) - .AsLast() - .Refs({0}) - .Build()); + buffer_->InsertFrame(test::FakeFrameBuilder() + .Id(2) + .Time(2 * kFps30Rtp) + .AsLast() + .Refs({0}) + .Build()); // Halting time should result in F1 being skipped. time_controller_.AdvanceTime(kFps30Delay * 2); @@ -430,54 +435,54 @@ TEST_P(FrameBufferProxyTest, FramesFastForwardOnSystemHalt) { EXPECT_EQ(dropped_frames(), 1); } -TEST_P(FrameBufferProxyTest, ForceKeyFrame) { +TEST_P(VideoStreamBufferControllerTest, ForceKeyFrame) { StartNextDecodeForceKeyframe(); // Initial keyframe. - proxy_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); + buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); StartNextDecodeForceKeyframe(); // F2 is the next keyframe, and should be extracted since a keyframe was // forced. - proxy_->InsertFrame(test::FakeFrameBuilder() - .Id(1) - .Time(kFps30Rtp) - .AsLast() - .Refs({0}) - .Build()); - proxy_->InsertFrame( + buffer_->InsertFrame(test::FakeFrameBuilder() + .Id(1) + .Time(kFps30Rtp) + .AsLast() + .Refs({0}) + .Build()); + buffer_->InsertFrame( test::FakeFrameBuilder().Id(2).Time(kFps30Rtp * 2).AsLast().Build()); EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay * 3), Frame(test::WithId(2))); } -TEST_P(FrameBufferProxyTest, SlowDecoderDropsTemporalLayers) { +TEST_P(VideoStreamBufferControllerTest, SlowDecoderDropsTemporalLayers) { StartNextDecodeForceKeyframe(); // 2 temporal layers, at 15fps per layer to make 30fps total. // Decoder is slower than 30fps, so last_frame() will be skipped. // F1 --> F3 --> F5 // / / / // F0 --> F2 --> F4 - proxy_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); + buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); // Keyframe received. // Don't start next decode until slow delay. EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); time_controller_.AdvanceTime(kFps30Delay); - proxy_->InsertFrame(test::FakeFrameBuilder() - .Id(1) - .Time(1 * kFps30Rtp) - .Refs({0}) - .AsLast() - .Build()); + buffer_->InsertFrame(test::FakeFrameBuilder() + .Id(1) + .Time(1 * kFps30Rtp) + .Refs({0}) + .AsLast() + .Build()); time_controller_.AdvanceTime(kFps30Delay); - proxy_->InsertFrame(test::FakeFrameBuilder() - .Id(2) - .Time(2 * kFps30Rtp) - .Refs({0}) - .AsLast() - .Build()); + buffer_->InsertFrame(test::FakeFrameBuilder() + .Id(2) + .Time(2 * kFps30Rtp) + .Refs({0}) + .AsLast() + .Build()); // Simulate decode taking 3x FPS rate. time_controller_.AdvanceTime(kFps30Delay * 1.5); @@ -487,19 +492,19 @@ TEST_P(FrameBufferProxyTest, SlowDecoderDropsTemporalLayers) { EXPECT_EQ(dropped_frames(), 1); time_controller_.AdvanceTime(kFps30Delay / 2); - proxy_->InsertFrame(test::FakeFrameBuilder() - .Id(3) - .Time(3 * kFps30Rtp) - .Refs({1, 2}) - .AsLast() - .Build()); + buffer_->InsertFrame(test::FakeFrameBuilder() + .Id(3) + .Time(3 * kFps30Rtp) + .Refs({1, 2}) + .AsLast() + .Build()); time_controller_.AdvanceTime(kFps30Delay / 2); - proxy_->InsertFrame(test::FakeFrameBuilder() - .Id(4) - .Time(4 * kFps30Rtp) - .Refs({2}) - .AsLast() - .Build()); + buffer_->InsertFrame(test::FakeFrameBuilder() + .Id(4) + .Time(4 * kFps30Rtp) + .Refs({2}) + .AsLast() + .Build()); time_controller_.AdvanceTime(kFps30Delay / 2); // F4 is the best frame since decoding was so slow that F1 is too old. @@ -507,12 +512,12 @@ TEST_P(FrameBufferProxyTest, SlowDecoderDropsTemporalLayers) { StartNextDecode(); EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(4))); - proxy_->InsertFrame(test::FakeFrameBuilder() - .Id(5) - .Time(5 * kFps30Rtp) - .Refs({3, 4}) - .AsLast() - .Build()); + buffer_->InsertFrame(test::FakeFrameBuilder() + .Id(5) + .Time(5 * kFps30Rtp) + .Refs({3, 4}) + .AsLast() + .Build()); time_controller_.AdvanceTime(kFps30Delay / 2); // F5 is not decodable since F4 was decoded, so a timeout is expected. @@ -526,35 +531,36 @@ TEST_P(FrameBufferProxyTest, SlowDecoderDropsTemporalLayers) { // EXPECT_EQ(dropped_frames(), 2); } -TEST_P(FrameBufferProxyTest, NewFrameInsertedWhileWaitingToReleaseFrame) { +TEST_P(VideoStreamBufferControllerTest, + NewFrameInsertedWhileWaitingToReleaseFrame) { StartNextDecodeForceKeyframe(); // Initial keyframe. - proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp( + buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build())); EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); time_controller_.AdvanceTime(kFps30Delay / 2); - proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder() - .Id(1) - .Time(kFps30Rtp) - .Refs({0}) - .AsLast() - .Build())); + buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder() + .Id(1) + .Time(kFps30Rtp) + .Refs({0}) + .AsLast() + .Build())); StartNextDecode(); EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Eq(absl::nullopt)); // Scheduler is waiting to deliver Frame 1 now. Insert Frame 2. Frame 1 should // be delivered still. - proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder() - .Id(2) - .Time(kFps30Rtp * 2) - .Refs({0}) - .AsLast() - .Build())); + buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp(test::FakeFrameBuilder() + .Id(2) + .Time(kFps30Rtp * 2) + .Refs({0}) + .AsLast() + .Build())); EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(1))); } -TEST_P(FrameBufferProxyTest, SameFrameNotScheduledTwice) { +TEST_P(VideoStreamBufferControllerTest, SameFrameNotScheduledTwice) { // A frame could be scheduled twice if last_frame() arrive out-of-order but // the older frame is old enough to be fast forwarded. // @@ -567,7 +573,7 @@ TEST_P(FrameBufferProxyTest, SameFrameNotScheduledTwice) { StartNextDecodeForceKeyframe(); // First keyframe. - proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp( + buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build())); EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Millis(15)), Frame(test::WithId(0))); @@ -576,25 +582,25 @@ TEST_P(FrameBufferProxyTest, SameFrameNotScheduledTwice) { // Warmup VCMTiming for 30fps. for (int i = 1; i <= 30; ++i) { - proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp( + buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( test::FakeFrameBuilder().Id(i).Time(i * kFps30Rtp).AsLast().Build())); EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(i))); StartNextDecode(); } // F2 arrives and is scheduled. - proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp( + buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( test::FakeFrameBuilder().Id(32).Time(32 * kFps30Rtp).AsLast().Build())); // F3 arrives before F2 is extracted. time_controller_.AdvanceTime(kFps30Delay); - proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp( + buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( test::FakeFrameBuilder().Id(33).Time(33 * kFps30Rtp).AsLast().Build())); // F1 arrives and is fast-forwarded since it is too late. // F2 is already scheduled and should not be rescheduled. time_controller_.AdvanceTime(kFps30Delay / 2); - proxy_->InsertFrame(WithReceiveTimeFromRtpTimestamp( + buffer_->InsertFrame(WithReceiveTimeFromRtpTimestamp( test::FakeFrameBuilder().Id(31).Time(31 * kFps30Rtp).AsLast().Build())); EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(32))); @@ -606,7 +612,7 @@ TEST_P(FrameBufferProxyTest, SameFrameNotScheduledTwice) { EXPECT_EQ(dropped_frames(), 1); } -TEST_P(FrameBufferProxyTest, TestStatsCallback) { +TEST_P(VideoStreamBufferControllerTest, TestStatsCallback) { EXPECT_CALL(stats_callback_, OnCompleteFrame(true, kFrameSize, VideoContentType::UNSPECIFIED)); EXPECT_CALL(stats_callback_, OnFrameBufferTimingsUpdated); @@ -614,33 +620,36 @@ TEST_P(FrameBufferProxyTest, TestStatsCallback) { // Fake timing having received decoded frame. timing_.StopDecodeTimer(TimeDelta::Millis(1), clock_->CurrentTime()); StartNextDecodeForceKeyframe(); - proxy_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); + buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); // Flush stats posted on the decode queue. time_controller_.AdvanceTime(TimeDelta::Zero()); } -TEST_P(FrameBufferProxyTest, FrameCompleteCalledOnceForDuplicateFrame) { +TEST_P(VideoStreamBufferControllerTest, + FrameCompleteCalledOnceForDuplicateFrame) { EXPECT_CALL(stats_callback_, OnCompleteFrame(true, kFrameSize, VideoContentType::UNSPECIFIED)) .Times(1); StartNextDecodeForceKeyframe(); - proxy_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); - proxy_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); + buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); + buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build()); // Flush stats posted on the decode queue. time_controller_.AdvanceTime(TimeDelta::Zero()); } -TEST_P(FrameBufferProxyTest, FrameCompleteCalledOnceForSingleTemporalUnit) { +TEST_P(VideoStreamBufferControllerTest, + FrameCompleteCalledOnceForSingleTemporalUnit) { StartNextDecodeForceKeyframe(); // `OnCompleteFrame` should not be called for the first two frames since they // do not complete the temporal layer. EXPECT_CALL(stats_callback_, OnCompleteFrame(_, _, _)).Times(0); - proxy_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).Build()); - proxy_->InsertFrame(test::FakeFrameBuilder().Id(1).Time(0).Refs({0}).Build()); + buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).Build()); + buffer_->InsertFrame( + test::FakeFrameBuilder().Id(1).Time(0).Refs({0}).Build()); time_controller_.AdvanceTime(TimeDelta::Zero()); // Flush stats posted on the decode queue. ::testing::Mock::VerifyAndClearExpectations(&stats_callback_); @@ -650,13 +659,14 @@ TEST_P(FrameBufferProxyTest, FrameCompleteCalledOnceForSingleTemporalUnit) { EXPECT_CALL(stats_callback_, OnCompleteFrame(false, kFrameSize, VideoContentType::UNSPECIFIED)) .Times(1); - proxy_->InsertFrame( + buffer_->InsertFrame( test::FakeFrameBuilder().Id(2).Time(0).Refs({0, 1}).AsLast().Build()); // Flush stats posted on the decode queue. time_controller_.AdvanceTime(TimeDelta::Zero()); } -TEST_P(FrameBufferProxyTest, FrameCompleteCalledOnceForCompleteTemporalUnit) { +TEST_P(VideoStreamBufferControllerTest, + FrameCompleteCalledOnceForCompleteTemporalUnit) { // FrameBuffer2 logs the complete frame on the arrival of the last layer. StartNextDecodeForceKeyframe(); @@ -664,8 +674,8 @@ TEST_P(FrameBufferProxyTest, FrameCompleteCalledOnceForCompleteTemporalUnit) { // do not complete the temporal layer. Frame 1 arrives later, at which time // this frame can finally be considered complete. EXPECT_CALL(stats_callback_, OnCompleteFrame(_, _, _)).Times(0); - proxy_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).Build()); - proxy_->InsertFrame( + buffer_->InsertFrame(test::FakeFrameBuilder().Id(0).Time(0).Build()); + buffer_->InsertFrame( test::FakeFrameBuilder().Id(2).Time(0).Refs({0, 1}).AsLast().Build()); time_controller_.AdvanceTime(TimeDelta::Zero()); // Flush stats posted on the decode queue. @@ -674,7 +684,8 @@ TEST_P(FrameBufferProxyTest, FrameCompleteCalledOnceForCompleteTemporalUnit) { EXPECT_CALL(stats_callback_, OnCompleteFrame(false, kFrameSize, VideoContentType::UNSPECIFIED)) .Times(1); - proxy_->InsertFrame(test::FakeFrameBuilder().Id(1).Time(0).Refs({0}).Build()); + buffer_->InsertFrame( + test::FakeFrameBuilder().Id(1).Time(0).Refs({0}).Build()); // Flush stats posted on the decode queue. time_controller_.AdvanceTime(TimeDelta::Zero()); } @@ -683,7 +694,7 @@ TEST_P(FrameBufferProxyTest, FrameCompleteCalledOnceForCompleteTemporalUnit) { // Since the test needs to wait for the timestamp to rollover, it has a fake // delay of around 6.5 hours. Even though time is simulated, this will be // around 1,500,000 metronome tick invocations. -TEST_P(FrameBufferProxyTest, NextFrameWithOldTimestamp) { +TEST_P(VideoStreamBufferControllerTest, NextFrameWithOldTimestamp) { // Test inserting 31 frames and pause the stream for a long time before // frame 32. StartNextDecodeForceKeyframe(); @@ -692,22 +703,22 @@ TEST_P(FrameBufferProxyTest, NextFrameWithOldTimestamp) { // First keyframe. The receive time must be explicitly set in this test since // the RTP derived time used in all tests does not work when the long pause // happens later in the test. - proxy_->InsertFrame(test::FakeFrameBuilder() - .Id(0) - .Time(kBaseRtp) - .ReceivedTime(clock_->CurrentTime()) - .AsLast() - .Build()); + buffer_->InsertFrame(test::FakeFrameBuilder() + .Id(0) + .Time(kBaseRtp) + .ReceivedTime(clock_->CurrentTime()) + .AsLast() + .Build()); EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(0))); // 1 more frame to warmup VCMTiming for 30fps. StartNextDecode(); - proxy_->InsertFrame(test::FakeFrameBuilder() - .Id(1) - .Time(kBaseRtp + kFps30Rtp) - .ReceivedTime(clock_->CurrentTime()) - .AsLast() - .Build()); + buffer_->InsertFrame(test::FakeFrameBuilder() + .Id(1) + .Time(kBaseRtp + kFps30Rtp) + .ReceivedTime(clock_->CurrentTime()) + .AsLast() + .Build()); EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(1))); // Pause the stream for such a long time it incurs an RTP timestamp rollover @@ -727,26 +738,27 @@ TEST_P(FrameBufferProxyTest, NextFrameWithOldTimestamp) { EXPECT_THAT(WaitForFrameOrTimeout(kMaxWaitForFrame), Eq(absl::nullopt)); time_controller_.AdvanceTime(kRolloverDelay - kMaxWaitForFrame); StartNextDecode(); - proxy_->InsertFrame(test::FakeFrameBuilder() - .Id(2) - .Time(kRolloverRtp) - .ReceivedTime(clock_->CurrentTime()) - .AsLast() - .Build()); + buffer_->InsertFrame(test::FakeFrameBuilder() + .Id(2) + .Time(kRolloverRtp) + .ReceivedTime(clock_->CurrentTime()) + .AsLast() + .Build()); // FrameBuffer2 drops the frame, while FrameBuffer3 will continue the stream. EXPECT_THAT(WaitForFrameOrTimeout(kFps30Delay), Frame(test::WithId(2))); } INSTANTIATE_TEST_SUITE_P( - FrameBufferProxy, - FrameBufferProxyTest, + VideoStreamBufferController, + VideoStreamBufferControllerTest, ::testing::Values("WebRTC-FrameBuffer3/arm:FrameBuffer3/", "WebRTC-FrameBuffer3/arm:SyncDecoding/")); -class LowLatencyFrameBufferProxyTest : public ::testing::Test, - public FrameBufferProxyFixture {}; +class LowLatencyVideoStreamBufferControllerTest + : public ::testing::Test, + public VideoStreamBufferControllerFixture {}; -TEST_P(LowLatencyFrameBufferProxyTest, +TEST_P(LowLatencyVideoStreamBufferControllerTest, FramesDecodedInstantlyWithLowLatencyRendering) { // Initial keyframe. StartNextDecodeForceKeyframe(); @@ -755,7 +767,7 @@ TEST_P(LowLatencyFrameBufferProxyTest, auto frame = test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build(); // Playout delay of 0 implies low-latency rendering. frame->SetPlayoutDelay({0, 10}); - proxy_->InsertFrame(std::move(frame)); + buffer_->InsertFrame(std::move(frame)); EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); // Delta frame would normally wait here, but should decode at the pacing rate @@ -763,14 +775,14 @@ TEST_P(LowLatencyFrameBufferProxyTest, StartNextDecode(); frame = test::FakeFrameBuilder().Id(1).Time(kFps30Rtp).AsLast().Build(); frame->SetPlayoutDelay({0, 10}); - proxy_->InsertFrame(std::move(frame)); + buffer_->InsertFrame(std::move(frame)); // Pacing is set to 16ms in the field trial so we should not decode yet. EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Eq(absl::nullopt)); time_controller_.AdvanceTime(TimeDelta::Millis(16)); EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(1))); } -TEST_P(LowLatencyFrameBufferProxyTest, ZeroPlayoutDelayFullQueue) { +TEST_P(LowLatencyVideoStreamBufferControllerTest, ZeroPlayoutDelayFullQueue) { // Initial keyframe. StartNextDecodeForceKeyframe(); timing_.set_min_playout_delay(TimeDelta::Zero()); @@ -778,7 +790,7 @@ TEST_P(LowLatencyFrameBufferProxyTest, ZeroPlayoutDelayFullQueue) { auto frame = test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build(); // Playout delay of 0 implies low-latency rendering. frame->SetPlayoutDelay({0, 10}); - proxy_->InsertFrame(std::move(frame)); + buffer_->InsertFrame(std::move(frame)); EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); // Queue up 5 frames (configured max queue size for 0-playout delay pacing). @@ -786,7 +798,7 @@ TEST_P(LowLatencyFrameBufferProxyTest, ZeroPlayoutDelayFullQueue) { frame = test::FakeFrameBuilder().Id(id).Time(kFps30Rtp * id).AsLast().Build(); frame->SetPlayoutDelay({0, 10}); - proxy_->InsertFrame(std::move(frame)); + buffer_->InsertFrame(std::move(frame)); } // The queue is at its max size for zero playout delay pacing, so the pacing @@ -795,7 +807,8 @@ TEST_P(LowLatencyFrameBufferProxyTest, ZeroPlayoutDelayFullQueue) { EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(1))); } -TEST_P(LowLatencyFrameBufferProxyTest, MinMaxDelayZeroLowLatencyMode) { +TEST_P(LowLatencyVideoStreamBufferControllerTest, + MinMaxDelayZeroLowLatencyMode) { // Initial keyframe. StartNextDecodeForceKeyframe(); timing_.set_min_playout_delay(TimeDelta::Zero()); @@ -803,7 +816,7 @@ TEST_P(LowLatencyFrameBufferProxyTest, MinMaxDelayZeroLowLatencyMode) { auto frame = test::FakeFrameBuilder().Id(0).Time(0).AsLast().Build(); // Playout delay of 0 implies low-latency rendering. frame->SetPlayoutDelay({0, 0}); - proxy_->InsertFrame(std::move(frame)); + buffer_->InsertFrame(std::move(frame)); EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(0))); // Delta frame would normally wait here, but should decode at the pacing rate @@ -811,15 +824,15 @@ TEST_P(LowLatencyFrameBufferProxyTest, MinMaxDelayZeroLowLatencyMode) { StartNextDecode(); frame = test::FakeFrameBuilder().Id(1).Time(kFps30Rtp).AsLast().Build(); frame->SetPlayoutDelay({0, 0}); - proxy_->InsertFrame(std::move(frame)); + buffer_->InsertFrame(std::move(frame)); // The min/max=0 version of low-latency rendering will result in a large // negative decode wait time, so the frame should be ready right away. EXPECT_THAT(WaitForFrameOrTimeout(TimeDelta::Zero()), Frame(test::WithId(1))); } INSTANTIATE_TEST_SUITE_P( - FrameBufferProxy, - LowLatencyFrameBufferProxyTest, + VideoStreamBufferController, + LowLatencyVideoStreamBufferControllerTest, ::testing::Values( "WebRTC-FrameBuffer3/arm:FrameBuffer3/" "WebRTC-ZeroPlayoutDelay/min_pacing:16ms,max_decode_queue_size:5/",