diff --git a/api/video/video_frame.cc b/api/video/video_frame.cc index d97e3aa82a..130820a886 100644 --- a/api/video/video_frame.cc +++ b/api/video/video_frame.cc @@ -164,8 +164,8 @@ VideoFrame::Builder::~Builder() = default; VideoFrame VideoFrame::Builder::build() { RTC_CHECK(video_frame_buffer_ != nullptr); return VideoFrame(id_, video_frame_buffer_, timestamp_us_, timestamp_rtp_, - ntp_time_ms_, rotation_, color_space_, update_rect_, - packet_infos_); + ntp_time_ms_, rotation_, color_space_, render_parameters_, + update_rect_, packet_infos_); } VideoFrame::Builder& VideoFrame::Builder::set_video_frame_buffer( @@ -260,6 +260,7 @@ VideoFrame::VideoFrame(uint16_t id, int64_t ntp_time_ms, VideoRotation rotation, const absl::optional& color_space, + const RenderParameters& render_parameters, const absl::optional& update_rect, RtpPacketInfos packet_infos) : id_(id), @@ -269,6 +270,7 @@ VideoFrame::VideoFrame(uint16_t id, timestamp_us_(timestamp_us), rotation_(rotation), color_space_(color_space), + render_parameters_(render_parameters), update_rect_(update_rect), packet_infos_(std::move(packet_infos)) { if (update_rect_) { diff --git a/api/video/video_frame.h b/api/video/video_frame.h index 89f3f9c99b..3d2c78598c 100644 --- a/api/video/video_frame.h +++ b/api/video/video_frame.h @@ -81,6 +81,21 @@ class RTC_EXPORT VideoFrame { Timestamp finish; }; + struct RTC_EXPORT RenderParameters { + bool use_low_latency_rendering = false; + absl::optional max_composition_delay_in_frames; + + bool operator==(const RenderParameters& other) const { + return other.use_low_latency_rendering == use_low_latency_rendering && + other.max_composition_delay_in_frames == + max_composition_delay_in_frames; + } + + bool operator!=(const RenderParameters& other) const { + return !(*this == other); + } + }; + // Preferred way of building VideoFrame objects. class RTC_EXPORT Builder { public: @@ -109,6 +124,7 @@ class RTC_EXPORT VideoFrame { int64_t ntp_time_ms_ = 0; VideoRotation rotation_ = kVideoRotation_0; absl::optional color_space_; + RenderParameters render_parameters_; absl::optional update_rect_; RtpPacketInfos packet_infos_; }; @@ -189,14 +205,17 @@ class RTC_EXPORT VideoFrame { color_space_ = color_space; } - // max_composition_delay_in_frames() is used in an experiment of a low-latency - // renderer algorithm see crbug.com/1138888. - absl::optional max_composition_delay_in_frames() const { - return max_composition_delay_in_frames_; + RenderParameters render_parameters() const { return render_parameters_; } + void set_render_parameters(const RenderParameters& render_parameters) { + render_parameters_ = render_parameters; } - void set_max_composition_delay_in_frames( - absl::optional max_composition_delay_in_frames) { - max_composition_delay_in_frames_ = max_composition_delay_in_frames; + + // Deprecated in favor of render_parameters, will be removed once Chromium is + // updated. max_composition_delay_in_frames() is used in an experiment of a + // low-latency renderer algorithm see crbug.com/1138888. + [[deprecated("Use render_parameters() instead.")]] absl::optional + max_composition_delay_in_frames() const { + return render_parameters_.max_composition_delay_in_frames; } // Get render time in milliseconds. @@ -257,6 +276,7 @@ class RTC_EXPORT VideoFrame { int64_t ntp_time_ms, VideoRotation rotation, const absl::optional& color_space, + const RenderParameters& render_parameters, const absl::optional& update_rect, RtpPacketInfos packet_infos); @@ -268,7 +288,8 @@ class RTC_EXPORT VideoFrame { int64_t timestamp_us_; VideoRotation rotation_; absl::optional color_space_; - absl::optional max_composition_delay_in_frames_; + // Contains parameters that affect have the frame should be rendered. + RenderParameters render_parameters_; // Updated since the last frame area. If present it means that the bounding // box of all the changes is within the rectangular area and is close to it. // If absent, it means that there's no information about the change at all and diff --git a/modules/video_coding/generic_decoder.cc b/modules/video_coding/generic_decoder.cc index 7070fa2563..68eb1fc7e6 100644 --- a/modules/video_coding/generic_decoder.cc +++ b/modules/video_coding/generic_decoder.cc @@ -104,17 +104,14 @@ void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, decodedImage.set_ntp_time_ms(frameInfo->ntp_time_ms); decodedImage.set_packet_infos(frameInfo->packet_infos); decodedImage.set_rotation(frameInfo->rotation); - - absl::optional max_composition_delay_in_frames = - _timing->MaxCompositionDelayInFrames(); - if (max_composition_delay_in_frames) { + VideoFrame::RenderParameters render_parameters = _timing->RenderParameters(); + if (render_parameters.max_composition_delay_in_frames) { // Subtract frames that are in flight. - *max_composition_delay_in_frames -= timestamp_map_size; - *max_composition_delay_in_frames = - std::max(0, *max_composition_delay_in_frames); - decodedImage.set_max_composition_delay_in_frames( - max_composition_delay_in_frames); + render_parameters.max_composition_delay_in_frames = + std::max(0, *render_parameters.max_composition_delay_in_frames - + timestamp_map_size); } + decodedImage.set_render_parameters(render_parameters); RTC_DCHECK(frameInfo->decodeStart); const Timestamp now = _clock->CurrentTime(); diff --git a/modules/video_coding/generic_decoder_unittest.cc b/modules/video_coding/generic_decoder_unittest.cc index 20ea9c3469..65e8dba731 100644 --- a/modules/video_coding/generic_decoder_unittest.cc +++ b/modules/video_coding/generic_decoder_unittest.cc @@ -124,23 +124,43 @@ TEST_F(GenericDecoderTest, MaxCompositionDelayNotSetByDefault) { generic_decoder_.Decode(encoded_frame, clock_.CurrentTime()); absl::optional decoded_frame = user_callback_.WaitForFrame(10); ASSERT_TRUE(decoded_frame.has_value()); - EXPECT_FALSE(decoded_frame->max_composition_delay_in_frames()); + EXPECT_THAT( + decoded_frame->render_parameters().max_composition_delay_in_frames, + testing::Eq(absl::nullopt)); } TEST_F(GenericDecoderTest, MaxCompositionDelayActivatedByPlayoutDelay) { VCMEncodedFrame encoded_frame; // VideoReceiveStream2 would set MaxCompositionDelayInFrames if playout delay // is specified as X,Y, where X=0, Y>0. - const VideoPlayoutDelay kPlayoutDelay = {0, 50}; constexpr int kMaxCompositionDelayInFrames = 3; // ~50 ms at 60 fps. - encoded_frame.SetPlayoutDelay(kPlayoutDelay); timing_.SetMaxCompositionDelayInFrames( absl::make_optional(kMaxCompositionDelayInFrames)); generic_decoder_.Decode(encoded_frame, clock_.CurrentTime()); absl::optional decoded_frame = user_callback_.WaitForFrame(10); ASSERT_TRUE(decoded_frame.has_value()); - EXPECT_EQ(kMaxCompositionDelayInFrames, - decoded_frame->max_composition_delay_in_frames()); + EXPECT_THAT( + decoded_frame->render_parameters().max_composition_delay_in_frames, + testing::Optional(kMaxCompositionDelayInFrames)); +} + +TEST_F(GenericDecoderTest, IsLowLatencyStreamFalseByDefault) { + VCMEncodedFrame encoded_frame; + generic_decoder_.Decode(encoded_frame, clock_.CurrentTime()); + absl::optional decoded_frame = user_callback_.WaitForFrame(10); + ASSERT_TRUE(decoded_frame.has_value()); + EXPECT_FALSE(decoded_frame->render_parameters().use_low_latency_rendering); +} + +TEST_F(GenericDecoderTest, IsLowLatencyStreamActivatedByPlayoutDelay) { + VCMEncodedFrame encoded_frame; + const VideoPlayoutDelay kPlayoutDelay = {0, 50}; + timing_.set_min_playout_delay(TimeDelta::Millis(kPlayoutDelay.min_ms)); + timing_.set_max_playout_delay(TimeDelta::Millis(kPlayoutDelay.max_ms)); + generic_decoder_.Decode(encoded_frame, clock_.CurrentTime()); + absl::optional decoded_frame = user_callback_.WaitForFrame(10); + ASSERT_TRUE(decoded_frame.has_value()); + EXPECT_TRUE(decoded_frame->render_parameters().use_low_latency_rendering); } } // namespace video_coding diff --git a/modules/video_coding/timing/BUILD.gn b/modules/video_coding/timing/BUILD.gn index 369a680ed7..b6415f5ca6 100644 --- a/modules/video_coding/timing/BUILD.gn +++ b/modules/video_coding/timing/BUILD.gn @@ -71,6 +71,7 @@ rtc_library("timing_module") { ":codec_timer", "../../../api:field_trials_view", "../../../api/units:time_delta", + "../../../api/video:video_frame", "../../../api/video:video_rtp_headers", "../../../rtc_base:logging", "../../../rtc_base:macromagic", diff --git a/modules/video_coding/timing/timing.cc b/modules/video_coding/timing/timing.cc index 06754b2aad..0975195799 100644 --- a/modules/video_coding/timing/timing.cc +++ b/modules/video_coding/timing/timing.cc @@ -23,7 +23,8 @@ namespace { // Default pacing that is used for the low-latency renderer path. constexpr TimeDelta kZeroPlayoutDelayDefaultMinPacing = TimeDelta::Millis(8); -constexpr TimeDelta kLowLatencyRendererMaxPlayoutDelay = TimeDelta::Millis(500); +constexpr TimeDelta kLowLatencyStreamMaxPlayoutDelayThreshold = + TimeDelta::Millis(500); void CheckDelaysValid(TimeDelta min_delay, TimeDelta max_delay) { if (min_delay > max_delay) { @@ -191,9 +192,7 @@ void VCMTiming::SetLastDecodeScheduledTimestamp( Timestamp VCMTiming::RenderTimeInternal(uint32_t frame_timestamp, Timestamp now) const { - if (min_playout_delay_.IsZero() && - (max_playout_delay_.IsZero() || - max_playout_delay_ <= kLowLatencyRendererMaxPlayoutDelay)) { + if (UseLowLatencyRendering()) { // Render as soon as possible or with low-latency renderer algorithm. return Timestamp::Zero(); } @@ -250,6 +249,21 @@ TimeDelta VCMTiming::TargetDelayInternal() const { jitter_delay_ + RequiredDecodeTime() + render_delay_); } +VideoFrame::RenderParameters VCMTiming::RenderParameters() const { + MutexLock lock(&mutex_); + return {.use_low_latency_rendering = UseLowLatencyRendering(), + .max_composition_delay_in_frames = max_composition_delay_in_frames_}; +} + +bool VCMTiming::UseLowLatencyRendering() const { + // min_playout_delay_==0, + // max_playout_delay_<=kLowLatencyStreamMaxPlayoutDelayThreshold indicates + // that the low-latency path should be used, which means that frames should be + // decoded and rendered as soon as possible. + return min_playout_delay_.IsZero() && + max_playout_delay_ <= kLowLatencyStreamMaxPlayoutDelayThreshold; +} + VCMTiming::VideoDelayTimings VCMTiming::GetTimings() const { MutexLock lock(&mutex_); return VideoDelayTimings{.max_decode_duration = RequiredDecodeTime(), @@ -278,9 +292,4 @@ void VCMTiming::SetMaxCompositionDelayInFrames( max_composition_delay_in_frames_ = max_composition_delay_in_frames; } -absl::optional VCMTiming::MaxCompositionDelayInFrames() const { - MutexLock lock(&mutex_); - return max_composition_delay_in_frames_; -} - } // namespace webrtc diff --git a/modules/video_coding/timing/timing.h b/modules/video_coding/timing/timing.h index 94950ba9dd..6ee1cf4d6f 100644 --- a/modules/video_coding/timing/timing.h +++ b/modules/video_coding/timing/timing.h @@ -16,6 +16,7 @@ #include "absl/types/optional.h" #include "api/field_trials_view.h" #include "api/units/time_delta.h" +#include "api/video/video_frame.h" #include "api/video/video_timing.h" #include "modules/video_coding/timing/codec_timer.h" #include "rtc_base/experiments/field_trial_parser.h" @@ -111,7 +112,8 @@ class VCMTiming { void SetMaxCompositionDelayInFrames( absl::optional max_composition_delay_in_frames); - absl::optional MaxCompositionDelayInFrames() const; + + VideoFrame::RenderParameters RenderParameters() const; // Updates the last time a frame was scheduled for decoding. void SetLastDecodeScheduledTimestamp(Timestamp last_decode_scheduled); @@ -121,6 +123,7 @@ class VCMTiming { Timestamp RenderTimeInternal(uint32_t frame_timestamp, Timestamp now) const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); TimeDelta TargetDelayInternal() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + bool UseLowLatencyRendering() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); private: mutable Mutex mutex_; diff --git a/modules/video_coding/timing/timing_unittest.cc b/modules/video_coding/timing/timing_unittest.cc index 31ee44a503..e110071006 100644 --- a/modules/video_coding/timing/timing_unittest.cc +++ b/modules/video_coding/timing/timing_unittest.cc @@ -138,6 +138,31 @@ TEST(ReceiverTimingTest, TimestampWrapAround) { } } +TEST(ReceiverTimingTest, UseLowLatencyRenderer) { + test::ScopedKeyValueConfig field_trials; + SimulatedClock clock(0); + VCMTiming timing(&clock, field_trials); + timing.Reset(); + // Default is false. + EXPECT_FALSE(timing.RenderParameters().use_low_latency_rendering); + // False if min playout delay > 0. + timing.set_min_playout_delay(TimeDelta::Millis(10)); + timing.set_max_playout_delay(TimeDelta::Millis(20)); + EXPECT_FALSE(timing.RenderParameters().use_low_latency_rendering); + // True if min==0, max > 0. + timing.set_min_playout_delay(TimeDelta::Millis(0)); + EXPECT_TRUE(timing.RenderParameters().use_low_latency_rendering); + // True if min==max==0. + timing.set_max_playout_delay(TimeDelta::Millis(0)); + EXPECT_TRUE(timing.RenderParameters().use_low_latency_rendering); + // True also for max playout delay==500 ms. + timing.set_max_playout_delay(TimeDelta::Millis(500)); + EXPECT_TRUE(timing.RenderParameters().use_low_latency_rendering); + // False if max playout delay > 500 ms. + timing.set_max_playout_delay(TimeDelta::Millis(501)); + EXPECT_FALSE(timing.RenderParameters().use_low_latency_rendering); +} + TEST(ReceiverTimingTest, MaxWaitingTimeIsZeroForZeroRenderTime) { // This is the default path when the RTP playout delay header extension is set // to min==0 and max==0. diff --git a/video/video_receive_stream2.cc b/video/video_receive_stream2.cc index ee708697dd..cd38d03ea4 100644 --- a/video/video_receive_stream2.cc +++ b/video/video_receive_stream2.cc @@ -992,6 +992,7 @@ void VideoReceiveStream2::UpdatePlayoutDelays() const { const std::initializer_list> min_delays = { frame_minimum_playout_delay_, base_minimum_playout_delay_, syncable_minimum_playout_delay_}; + // Since nullopt < anything, this will return the largest of the minumum // delays, or nullopt if all are nullopt. absl::optional minimum_delay = std::max(min_delays); diff --git a/video/video_receive_stream2_unittest.cc b/video/video_receive_stream2_unittest.cc index 5236bb5c39..1846e51ea9 100644 --- a/video/video_receive_stream2_unittest.cc +++ b/video/video_receive_stream2_unittest.cc @@ -88,6 +88,7 @@ using ::testing::Field; using ::testing::InSequence; using ::testing::Invoke; using ::testing::IsEmpty; +using ::testing::Optional; using ::testing::Pointee; using ::testing::Property; using ::testing::Return; @@ -386,38 +387,94 @@ TEST_P(VideoReceiveStream2Test, PlayoutDelayPreservesDefaultMinValue) { EXPECT_EQ(default_min_playout_latency, timings.min_playout_delay); } -TEST_P(VideoReceiveStream2Test, MaxCompositionDelayNotSetByDefault) { +TEST_P(VideoReceiveStream2Test, RenderParametersSetToDefaultValues) { + // Default render parameters. + const VideoFrame::RenderParameters kDefaultRenderParameters; // Default with no playout delay set. std::unique_ptr test_frame0 = test::FakeFrameBuilder().Id(0).AsLast().Build(); video_receive_stream_->OnCompleteFrame(std::move(test_frame0)); - EXPECT_FALSE(timing_->MaxCompositionDelayInFrames()); + EXPECT_EQ(timing_->RenderParameters(), kDefaultRenderParameters); +} + +TEST_P(VideoReceiveStream2Test, UseLowLatencyRenderingSetFromPlayoutDelay) { + // use_low_latency_rendering set if playout delay set to min=0, max<=500 ms. + std::unique_ptr test_frame0 = + test::FakeFrameBuilder().Id(0).AsLast().Build(); + test_frame0->SetPlayoutDelay({/*min_ms=*/0, /*max_ms=*/0}); + video_receive_stream_->OnCompleteFrame(std::move(test_frame0)); + EXPECT_TRUE(timing_->RenderParameters().use_low_latency_rendering); - // Max composition delay not set for playout delay 0,0. std::unique_ptr test_frame1 = test::FakeFrameBuilder().Id(1).AsLast().Build(); - test_frame1->SetPlayoutDelay({0, 0}); + test_frame1->SetPlayoutDelay({/*min_ms=*/0, /*max_ms=*/500}); video_receive_stream_->OnCompleteFrame(std::move(test_frame1)); - EXPECT_FALSE(timing_->MaxCompositionDelayInFrames()); - - // Max composition delay not set for playout delay X,Y, where X,Y>0. - std::unique_ptr test_frame2 = - test::FakeFrameBuilder().Id(2).AsLast().Build(); - test_frame2->SetPlayoutDelay({10, 30}); - video_receive_stream_->OnCompleteFrame(std::move(test_frame2)); - EXPECT_FALSE(timing_->MaxCompositionDelayInFrames()); + EXPECT_TRUE(timing_->RenderParameters().use_low_latency_rendering); } TEST_P(VideoReceiveStream2Test, MaxCompositionDelaySetFromMaxPlayoutDelay) { + // The max composition delay is dependent on the number of frames in the + // pre-decode queue. It's therefore important to advance the time as the test + // runs to get the correct expectations of max_composition_delay_in_frames. + video_receive_stream_->Start(); + // Max composition delay not set if no playout delay is set. + std::unique_ptr test_frame0 = + test::FakeFrameBuilder() + .Id(0) + .Time(RtpTimestampForFrame(0)) + .ReceivedTime(ReceiveTimeForFrame(0)) + .AsLast() + .Build(); + video_receive_stream_->OnCompleteFrame(std::move(test_frame0)); + EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames, + Eq(absl::nullopt)); + time_controller_.AdvanceTime(k30FpsDelay); + loop_.Flush(); + + // Max composition delay not set for playout delay 0,0. + std::unique_ptr test_frame1 = + test::FakeFrameBuilder() + .Id(1) + .Time(RtpTimestampForFrame(1)) + .ReceivedTime(ReceiveTimeForFrame(1)) + .AsLast() + .Build(); + test_frame1->SetPlayoutDelay({0, 0}); + video_receive_stream_->OnCompleteFrame(std::move(test_frame1)); + EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames, + Eq(absl::nullopt)); + time_controller_.AdvanceTime(k30FpsDelay); + loop_.Flush(); + + // Max composition delay not set for playout delay X,Y, where X,Y>0. + std::unique_ptr test_frame2 = + test::FakeFrameBuilder() + .Id(2) + .Time(RtpTimestampForFrame(2)) + .ReceivedTime(ReceiveTimeForFrame(2)) + .AsLast() + .Build(); + test_frame2->SetPlayoutDelay({10, 30}); + video_receive_stream_->OnCompleteFrame(std::move(test_frame2)); + EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames, + Eq(absl::nullopt)); + + time_controller_.AdvanceTime(k30FpsDelay); + loop_.Flush(); + // Max composition delay set if playout delay X,Y, where X=0,Y>0. - const VideoPlayoutDelay kPlayoutDelayMs = {0, 50}; const int kExpectedMaxCompositionDelayInFrames = 3; // ~50 ms at 60 fps. - std::unique_ptr test_frame = - test::FakeFrameBuilder().Id(0).AsLast().Build(); - test_frame->SetPlayoutDelay(kPlayoutDelayMs); - video_receive_stream_->OnCompleteFrame(std::move(test_frame)); - EXPECT_EQ(kExpectedMaxCompositionDelayInFrames, - timing_->MaxCompositionDelayInFrames()); + std::unique_ptr test_frame3 = + test::FakeFrameBuilder() + .Id(3) + .Time(RtpTimestampForFrame(3)) + .ReceivedTime(ReceiveTimeForFrame(3)) + .AsLast() + .Build(); + test_frame3->SetPlayoutDelay({0, 50}); + video_receive_stream_->OnCompleteFrame(std::move(test_frame3)); + EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames, + Optional(kExpectedMaxCompositionDelayInFrames)); } TEST_P(VideoReceiveStream2Test, LazyDecoderCreation) {