Add low-latency stream signaling to VideoFrame and VCMTiming

This is the first CL out of three to make the low-latency stream signaling
explicit. At the moment this is done by setting the render time to 0.
There's a dependency between Chromium and WebRTC which is why this is
split into three CLs to not break any existing functionality.

Bug: chromium:1327251
Change-Id: Ie6b268746d587a99334485db77181fb2c6e9b567
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/264502
Reviewed-by: Evan Shrubsole <eshr@webrtc.org>
Reviewed-by: Harald Alvestrand <hta@webrtc.org>
Commit-Queue: Johannes Kron <kron@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#37225}

modules/video_coding/generic_decoder.cc

@@ -104,17 +104,14 @@ void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage,
   decodedImage.set_ntp_time_ms(frameInfo->ntp_time_ms);
   decodedImage.set_packet_infos(frameInfo->packet_infos);
   decodedImage.set_rotation(frameInfo->rotation);
-
-  absl::optional<int> max_composition_delay_in_frames =
-      _timing->MaxCompositionDelayInFrames();
-  if (max_composition_delay_in_frames) {
+  VideoFrame::RenderParameters render_parameters = _timing->RenderParameters();
+  if (render_parameters.max_composition_delay_in_frames) {
     // Subtract frames that are in flight.
-    *max_composition_delay_in_frames -= timestamp_map_size;
-    *max_composition_delay_in_frames =
-        std::max(0, *max_composition_delay_in_frames);
-    decodedImage.set_max_composition_delay_in_frames(
-        max_composition_delay_in_frames);
+    render_parameters.max_composition_delay_in_frames =
+        std::max(0, *render_parameters.max_composition_delay_in_frames -
+                        timestamp_map_size);
   }
+  decodedImage.set_render_parameters(render_parameters);
 
   RTC_DCHECK(frameInfo->decodeStart);
   const Timestamp now = _clock->CurrentTime();

modules/video_coding/generic_decoder_unittest.cc

@@ -124,23 +124,43 @@ TEST_F(GenericDecoderTest, MaxCompositionDelayNotSetByDefault) {
   generic_decoder_.Decode(encoded_frame, clock_.CurrentTime());
   absl::optional<VideoFrame> decoded_frame = user_callback_.WaitForFrame(10);
   ASSERT_TRUE(decoded_frame.has_value());
-  EXPECT_FALSE(decoded_frame->max_composition_delay_in_frames());
+  EXPECT_THAT(
+      decoded_frame->render_parameters().max_composition_delay_in_frames,
+      testing::Eq(absl::nullopt));
 }
 
 TEST_F(GenericDecoderTest, MaxCompositionDelayActivatedByPlayoutDelay) {
   VCMEncodedFrame encoded_frame;
   // VideoReceiveStream2 would set MaxCompositionDelayInFrames if playout delay
   // is specified as X,Y, where X=0, Y>0.
-  const VideoPlayoutDelay kPlayoutDelay = {0, 50};
   constexpr int kMaxCompositionDelayInFrames = 3;  // ~50 ms at 60 fps.
-  encoded_frame.SetPlayoutDelay(kPlayoutDelay);
   timing_.SetMaxCompositionDelayInFrames(
       absl::make_optional(kMaxCompositionDelayInFrames));
   generic_decoder_.Decode(encoded_frame, clock_.CurrentTime());
   absl::optional<VideoFrame> decoded_frame = user_callback_.WaitForFrame(10);
   ASSERT_TRUE(decoded_frame.has_value());
-  EXPECT_EQ(kMaxCompositionDelayInFrames,
-            decoded_frame->max_composition_delay_in_frames());
+  EXPECT_THAT(
+      decoded_frame->render_parameters().max_composition_delay_in_frames,
+      testing::Optional(kMaxCompositionDelayInFrames));
+}
+
+TEST_F(GenericDecoderTest, IsLowLatencyStreamFalseByDefault) {
+  VCMEncodedFrame encoded_frame;
+  generic_decoder_.Decode(encoded_frame, clock_.CurrentTime());
+  absl::optional<VideoFrame> decoded_frame = user_callback_.WaitForFrame(10);
+  ASSERT_TRUE(decoded_frame.has_value());
+  EXPECT_FALSE(decoded_frame->render_parameters().use_low_latency_rendering);
+}
+
+TEST_F(GenericDecoderTest, IsLowLatencyStreamActivatedByPlayoutDelay) {
+  VCMEncodedFrame encoded_frame;
+  const VideoPlayoutDelay kPlayoutDelay = {0, 50};
+  timing_.set_min_playout_delay(TimeDelta::Millis(kPlayoutDelay.min_ms));
+  timing_.set_max_playout_delay(TimeDelta::Millis(kPlayoutDelay.max_ms));
+  generic_decoder_.Decode(encoded_frame, clock_.CurrentTime());
+  absl::optional<VideoFrame> decoded_frame = user_callback_.WaitForFrame(10);
+  ASSERT_TRUE(decoded_frame.has_value());
+  EXPECT_TRUE(decoded_frame->render_parameters().use_low_latency_rendering);
 }
 
 }  // namespace video_coding

modules/video_coding/timing/BUILD.gn

@@ -71,6 +71,7 @@ rtc_library("timing_module") {
     ":codec_timer",
     "../../../api:field_trials_view",
     "../../../api/units:time_delta",
+    "../../../api/video:video_frame",
     "../../../api/video:video_rtp_headers",
     "../../../rtc_base:logging",
     "../../../rtc_base:macromagic",

modules/video_coding/timing/timing.cc

@@ -23,7 +23,8 @@ namespace {
 
 // Default pacing that is used for the low-latency renderer path.
 constexpr TimeDelta kZeroPlayoutDelayDefaultMinPacing = TimeDelta::Millis(8);
-constexpr TimeDelta kLowLatencyRendererMaxPlayoutDelay = TimeDelta::Millis(500);
+constexpr TimeDelta kLowLatencyStreamMaxPlayoutDelayThreshold =
+    TimeDelta::Millis(500);
 
 void CheckDelaysValid(TimeDelta min_delay, TimeDelta max_delay) {
   if (min_delay > max_delay) {
@@ -191,9 +192,7 @@ void VCMTiming::SetLastDecodeScheduledTimestamp(
 
 Timestamp VCMTiming::RenderTimeInternal(uint32_t frame_timestamp,
                                         Timestamp now) const {
-  if (min_playout_delay_.IsZero() &&
-      (max_playout_delay_.IsZero() ||
-       max_playout_delay_ <= kLowLatencyRendererMaxPlayoutDelay)) {
+  if (UseLowLatencyRendering()) {
     // Render as soon as possible or with low-latency renderer algorithm.
     return Timestamp::Zero();
   }
@@ -250,6 +249,21 @@ TimeDelta VCMTiming::TargetDelayInternal() const {
                   jitter_delay_ + RequiredDecodeTime() + render_delay_);
 }
 
+VideoFrame::RenderParameters VCMTiming::RenderParameters() const {
+  MutexLock lock(&mutex_);
+  return {.use_low_latency_rendering = UseLowLatencyRendering(),
+          .max_composition_delay_in_frames = max_composition_delay_in_frames_};
+}
+
+bool VCMTiming::UseLowLatencyRendering() const {
+  // min_playout_delay_==0,
+  // max_playout_delay_<=kLowLatencyStreamMaxPlayoutDelayThreshold indicates
+  // that the low-latency path should be used, which means that frames should be
+  // decoded and rendered as soon as possible.
+  return min_playout_delay_.IsZero() &&
+         max_playout_delay_ <= kLowLatencyStreamMaxPlayoutDelayThreshold;
+}
+
 VCMTiming::VideoDelayTimings VCMTiming::GetTimings() const {
   MutexLock lock(&mutex_);
   return VideoDelayTimings{.max_decode_duration = RequiredDecodeTime(),
@@ -278,9 +292,4 @@ void VCMTiming::SetMaxCompositionDelayInFrames(
   max_composition_delay_in_frames_ = max_composition_delay_in_frames;
 }
 
-absl::optional<int> VCMTiming::MaxCompositionDelayInFrames() const {
-  MutexLock lock(&mutex_);
-  return max_composition_delay_in_frames_;
-}
-
 }  // namespace webrtc

modules/video_coding/timing/timing.h

@@ -16,6 +16,7 @@
 #include "absl/types/optional.h"
 #include "api/field_trials_view.h"
 #include "api/units/time_delta.h"
+#include "api/video/video_frame.h"
 #include "api/video/video_timing.h"
 #include "modules/video_coding/timing/codec_timer.h"
 #include "rtc_base/experiments/field_trial_parser.h"
@@ -111,7 +112,8 @@ class VCMTiming {
 
   void SetMaxCompositionDelayInFrames(
       absl::optional<int> max_composition_delay_in_frames);
-  absl::optional<int> MaxCompositionDelayInFrames() const;
+
+  VideoFrame::RenderParameters RenderParameters() const;
 
   // Updates the last time a frame was scheduled for decoding.
   void SetLastDecodeScheduledTimestamp(Timestamp last_decode_scheduled);
@@ -121,6 +123,7 @@ class VCMTiming {
   Timestamp RenderTimeInternal(uint32_t frame_timestamp, Timestamp now) const
       RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
   TimeDelta TargetDelayInternal() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+  bool UseLowLatencyRendering() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
 
  private:
   mutable Mutex mutex_;

modules/video_coding/timing/timing_unittest.cc

@@ -138,6 +138,31 @@ TEST(ReceiverTimingTest, TimestampWrapAround) {
   }
 }
 
+TEST(ReceiverTimingTest, UseLowLatencyRenderer) {
+  test::ScopedKeyValueConfig field_trials;
+  SimulatedClock clock(0);
+  VCMTiming timing(&clock, field_trials);
+  timing.Reset();
+  // Default is false.
+  EXPECT_FALSE(timing.RenderParameters().use_low_latency_rendering);
+  // False if min playout delay > 0.
+  timing.set_min_playout_delay(TimeDelta::Millis(10));
+  timing.set_max_playout_delay(TimeDelta::Millis(20));
+  EXPECT_FALSE(timing.RenderParameters().use_low_latency_rendering);
+  // True if min==0, max > 0.
+  timing.set_min_playout_delay(TimeDelta::Millis(0));
+  EXPECT_TRUE(timing.RenderParameters().use_low_latency_rendering);
+  // True if min==max==0.
+  timing.set_max_playout_delay(TimeDelta::Millis(0));
+  EXPECT_TRUE(timing.RenderParameters().use_low_latency_rendering);
+  // True also for max playout delay==500 ms.
+  timing.set_max_playout_delay(TimeDelta::Millis(500));
+  EXPECT_TRUE(timing.RenderParameters().use_low_latency_rendering);
+  // False if max playout delay > 500 ms.
+  timing.set_max_playout_delay(TimeDelta::Millis(501));
+  EXPECT_FALSE(timing.RenderParameters().use_low_latency_rendering);
+}
+
 TEST(ReceiverTimingTest, MaxWaitingTimeIsZeroForZeroRenderTime) {
   // This is the default path when the RTP playout delay header extension is set
   // to min==0 and max==0.