Add low-latency stream signaling to VideoFrame and VCMTiming

This is the first CL out of three to make the low-latency stream signaling
explicit. At the moment this is done by setting the render time to 0.
There's a dependency between Chromium and WebRTC which is why this is
split into three CLs to not break any existing functionality.

Bug: chromium:1327251
Change-Id: Ie6b268746d587a99334485db77181fb2c6e9b567
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/264502
Reviewed-by: Evan Shrubsole <eshr@webrtc.org>
Reviewed-by: Harald Alvestrand <hta@webrtc.org>
Commit-Queue: Johannes Kron <kron@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#37225}

api/video/video_frame.cc

@@ -164,8 +164,8 @@ VideoFrame::Builder::~Builder() = default;
 VideoFrame VideoFrame::Builder::build() {
   RTC_CHECK(video_frame_buffer_ != nullptr);
   return VideoFrame(id_, video_frame_buffer_, timestamp_us_, timestamp_rtp_,
-                    ntp_time_ms_, rotation_, color_space_, update_rect_,
+                    ntp_time_ms_, rotation_, color_space_, render_parameters_,
-                    packet_infos_);
+                    update_rect_, packet_infos_);
 }
 
 VideoFrame::Builder& VideoFrame::Builder::set_video_frame_buffer(
@@ -260,6 +260,7 @@ VideoFrame::VideoFrame(uint16_t id,
                        int64_t ntp_time_ms,
                        VideoRotation rotation,
                        const absl::optional<ColorSpace>& color_space,
+                       const RenderParameters& render_parameters,
                        const absl::optional<UpdateRect>& update_rect,
                        RtpPacketInfos packet_infos)
     : id_(id),
@@ -269,6 +270,7 @@ VideoFrame::VideoFrame(uint16_t id,
       timestamp_us_(timestamp_us),
       rotation_(rotation),
       color_space_(color_space),
+      render_parameters_(render_parameters),
       update_rect_(update_rect),
       packet_infos_(std::move(packet_infos)) {
   if (update_rect_) {

api/video/video_frame.h

@@ -81,6 +81,21 @@ class RTC_EXPORT VideoFrame {
     Timestamp finish;
   };
 
+  struct RTC_EXPORT RenderParameters {
+    bool use_low_latency_rendering = false;
+    absl::optional<int32_t> max_composition_delay_in_frames;
+
+    bool operator==(const RenderParameters& other) const {
+      return other.use_low_latency_rendering == use_low_latency_rendering &&
+             other.max_composition_delay_in_frames ==
+                 max_composition_delay_in_frames;
+    }
+
+    bool operator!=(const RenderParameters& other) const {
+      return !(*this == other);
+    }
+  };
+
   // Preferred way of building VideoFrame objects.
   class RTC_EXPORT Builder {
    public:
@@ -109,6 +124,7 @@ class RTC_EXPORT VideoFrame {
     int64_t ntp_time_ms_ = 0;
     VideoRotation rotation_ = kVideoRotation_0;
     absl::optional<ColorSpace> color_space_;
+    RenderParameters render_parameters_;
     absl::optional<UpdateRect> update_rect_;
     RtpPacketInfos packet_infos_;
   };
@@ -189,14 +205,17 @@ class RTC_EXPORT VideoFrame {
     color_space_ = color_space;
   }
 
-  // max_composition_delay_in_frames() is used in an experiment of a low-latency
+  RenderParameters render_parameters() const { return render_parameters_; }
-  // renderer algorithm see crbug.com/1138888.
+  void set_render_parameters(const RenderParameters& render_parameters) {
-  absl::optional<int32_t> max_composition_delay_in_frames() const {
+    render_parameters_ = render_parameters;
-    return max_composition_delay_in_frames_;
   }
-  void set_max_composition_delay_in_frames(
+
-      absl::optional<int32_t> max_composition_delay_in_frames) {
+  // Deprecated in favor of render_parameters, will be removed once Chromium is
-    max_composition_delay_in_frames_ = max_composition_delay_in_frames;
+  // updated. max_composition_delay_in_frames() is used in an experiment of a
+  // low-latency renderer algorithm see crbug.com/1138888.
+  [[deprecated("Use render_parameters() instead.")]] absl::optional<int32_t>
+  max_composition_delay_in_frames() const {
+    return render_parameters_.max_composition_delay_in_frames;
   }
 
   // Get render time in milliseconds.
@@ -257,6 +276,7 @@ class RTC_EXPORT VideoFrame {
              int64_t ntp_time_ms,
              VideoRotation rotation,
              const absl::optional<ColorSpace>& color_space,
+             const RenderParameters& render_parameters,
              const absl::optional<UpdateRect>& update_rect,
              RtpPacketInfos packet_infos);
 
@@ -268,7 +288,8 @@ class RTC_EXPORT VideoFrame {
   int64_t timestamp_us_;
   VideoRotation rotation_;
   absl::optional<ColorSpace> color_space_;
-  absl::optional<int32_t> max_composition_delay_in_frames_;
+  // Contains parameters that affect have the frame should be rendered.
+  RenderParameters render_parameters_;
   // Updated since the last frame area. If present it means that the bounding
   // box of all the changes is within the rectangular area and is close to it.
   // If absent, it means that there's no information about the change at all and

modules/video_coding/generic_decoder.cc

@@ -104,17 +104,14 @@ void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage,
   decodedImage.set_ntp_time_ms(frameInfo->ntp_time_ms);
   decodedImage.set_packet_infos(frameInfo->packet_infos);
   decodedImage.set_rotation(frameInfo->rotation);
-
+  VideoFrame::RenderParameters render_parameters = _timing->RenderParameters();
-  absl::optional<int> max_composition_delay_in_frames =
+  if (render_parameters.max_composition_delay_in_frames) {
-      _timing->MaxCompositionDelayInFrames();
-  if (max_composition_delay_in_frames) {
     // Subtract frames that are in flight.
-    *max_composition_delay_in_frames -= timestamp_map_size;
+    render_parameters.max_composition_delay_in_frames =
-    *max_composition_delay_in_frames =
+        std::max(0, *render_parameters.max_composition_delay_in_frames -
-        std::max(0, *max_composition_delay_in_frames);
+                        timestamp_map_size);
-    decodedImage.set_max_composition_delay_in_frames(
-        max_composition_delay_in_frames);
   }
+  decodedImage.set_render_parameters(render_parameters);
 
   RTC_DCHECK(frameInfo->decodeStart);
   const Timestamp now = _clock->CurrentTime();

modules/video_coding/generic_decoder_unittest.cc

@@ -124,23 +124,43 @@ TEST_F(GenericDecoderTest, MaxCompositionDelayNotSetByDefault) {
   generic_decoder_.Decode(encoded_frame, clock_.CurrentTime());
   absl::optional<VideoFrame> decoded_frame = user_callback_.WaitForFrame(10);
   ASSERT_TRUE(decoded_frame.has_value());
-  EXPECT_FALSE(decoded_frame->max_composition_delay_in_frames());
+  EXPECT_THAT(
+      decoded_frame->render_parameters().max_composition_delay_in_frames,
+      testing::Eq(absl::nullopt));
 }
 
 TEST_F(GenericDecoderTest, MaxCompositionDelayActivatedByPlayoutDelay) {
   VCMEncodedFrame encoded_frame;
   // VideoReceiveStream2 would set MaxCompositionDelayInFrames if playout delay
   // is specified as X,Y, where X=0, Y>0.
-  const VideoPlayoutDelay kPlayoutDelay = {0, 50};
   constexpr int kMaxCompositionDelayInFrames = 3;  // ~50 ms at 60 fps.
-  encoded_frame.SetPlayoutDelay(kPlayoutDelay);
   timing_.SetMaxCompositionDelayInFrames(
       absl::make_optional(kMaxCompositionDelayInFrames));
   generic_decoder_.Decode(encoded_frame, clock_.CurrentTime());
   absl::optional<VideoFrame> decoded_frame = user_callback_.WaitForFrame(10);
   ASSERT_TRUE(decoded_frame.has_value());
-  EXPECT_EQ(kMaxCompositionDelayInFrames,
+  EXPECT_THAT(
-            decoded_frame->max_composition_delay_in_frames());
+      decoded_frame->render_parameters().max_composition_delay_in_frames,
+      testing::Optional(kMaxCompositionDelayInFrames));
+}
+
+TEST_F(GenericDecoderTest, IsLowLatencyStreamFalseByDefault) {
+  VCMEncodedFrame encoded_frame;
+  generic_decoder_.Decode(encoded_frame, clock_.CurrentTime());
+  absl::optional<VideoFrame> decoded_frame = user_callback_.WaitForFrame(10);
+  ASSERT_TRUE(decoded_frame.has_value());
+  EXPECT_FALSE(decoded_frame->render_parameters().use_low_latency_rendering);
+}
+
+TEST_F(GenericDecoderTest, IsLowLatencyStreamActivatedByPlayoutDelay) {
+  VCMEncodedFrame encoded_frame;
+  const VideoPlayoutDelay kPlayoutDelay = {0, 50};
+  timing_.set_min_playout_delay(TimeDelta::Millis(kPlayoutDelay.min_ms));
+  timing_.set_max_playout_delay(TimeDelta::Millis(kPlayoutDelay.max_ms));
+  generic_decoder_.Decode(encoded_frame, clock_.CurrentTime());
+  absl::optional<VideoFrame> decoded_frame = user_callback_.WaitForFrame(10);
+  ASSERT_TRUE(decoded_frame.has_value());
+  EXPECT_TRUE(decoded_frame->render_parameters().use_low_latency_rendering);
 }
 
 }  // namespace video_coding

modules/video_coding/timing/BUILD.gn

@@ -71,6 +71,7 @@ rtc_library("timing_module") {
     ":codec_timer",
     "../../../api:field_trials_view",
     "../../../api/units:time_delta",
+    "../../../api/video:video_frame",
     "../../../api/video:video_rtp_headers",
     "../../../rtc_base:logging",
     "../../../rtc_base:macromagic",

modules/video_coding/timing/timing.cc

@@ -23,7 +23,8 @@ namespace {
 
 // Default pacing that is used for the low-latency renderer path.
 constexpr TimeDelta kZeroPlayoutDelayDefaultMinPacing = TimeDelta::Millis(8);
-constexpr TimeDelta kLowLatencyRendererMaxPlayoutDelay = TimeDelta::Millis(500);
+constexpr TimeDelta kLowLatencyStreamMaxPlayoutDelayThreshold =
+    TimeDelta::Millis(500);
 
 void CheckDelaysValid(TimeDelta min_delay, TimeDelta max_delay) {
   if (min_delay > max_delay) {
@@ -191,9 +192,7 @@ void VCMTiming::SetLastDecodeScheduledTimestamp(
 
 Timestamp VCMTiming::RenderTimeInternal(uint32_t frame_timestamp,
                                         Timestamp now) const {
-  if (min_playout_delay_.IsZero() &&
+  if (UseLowLatencyRendering()) {
-      (max_playout_delay_.IsZero() ||
-       max_playout_delay_ <= kLowLatencyRendererMaxPlayoutDelay)) {
     // Render as soon as possible or with low-latency renderer algorithm.
     return Timestamp::Zero();
   }
@@ -250,6 +249,21 @@ TimeDelta VCMTiming::TargetDelayInternal() const {
                   jitter_delay_ + RequiredDecodeTime() + render_delay_);
 }
 
+VideoFrame::RenderParameters VCMTiming::RenderParameters() const {
+  MutexLock lock(&mutex_);
+  return {.use_low_latency_rendering = UseLowLatencyRendering(),
+          .max_composition_delay_in_frames = max_composition_delay_in_frames_};
+}
+
+bool VCMTiming::UseLowLatencyRendering() const {
+  // min_playout_delay_==0,
+  // max_playout_delay_<=kLowLatencyStreamMaxPlayoutDelayThreshold indicates
+  // that the low-latency path should be used, which means that frames should be
+  // decoded and rendered as soon as possible.
+  return min_playout_delay_.IsZero() &&
+         max_playout_delay_ <= kLowLatencyStreamMaxPlayoutDelayThreshold;
+}
+
 VCMTiming::VideoDelayTimings VCMTiming::GetTimings() const {
   MutexLock lock(&mutex_);
   return VideoDelayTimings{.max_decode_duration = RequiredDecodeTime(),
@@ -278,9 +292,4 @@ void VCMTiming::SetMaxCompositionDelayInFrames(
   max_composition_delay_in_frames_ = max_composition_delay_in_frames;
 }
 
-absl::optional<int> VCMTiming::MaxCompositionDelayInFrames() const {
-  MutexLock lock(&mutex_);
-  return max_composition_delay_in_frames_;
-}
-
 }  // namespace webrtc

modules/video_coding/timing/timing.h

@@ -16,6 +16,7 @@
 #include "absl/types/optional.h"
 #include "api/field_trials_view.h"
 #include "api/units/time_delta.h"
+#include "api/video/video_frame.h"
 #include "api/video/video_timing.h"
 #include "modules/video_coding/timing/codec_timer.h"
 #include "rtc_base/experiments/field_trial_parser.h"
@@ -111,7 +112,8 @@ class VCMTiming {
 
   void SetMaxCompositionDelayInFrames(
       absl::optional<int> max_composition_delay_in_frames);
-  absl::optional<int> MaxCompositionDelayInFrames() const;
+
+  VideoFrame::RenderParameters RenderParameters() const;
 
   // Updates the last time a frame was scheduled for decoding.
   void SetLastDecodeScheduledTimestamp(Timestamp last_decode_scheduled);
@@ -121,6 +123,7 @@ class VCMTiming {
   Timestamp RenderTimeInternal(uint32_t frame_timestamp, Timestamp now) const
       RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
   TimeDelta TargetDelayInternal() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+  bool UseLowLatencyRendering() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
 
  private:
   mutable Mutex mutex_;

modules/video_coding/timing/timing_unittest.cc

@@ -138,6 +138,31 @@ TEST(ReceiverTimingTest, TimestampWrapAround) {
   }
 }
 
+TEST(ReceiverTimingTest, UseLowLatencyRenderer) {
+  test::ScopedKeyValueConfig field_trials;
+  SimulatedClock clock(0);
+  VCMTiming timing(&clock, field_trials);
+  timing.Reset();
+  // Default is false.
+  EXPECT_FALSE(timing.RenderParameters().use_low_latency_rendering);
+  // False if min playout delay > 0.
+  timing.set_min_playout_delay(TimeDelta::Millis(10));
+  timing.set_max_playout_delay(TimeDelta::Millis(20));
+  EXPECT_FALSE(timing.RenderParameters().use_low_latency_rendering);
+  // True if min==0, max > 0.
+  timing.set_min_playout_delay(TimeDelta::Millis(0));
+  EXPECT_TRUE(timing.RenderParameters().use_low_latency_rendering);
+  // True if min==max==0.
+  timing.set_max_playout_delay(TimeDelta::Millis(0));
+  EXPECT_TRUE(timing.RenderParameters().use_low_latency_rendering);
+  // True also for max playout delay==500 ms.
+  timing.set_max_playout_delay(TimeDelta::Millis(500));
+  EXPECT_TRUE(timing.RenderParameters().use_low_latency_rendering);
+  // False if max playout delay > 500 ms.
+  timing.set_max_playout_delay(TimeDelta::Millis(501));
+  EXPECT_FALSE(timing.RenderParameters().use_low_latency_rendering);
+}
+
 TEST(ReceiverTimingTest, MaxWaitingTimeIsZeroForZeroRenderTime) {
   // This is the default path when the RTP playout delay header extension is set
   // to min==0 and max==0.

video/video_receive_stream2.cc

@@ -992,6 +992,7 @@ void VideoReceiveStream2::UpdatePlayoutDelays() const {
   const std::initializer_list<absl::optional<TimeDelta>> min_delays = {
       frame_minimum_playout_delay_, base_minimum_playout_delay_,
       syncable_minimum_playout_delay_};
+
   // Since nullopt < anything, this will return the largest of the minumum
   // delays, or nullopt if all are nullopt.
   absl::optional<TimeDelta> minimum_delay = std::max(min_delays);

video/video_receive_stream2_unittest.cc

@@ -88,6 +88,7 @@ using ::testing::Field;
 using ::testing::InSequence;
 using ::testing::Invoke;
 using ::testing::IsEmpty;
+using ::testing::Optional;
 using ::testing::Pointee;
 using ::testing::Property;
 using ::testing::Return;
@@ -386,38 +387,94 @@ TEST_P(VideoReceiveStream2Test, PlayoutDelayPreservesDefaultMinValue) {
   EXPECT_EQ(default_min_playout_latency, timings.min_playout_delay);
 }
 
-TEST_P(VideoReceiveStream2Test, MaxCompositionDelayNotSetByDefault) {
+TEST_P(VideoReceiveStream2Test, RenderParametersSetToDefaultValues) {
+  // Default render parameters.
+  const VideoFrame::RenderParameters kDefaultRenderParameters;
   // Default with no playout delay set.
   std::unique_ptr<test::FakeEncodedFrame> test_frame0 =
       test::FakeFrameBuilder().Id(0).AsLast().Build();
   video_receive_stream_->OnCompleteFrame(std::move(test_frame0));
-  EXPECT_FALSE(timing_->MaxCompositionDelayInFrames());
+  EXPECT_EQ(timing_->RenderParameters(), kDefaultRenderParameters);
+}
+
+TEST_P(VideoReceiveStream2Test, UseLowLatencyRenderingSetFromPlayoutDelay) {
+  // use_low_latency_rendering set if playout delay set to min=0, max<=500 ms.
+  std::unique_ptr<test::FakeEncodedFrame> test_frame0 =
+      test::FakeFrameBuilder().Id(0).AsLast().Build();
+  test_frame0->SetPlayoutDelay({/*min_ms=*/0, /*max_ms=*/0});
+  video_receive_stream_->OnCompleteFrame(std::move(test_frame0));
+  EXPECT_TRUE(timing_->RenderParameters().use_low_latency_rendering);
 
-  // Max composition delay not set for playout delay 0,0.
   std::unique_ptr<test::FakeEncodedFrame> test_frame1 =
       test::FakeFrameBuilder().Id(1).AsLast().Build();
-  test_frame1->SetPlayoutDelay({0, 0});
+  test_frame1->SetPlayoutDelay({/*min_ms=*/0, /*max_ms=*/500});
   video_receive_stream_->OnCompleteFrame(std::move(test_frame1));
-  EXPECT_FALSE(timing_->MaxCompositionDelayInFrames());
+  EXPECT_TRUE(timing_->RenderParameters().use_low_latency_rendering);
-
-  // Max composition delay not set for playout delay X,Y, where X,Y>0.
-  std::unique_ptr<test::FakeEncodedFrame> test_frame2 =
-      test::FakeFrameBuilder().Id(2).AsLast().Build();
-  test_frame2->SetPlayoutDelay({10, 30});
-  video_receive_stream_->OnCompleteFrame(std::move(test_frame2));
-  EXPECT_FALSE(timing_->MaxCompositionDelayInFrames());
 }
 
 TEST_P(VideoReceiveStream2Test, MaxCompositionDelaySetFromMaxPlayoutDelay) {
+  // The max composition delay is dependent on the number of frames in the
+  // pre-decode queue. It's therefore important to advance the time as the test
+  // runs to get the correct expectations of max_composition_delay_in_frames.
+  video_receive_stream_->Start();
+  // Max composition delay not set if no playout delay is set.
+  std::unique_ptr<test::FakeEncodedFrame> test_frame0 =
+      test::FakeFrameBuilder()
+          .Id(0)
+          .Time(RtpTimestampForFrame(0))
+          .ReceivedTime(ReceiveTimeForFrame(0))
+          .AsLast()
+          .Build();
+  video_receive_stream_->OnCompleteFrame(std::move(test_frame0));
+  EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames,
+              Eq(absl::nullopt));
+  time_controller_.AdvanceTime(k30FpsDelay);
+  loop_.Flush();
+
+  // Max composition delay not set for playout delay 0,0.
+  std::unique_ptr<test::FakeEncodedFrame> test_frame1 =
+      test::FakeFrameBuilder()
+          .Id(1)
+          .Time(RtpTimestampForFrame(1))
+          .ReceivedTime(ReceiveTimeForFrame(1))
+          .AsLast()
+          .Build();
+  test_frame1->SetPlayoutDelay({0, 0});
+  video_receive_stream_->OnCompleteFrame(std::move(test_frame1));
+  EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames,
+              Eq(absl::nullopt));
+  time_controller_.AdvanceTime(k30FpsDelay);
+  loop_.Flush();
+
+  // Max composition delay not set for playout delay X,Y, where X,Y>0.
+  std::unique_ptr<test::FakeEncodedFrame> test_frame2 =
+      test::FakeFrameBuilder()
+          .Id(2)
+          .Time(RtpTimestampForFrame(2))
+          .ReceivedTime(ReceiveTimeForFrame(2))
+          .AsLast()
+          .Build();
+  test_frame2->SetPlayoutDelay({10, 30});
+  video_receive_stream_->OnCompleteFrame(std::move(test_frame2));
+  EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames,
+              Eq(absl::nullopt));
+
+  time_controller_.AdvanceTime(k30FpsDelay);
+  loop_.Flush();
+
   // Max composition delay set if playout delay X,Y, where X=0,Y>0.
-  const VideoPlayoutDelay kPlayoutDelayMs = {0, 50};
   const int kExpectedMaxCompositionDelayInFrames = 3;  // ~50 ms at 60 fps.
-  std::unique_ptr<test::FakeEncodedFrame> test_frame =
+  std::unique_ptr<test::FakeEncodedFrame> test_frame3 =
-      test::FakeFrameBuilder().Id(0).AsLast().Build();
+      test::FakeFrameBuilder()
-  test_frame->SetPlayoutDelay(kPlayoutDelayMs);
+          .Id(3)
-  video_receive_stream_->OnCompleteFrame(std::move(test_frame));
+          .Time(RtpTimestampForFrame(3))
-  EXPECT_EQ(kExpectedMaxCompositionDelayInFrames,
+          .ReceivedTime(ReceiveTimeForFrame(3))
-            timing_->MaxCompositionDelayInFrames());
+          .AsLast()
+          .Build();
+  test_frame3->SetPlayoutDelay({0, 50});
+  video_receive_stream_->OnCompleteFrame(std::move(test_frame3));
+  EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames,
+              Optional(kExpectedMaxCompositionDelayInFrames));
 }
 
 TEST_P(VideoReceiveStream2Test, LazyDecoderCreation) {