Compute video freeze metrics on rendered frames instead of on decoded

Bug: webrtc:9828
Change-Id: I1390c736785759a2d8712e71398db4f5069ebcd4
Reviewed-on: https://webrtc-review.googlesource.com/c/105100
Reviewed-by: Stefan Holmer <stefan@webrtc.org>
Commit-Queue: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#25116}

video/receive_statistics_proxy.cc

@@ -759,6 +759,9 @@ void ReceiveStatisticsProxy::OnRenderedFrame(const VideoFrame& frame) {
   RTC_DCHECK_GT(height, 0);
   int64_t now_ms = clock_->TimeInMilliseconds();
   rtc::CritScope lock(&crit_);
+
+  video_quality_observer_->OnRenderedFrame(now_ms);
+
   ContentSpecificStats* content_specific_stats =
       &content_specific_stats_[last_content_type_];
   renders_fps_estimator_.Update(1, now_ms);

video/receive_statistics_proxy_unittest.cc

@@ -1057,15 +1057,19 @@ TEST_P(ReceiveStatisticsProxyTest, FreezesAreReported) {
   const int kFreezeDelayMs = 200;
   const int kCallDurationMs =
       kMinRequiredSamples * kInterFrameDelayMs + kFreezeDelayMs;
+  webrtc::VideoFrame frame(webrtc::I420Buffer::Create(1, 1), 0, 0,
+                           webrtc::kVideoRotation_0);
   for (int i = 0; i < kMinRequiredSamples; ++i) {
     statistics_proxy_->OnDecodedFrame(absl::nullopt, kWidth, kHeight,
                                       content_type);
+    statistics_proxy_->OnRenderedFrame(frame);
     fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs);
   }
   // Add extra freeze.
   fake_clock_.AdvanceTimeMilliseconds(kFreezeDelayMs);
   statistics_proxy_->OnDecodedFrame(absl::nullopt, kWidth, kHeight,
                                     content_type);
+  statistics_proxy_->OnRenderedFrame(frame);
 
   statistics_proxy_.reset();
   const int kExpectedTimeBetweenFreezes =
@@ -1095,9 +1099,12 @@ TEST_P(ReceiveStatisticsProxyTest, PausesAreIgnored) {
   const VideoContentType content_type = GetParam();
   const int kInterFrameDelayMs = 33;
   const int kPauseDurationMs = 10000;
+  webrtc::VideoFrame frame(webrtc::I420Buffer::Create(1, 1), 0, 0,
+                           webrtc::kVideoRotation_0);
   for (int i = 0; i <= kMinRequiredSamples; ++i) {
     statistics_proxy_->OnDecodedFrame(absl::nullopt, kWidth, kHeight,
                                       content_type);
+    statistics_proxy_->OnRenderedFrame(frame);
     fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs);
   }
   // Add a pause.
@@ -1108,6 +1115,7 @@ TEST_P(ReceiveStatisticsProxyTest, PausesAreIgnored) {
   for (int i = 0; i <= kMinRequiredSamples * 3; ++i) {
     statistics_proxy_->OnDecodedFrame(absl::nullopt, kWidth, kHeight,
                                       content_type);
+    statistics_proxy_->OnRenderedFrame(frame);
     fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs);
   }
 

video/video_quality_observer.cc

@@ -34,7 +34,9 @@ const int kBlockyQpThresholdVp9 = 60;  // TODO(ilnik): tune this value.
 
 VideoQualityObserver::VideoQualityObserver(VideoContentType content_type)
     : last_frame_decoded_ms_(-1),
+      last_frame_rendered_ms_(-1),
       num_frames_decoded_(0),
+      num_frames_rendered_(0),
       first_frame_decoded_ms_(-1),
       last_frame_pixels_(0),
       last_frame_qp_(0),
@@ -119,6 +121,46 @@ void VideoQualityObserver::UpdateHistograms() {
   RTC_LOG(LS_INFO) << log_stream.str();
 }
 
+void VideoQualityObserver::OnRenderedFrame(int64_t now_ms) {
+  if (num_frames_rendered_ == 0) {
+    last_unfreeze_time_ = now_ms;
+  }
+
+  ++num_frames_rendered_;
+
+  if (!is_paused_ && num_frames_rendered_ > 1) {
+    // Process inter-frame delay.
+    int64_t interframe_delay_ms = now_ms - last_frame_rendered_ms_;
+    render_interframe_delays_.Add(interframe_delay_ms);
+    absl::optional<int> avg_interframe_delay =
+        render_interframe_delays_.Avg(kMinFrameSamplesToDetectFreeze);
+    // Check if it was a freeze.
+    if (avg_interframe_delay &&
+        interframe_delay_ms >=
+            std::max(3 * *avg_interframe_delay,
+                     *avg_interframe_delay + kMinIncreaseForFreezeMs)) {
+      freezes_durations_.Add(interframe_delay_ms);
+      smooth_playback_durations_.Add(last_frame_rendered_ms_ -
+                                     last_unfreeze_time_);
+      last_unfreeze_time_ = now_ms;
+    }
+  }
+
+  if (is_paused_) {
+    // If the stream was paused since the previous frame, do not count the
+    // pause toward smooth playback. Explicitly count the part before it and
+    // start the new smooth playback interval from this frame.
+    is_paused_ = false;
+    if (last_frame_rendered_ms_ > last_unfreeze_time_) {
+      smooth_playback_durations_.Add(last_frame_rendered_ms_ -
+                                     last_unfreeze_time_);
+    }
+    last_unfreeze_time_ = now_ms;
+  }
+
+  last_frame_rendered_ms_ = now_ms;
+}
+
 void VideoQualityObserver::OnDecodedFrame(absl::optional<uint8_t> qp,
                                           int width,
                                           int height,
@@ -126,7 +168,6 @@ void VideoQualityObserver::OnDecodedFrame(absl::optional<uint8_t> qp,
                                           VideoCodecType codec) {
   if (num_frames_decoded_ == 0) {
     first_frame_decoded_ms_ = now_ms;
-    last_unfreeze_time_ = now_ms;
   }
 
   ++num_frames_decoded_;
@@ -134,21 +175,14 @@ void VideoQualityObserver::OnDecodedFrame(absl::optional<uint8_t> qp,
   if (!is_paused_ && num_frames_decoded_ > 1) {
     // Process inter-frame delay.
     int64_t interframe_delay_ms = now_ms - last_frame_decoded_ms_;
-    interframe_delays_.Add(interframe_delay_ms);
+    decode_interframe_delays_.Add(interframe_delay_ms);
     absl::optional<int> avg_interframe_delay =
-        interframe_delays_.Avg(kMinFrameSamplesToDetectFreeze);
+        decode_interframe_delays_.Avg(kMinFrameSamplesToDetectFreeze);
-    // Check if it was a freeze.
+    // Count spatial metrics if there were no freeze.
-    if (avg_interframe_delay &&
+    if (!avg_interframe_delay ||
-        interframe_delay_ms >=
+        interframe_delay_ms <
             std::max(3 * *avg_interframe_delay,
                      *avg_interframe_delay + kMinIncreaseForFreezeMs)) {
-      freezes_durations_.Add(interframe_delay_ms);
-      smooth_playback_durations_.Add(last_frame_decoded_ms_ -
-                                     last_unfreeze_time_);
-      last_unfreeze_time_ = now_ms;
-    } else {
-      // Only count inter-frame delay as playback time if there
-      // was no freeze.
       time_in_resolution_ms_[current_resolution_] += interframe_delay_ms;
       absl::optional<int> qp_blocky_threshold;
       // TODO(ilnik): add other codec types when we have QP for them.
@@ -168,18 +202,6 @@ void VideoQualityObserver::OnDecodedFrame(absl::optional<uint8_t> qp,
     }
   }
 
-  if (is_paused_) {
-    // If the stream was paused since the previous frame, do not count the
-    // pause toward smooth playback. Explicitly count the part before it and
-    // start the new smooth playback interval from this frame.
-    is_paused_ = false;
-    if (last_frame_decoded_ms_ > last_unfreeze_time_) {
-      smooth_playback_durations_.Add(last_frame_decoded_ms_ -
-                                     last_unfreeze_time_);
-    }
-    last_unfreeze_time_ = now_ms;
-  }
-
   int64_t pixels = width * height;
   if (pixels >= kPixelsInHighResolution) {
     current_resolution_ = Resolution::High;

video/video_quality_observer.h

@@ -35,6 +35,7 @@ class VideoQualityObserver {
                       int height,
                       int64_t now_ms,
                       VideoCodecType codec);
+  void OnRenderedFrame(int64_t now_ms);
 
   void OnStreamInactive();
 
@@ -48,13 +49,16 @@ class VideoQualityObserver {
   };
 
   int64_t last_frame_decoded_ms_;
+  int64_t last_frame_rendered_ms_;
   int64_t num_frames_decoded_;
+  int64_t num_frames_rendered_;
   int64_t first_frame_decoded_ms_;
   int64_t last_frame_pixels_;
   uint8_t last_frame_qp_;
   // Decoded timestamp of the last delayed frame.
   int64_t last_unfreeze_time_;
-  rtc::SampleCounter interframe_delays_;
+  rtc::SampleCounter render_interframe_delays_;
+  rtc::SampleCounter decode_interframe_delays_;
   // An inter-frame delay is counted as a freeze if it's significantly longer
   // than average inter-frame delay.
   rtc::SampleCounter freezes_durations_;