diff --git a/video/BUILD.gn b/video/BUILD.gn index bc026d22af..2ef8473825 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -38,6 +38,8 @@ rtc_static_library("video") { "stream_synchronization.h", "transport_adapter.cc", "transport_adapter.h", + "video_quality_observer.cc", + "video_quality_observer.h", "video_receive_stream.cc", "video_receive_stream.h", "video_send_stream.cc", diff --git a/video/receive_statistics_proxy.cc b/video/receive_statistics_proxy.cc index 2cccd5a973..2fd5d9be72 100644 --- a/video/receive_statistics_proxy.cc +++ b/video/receive_statistics_proxy.cc @@ -106,11 +106,14 @@ ReceiveStatisticsProxy::ReceiveStatisticsProxy( render_fps_tracker_(100, 10u), render_pixel_tracker_(100, 10u), total_byte_tracker_(100, 10u), // bucket_interval_ms, bucket_count + video_quality_observer_( + new VideoQualityObserver(VideoContentType::UNSPECIFIED)), interframe_delay_max_moving_(kMovingMaxWindowMs), freq_offset_counter_(clock, nullptr, kFreqOffsetProcessIntervalMs), first_report_block_time_ms_(-1), avg_rtt_ms_(0), last_content_type_(VideoContentType::UNSPECIFIED), + last_codec_type_(kVideoCodecVP8), timing_frame_info_counter_(kMovingMaxWindowMs) { decode_thread_.DetachFromThread(); network_thread_.DetachFromThread(); @@ -147,13 +150,14 @@ void ReceiveStatisticsProxy::UpdateHistograms() { << stream_duration_sec << '\n'; } - log_stream << "Frames decoded " << stats_.frames_decoded; + log_stream << "Frames decoded " << stats_.frames_decoded << '\n'; if (num_unique_frames_) { int num_dropped_frames = *num_unique_frames_ - stats_.frames_decoded; RTC_HISTOGRAM_COUNTS_1000("WebRTC.Video.DroppedFrames.Receiver", num_dropped_frames); - log_stream << "WebRTC.Video.DroppedFrames.Receiver " << num_dropped_frames; + log_stream << "WebRTC.Video.DroppedFrames.Receiver " << num_dropped_frames + << '\n'; } if (first_report_block_time_ms_ != -1 && @@ -680,12 +684,30 @@ void ReceiveStatisticsProxy::DataCountersUpdated( total_byte_tracker_.AddSamples(total_bytes - last_total_bytes); } +// Deprecated. TODO(ilnik): remove once all depending projects are updated. void ReceiveStatisticsProxy::OnDecodedFrame(rtc::Optional qp, VideoContentType content_type) { + OnDecodedFrame(qp, 0, 0, content_type); +} + +void ReceiveStatisticsProxy::OnDecodedFrame(rtc::Optional qp, + int width, + int height, + VideoContentType content_type) { rtc::CritScope lock(&crit_); uint64_t now = clock_->TimeInMilliseconds(); + if (videocontenttypehelpers::IsScreenshare(content_type) != + videocontenttypehelpers::IsScreenshare(last_content_type_)) { + // Reset the quality observer if content type is switched. This will + // report stats for the previous part of the call. + video_quality_observer_.reset(new VideoQualityObserver(content_type)); + } + + video_quality_observer_->OnDecodedFrame(qp, width, height, now, + last_codec_type_); + ContentSpecificStats* content_specific_stats = &content_specific_stats_[content_type]; ++stats_.frames_decoded; @@ -809,9 +831,10 @@ void ReceiveStatisticsProxy::OnPreDecode( if (!codec_specific_info || encoded_image.qp_ == -1) { return; } - if (codec_specific_info->codecType == kVideoCodecVP8) { + rtc::CritScope lock(&crit_); + last_codec_type_ = codec_specific_info->codecType; + if (last_codec_type_ == kVideoCodecVP8) { qp_counters_.vp8.Add(encoded_image.qp_); - rtc::CritScope lock(&crit_); qp_sample_.Add(encoded_image.qp_); } } @@ -822,6 +845,7 @@ void ReceiveStatisticsProxy::OnStreamInactive() { rtc::CritScope lock(&crit_); // Don't report inter-frame delay if stream was paused. last_decoded_frame_time_ms_.reset(); + video_quality_observer_->OnStreamInactive(); } void ReceiveStatisticsProxy::OnRttUpdate(int64_t avg_rtt_ms, diff --git a/video/receive_statistics_proxy.h b/video/receive_statistics_proxy.h index 9a2185e315..72fd2b2b21 100644 --- a/video/receive_statistics_proxy.h +++ b/video/receive_statistics_proxy.h @@ -12,6 +12,7 @@ #define VIDEO_RECEIVE_STATISTICS_PROXY_H_ #include +#include #include #include @@ -31,6 +32,7 @@ #include "video/quality_threshold.h" #include "video/report_block_stats.h" #include "video/stats_counter.h" +#include "video/video_quality_observer.h" #include "video/video_stream_decoder.h" namespace webrtc { @@ -50,7 +52,14 @@ class ReceiveStatisticsProxy : public VCMReceiveStatisticsCallback, VideoReceiveStream::Stats GetStats() const; - void OnDecodedFrame(rtc::Optional qp, VideoContentType content_type); + // Deprecated. TODO(ilnik): remove once all depending projects are updated. + RTC_DEPRECATED void OnDecodedFrame(rtc::Optional qp, + VideoContentType content_type); + + void OnDecodedFrame(rtc::Optional qp, + int width, + int height, + VideoContentType content_type); void OnSyncOffsetUpdated(int64_t sync_offset_ms, double estimated_freq_khz); void OnRenderedFrame(const VideoFrame& frame); void OnIncomingPayloadType(int payload_type); @@ -163,6 +172,8 @@ class ReceiveStatisticsProxy : public VCMReceiveStatisticsCallback, rtc::SampleCounter target_delay_counter_ RTC_GUARDED_BY(crit_); rtc::SampleCounter current_delay_counter_ RTC_GUARDED_BY(crit_); rtc::SampleCounter delay_counter_ RTC_GUARDED_BY(crit_); + std::unique_ptr video_quality_observer_ + RTC_GUARDED_BY(crit_); mutable rtc::MovingMaxCounter interframe_delay_max_moving_ RTC_GUARDED_BY(crit_); std::map content_specific_stats_ @@ -175,6 +186,7 @@ class ReceiveStatisticsProxy : public VCMReceiveStatisticsCallback, int64_t avg_rtt_ms_ RTC_GUARDED_BY(crit_); mutable std::map frame_window_ RTC_GUARDED_BY(&crit_); VideoContentType last_content_type_ RTC_GUARDED_BY(&crit_); + VideoCodecType last_codec_type_ RTC_GUARDED_BY(&crit_); rtc::Optional first_decoded_frame_time_ms_ RTC_GUARDED_BY(&crit_); rtc::Optional last_decoded_frame_time_ms_ RTC_GUARDED_BY(&crit_); // Mutable because calling Max() on MovingMaxCounter is not const. Yet it is diff --git a/video/receive_statistics_proxy_unittest.cc b/video/receive_statistics_proxy_unittest.cc index d604429939..f61f0865ac 100644 --- a/video/receive_statistics_proxy_unittest.cc +++ b/video/receive_statistics_proxy_unittest.cc @@ -28,6 +28,10 @@ const int64_t kFreqOffsetProcessIntervalInMs = 40000; const uint32_t kLocalSsrc = 123; const uint32_t kRemoteSsrc = 456; const int kMinRequiredSamples = 200; + +const int kWidth = 1280; +const int kHeight = 720; + } // namespace // TODO(sakal): ReceiveStatisticsProxy is lacking unittesting. @@ -70,7 +74,7 @@ class ReceiveStatisticsProxyTest TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameIncreasesFramesDecoded) { EXPECT_EQ(0u, statistics_proxy_->GetStats().frames_decoded); for (uint32_t i = 1; i <= 3; ++i) { - statistics_proxy_->OnDecodedFrame(rtc::nullopt, + statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight, VideoContentType::UNSPECIFIED); EXPECT_EQ(i, statistics_proxy_->GetStats().frames_decoded); } @@ -80,7 +84,7 @@ TEST_F(ReceiveStatisticsProxyTest, DecodedFpsIsReported) { const int kFps = 20; const int kRequiredSamples = metrics::kMinRunTimeInSeconds * kFps; for (int i = 0; i < kRequiredSamples; ++i) { - statistics_proxy_->OnDecodedFrame(rtc::Optional(), + statistics_proxy_->OnDecodedFrame(rtc::Optional(), kWidth, kHeight, VideoContentType::UNSPECIFIED); fake_clock_.AdvanceTimeMilliseconds(1000 / kFps); } @@ -93,7 +97,7 @@ TEST_F(ReceiveStatisticsProxyTest, DecodedFpsIsNotReportedForTooFewSamples) { const int kFps = 20; const int kRequiredSamples = metrics::kMinRunTimeInSeconds * kFps; for (int i = 0; i < kRequiredSamples - 1; ++i) { - statistics_proxy_->OnDecodedFrame(rtc::Optional(), + statistics_proxy_->OnDecodedFrame(rtc::Optional(), kWidth, kHeight, VideoContentType::UNSPECIFIED); fake_clock_.AdvanceTimeMilliseconds(1000 / kFps); } @@ -104,15 +108,15 @@ TEST_F(ReceiveStatisticsProxyTest, DecodedFpsIsNotReportedForTooFewSamples) { TEST_F(ReceiveStatisticsProxyTest, DecodedFpsIsReportedWithQpReset) { const int kFps1 = 10; for (int i = 0; i < metrics::kMinRunTimeInSeconds * kFps1; ++i) { - statistics_proxy_->OnDecodedFrame(rtc::Optional(), + statistics_proxy_->OnDecodedFrame(rtc::Optional(), kWidth, kHeight, VideoContentType::UNSPECIFIED); fake_clock_.AdvanceTimeMilliseconds(1000 / kFps1); } // First QP value received, resets frames decoded. const int kFps2 = 20; for (int i = 0; i < metrics::kMinRunTimeInSeconds * kFps2; ++i) { - statistics_proxy_->OnDecodedFrame(rtc::Optional(1u), - VideoContentType::UNSPECIFIED); + statistics_proxy_->OnDecodedFrame(rtc::Optional(1u), kWidth, + kHeight, VideoContentType::UNSPECIFIED); fake_clock_.AdvanceTimeMilliseconds(1000 / kFps2); } statistics_proxy_.reset(); @@ -124,19 +128,22 @@ TEST_F(ReceiveStatisticsProxyTest, DecodedFpsIsReportedWithQpReset) { TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameWithQpResetsFramesDecoded) { EXPECT_EQ(0u, statistics_proxy_->GetStats().frames_decoded); for (uint32_t i = 1; i <= 3; ++i) { - statistics_proxy_->OnDecodedFrame(rtc::nullopt, + statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight, VideoContentType::UNSPECIFIED); EXPECT_EQ(i, statistics_proxy_->GetStats().frames_decoded); } - statistics_proxy_->OnDecodedFrame(1u, VideoContentType::UNSPECIFIED); + statistics_proxy_->OnDecodedFrame(1u, kWidth, kHeight, + VideoContentType::UNSPECIFIED); EXPECT_EQ(1u, statistics_proxy_->GetStats().frames_decoded); } TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameIncreasesQpSum) { EXPECT_EQ(rtc::nullopt, statistics_proxy_->GetStats().qp_sum); - statistics_proxy_->OnDecodedFrame(3u, VideoContentType::UNSPECIFIED); + statistics_proxy_->OnDecodedFrame(3u, kWidth, kHeight, + VideoContentType::UNSPECIFIED); EXPECT_EQ(3u, statistics_proxy_->GetStats().qp_sum); - statistics_proxy_->OnDecodedFrame(127u, VideoContentType::UNSPECIFIED); + statistics_proxy_->OnDecodedFrame(127u, kWidth, kHeight, + VideoContentType::UNSPECIFIED); EXPECT_EQ(130u, statistics_proxy_->GetStats().qp_sum); } @@ -145,10 +152,12 @@ TEST_F(ReceiveStatisticsProxyTest, ReportsContentType) { const std::string kScreenshareString("screen"); EXPECT_EQ(kRealtimeString, videocontenttypehelpers::ToString( statistics_proxy_->GetStats().content_type)); - statistics_proxy_->OnDecodedFrame(3u, VideoContentType::SCREENSHARE); + statistics_proxy_->OnDecodedFrame(3u, kWidth, kHeight, + VideoContentType::SCREENSHARE); EXPECT_EQ(kScreenshareString, videocontenttypehelpers::ToString( statistics_proxy_->GetStats().content_type)); - statistics_proxy_->OnDecodedFrame(3u, VideoContentType::UNSPECIFIED); + statistics_proxy_->OnDecodedFrame(3u, kWidth, kHeight, + VideoContentType::UNSPECIFIED); EXPECT_EQ(kRealtimeString, videocontenttypehelpers::ToString( statistics_proxy_->GetStats().content_type)); } @@ -158,21 +167,25 @@ TEST_F(ReceiveStatisticsProxyTest, ReportsMaxInterframeDelay) { const int64_t kInterframeDelayMs2 = 200; const int64_t kInterframeDelayMs3 = 100; EXPECT_EQ(-1, statistics_proxy_->GetStats().interframe_delay_max_ms); - statistics_proxy_->OnDecodedFrame(3u, VideoContentType::UNSPECIFIED); + statistics_proxy_->OnDecodedFrame(3u, kWidth, kHeight, + VideoContentType::UNSPECIFIED); EXPECT_EQ(-1, statistics_proxy_->GetStats().interframe_delay_max_ms); fake_clock_.AdvanceTimeMilliseconds(kInterframeDelayMs1); - statistics_proxy_->OnDecodedFrame(127u, VideoContentType::UNSPECIFIED); + statistics_proxy_->OnDecodedFrame(127u, kWidth, kHeight, + VideoContentType::UNSPECIFIED); EXPECT_EQ(kInterframeDelayMs1, statistics_proxy_->GetStats().interframe_delay_max_ms); fake_clock_.AdvanceTimeMilliseconds(kInterframeDelayMs2); - statistics_proxy_->OnDecodedFrame(127u, VideoContentType::UNSPECIFIED); + statistics_proxy_->OnDecodedFrame(127u, kWidth, kHeight, + VideoContentType::UNSPECIFIED); EXPECT_EQ(kInterframeDelayMs2, statistics_proxy_->GetStats().interframe_delay_max_ms); fake_clock_.AdvanceTimeMilliseconds(kInterframeDelayMs3); - statistics_proxy_->OnDecodedFrame(127u, VideoContentType::UNSPECIFIED); + statistics_proxy_->OnDecodedFrame(127u, kWidth, kHeight, + VideoContentType::UNSPECIFIED); // kInterframeDelayMs3 is smaller than kInterframeDelayMs2. EXPECT_EQ(kInterframeDelayMs2, statistics_proxy_->GetStats().interframe_delay_max_ms); @@ -183,22 +196,26 @@ TEST_F(ReceiveStatisticsProxyTest, ReportInterframeDelayInWindow) { const int64_t kInterframeDelayMs2 = 750; const int64_t kInterframeDelayMs3 = 700; EXPECT_EQ(-1, statistics_proxy_->GetStats().interframe_delay_max_ms); - statistics_proxy_->OnDecodedFrame(3u, VideoContentType::UNSPECIFIED); + statistics_proxy_->OnDecodedFrame(3u, kWidth, kHeight, + VideoContentType::UNSPECIFIED); EXPECT_EQ(-1, statistics_proxy_->GetStats().interframe_delay_max_ms); fake_clock_.AdvanceTimeMilliseconds(kInterframeDelayMs1); - statistics_proxy_->OnDecodedFrame(127u, VideoContentType::UNSPECIFIED); + statistics_proxy_->OnDecodedFrame(127u, kWidth, kHeight, + VideoContentType::UNSPECIFIED); EXPECT_EQ(kInterframeDelayMs1, statistics_proxy_->GetStats().interframe_delay_max_ms); fake_clock_.AdvanceTimeMilliseconds(kInterframeDelayMs2); - statistics_proxy_->OnDecodedFrame(127u, VideoContentType::UNSPECIFIED); + statistics_proxy_->OnDecodedFrame(127u, kWidth, kHeight, + VideoContentType::UNSPECIFIED); // Still first delay is the maximum EXPECT_EQ(kInterframeDelayMs1, statistics_proxy_->GetStats().interframe_delay_max_ms); fake_clock_.AdvanceTimeMilliseconds(kInterframeDelayMs3); - statistics_proxy_->OnDecodedFrame(127u, VideoContentType::UNSPECIFIED); + statistics_proxy_->OnDecodedFrame(127u, kWidth, kHeight, + VideoContentType::UNSPECIFIED); // Now the first sample is out of the window, so the second is the maximum. EXPECT_EQ(kInterframeDelayMs2, statistics_proxy_->GetStats().interframe_delay_max_ms); @@ -206,16 +223,17 @@ TEST_F(ReceiveStatisticsProxyTest, ReportInterframeDelayInWindow) { TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameWithoutQpQpSumWontExist) { EXPECT_EQ(rtc::nullopt, statistics_proxy_->GetStats().qp_sum); - statistics_proxy_->OnDecodedFrame(rtc::nullopt, + statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight, VideoContentType::UNSPECIFIED); EXPECT_EQ(rtc::nullopt, statistics_proxy_->GetStats().qp_sum); } TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameWithoutQpResetsQpSum) { EXPECT_EQ(rtc::nullopt, statistics_proxy_->GetStats().qp_sum); - statistics_proxy_->OnDecodedFrame(3u, VideoContentType::UNSPECIFIED); + statistics_proxy_->OnDecodedFrame(3u, kWidth, kHeight, + VideoContentType::UNSPECIFIED); EXPECT_EQ(3u, statistics_proxy_->GetStats().qp_sum); - statistics_proxy_->OnDecodedFrame(rtc::nullopt, + statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight, VideoContentType::UNSPECIFIED); EXPECT_EQ(rtc::nullopt, statistics_proxy_->GetStats().qp_sum); } @@ -718,7 +736,7 @@ TEST_F(ReceiveStatisticsProxyTest, DoesNotReportStaleFramerates) { // Since OnRenderedFrame is never called the fps in each sample will be 0, // i.e. bad frame.set_ntp_time_ms(fake_clock_.CurrentNtpInMilliseconds()); - statistics_proxy_->OnDecodedFrame(rtc::nullopt, + statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight, VideoContentType::UNSPECIFIED); statistics_proxy_->OnRenderedFrame(frame); fake_clock_.AdvanceTimeMilliseconds(1000 / kDefaultFps); @@ -833,12 +851,14 @@ TEST_P(ReceiveStatisticsProxyTest, InterFrameDelaysAreReported) { const VideoContentType content_type = GetParam(); const int kInterFrameDelayMs = 33; for (int i = 0; i < kMinRequiredSamples; ++i) { - statistics_proxy_->OnDecodedFrame(rtc::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight, + content_type); fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); } // One extra with double the interval. fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); - statistics_proxy_->OnDecodedFrame(rtc::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight, + content_type); statistics_proxy_.reset(); const int kExpectedInterFrame = @@ -866,16 +886,19 @@ TEST_P(ReceiveStatisticsProxyTest, InterFrameDelaysPercentilesAreReported) { const int kLastFivePercentsSamples = kMinRequiredSamples * 5 / 100; for (int i = 0; i <= kMinRequiredSamples - kLastFivePercentsSamples; ++i) { fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); - statistics_proxy_->OnDecodedFrame(rtc::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight, + content_type); } // Last 5% of intervals are double in size. for (int i = 0; i < kLastFivePercentsSamples; ++i) { fake_clock_.AdvanceTimeMilliseconds(2 * kInterFrameDelayMs); - statistics_proxy_->OnDecodedFrame(rtc::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight, + content_type); } // Final sample is outlier and 10 times as big. fake_clock_.AdvanceTimeMilliseconds(10 * kInterFrameDelayMs); - statistics_proxy_->OnDecodedFrame(rtc::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight, + content_type); statistics_proxy_.reset(); const int kExpectedInterFrame = kInterFrameDelayMs * 2; @@ -894,7 +917,8 @@ TEST_P(ReceiveStatisticsProxyTest, MaxInterFrameDelayOnlyWithValidAverage) { const VideoContentType content_type = GetParam(); const int kInterFrameDelayMs = 33; for (int i = 0; i < kMinRequiredSamples; ++i) { - statistics_proxy_->OnDecodedFrame(rtc::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight, + content_type); fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); } @@ -913,7 +937,8 @@ TEST_P(ReceiveStatisticsProxyTest, MaxInterFrameDelayOnlyWithPause) { const VideoContentType content_type = GetParam(); const int kInterFrameDelayMs = 33; for (int i = 0; i <= kMinRequiredSamples; ++i) { - statistics_proxy_->OnDecodedFrame(rtc::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight, + content_type); fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); } @@ -924,9 +949,11 @@ TEST_P(ReceiveStatisticsProxyTest, MaxInterFrameDelayOnlyWithPause) { // Insert two more frames. The interval during the pause should be disregarded // in the stats. - statistics_proxy_->OnDecodedFrame(rtc::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight, + content_type); fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); - statistics_proxy_->OnDecodedFrame(rtc::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight, + content_type); statistics_proxy_.reset(); if (videocontenttypehelpers::IsScreenshare(content_type)) { @@ -950,6 +977,174 @@ TEST_P(ReceiveStatisticsProxyTest, MaxInterFrameDelayOnlyWithPause) { } } +TEST_P(ReceiveStatisticsProxyTest, FreezesAreReported) { + const VideoContentType content_type = GetParam(); + const int kInterFrameDelayMs = 33; + const int kFreezeDelayMs = 200; + for (int i = 0; i < kMinRequiredSamples; ++i) { + statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight, + content_type); + fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); + } + // Add extra freeze. + fake_clock_.AdvanceTimeMilliseconds(kFreezeDelayMs); + statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight, + content_type); + + statistics_proxy_.reset(); + const int kExpectedTimeBetweenFreezes = + kInterFrameDelayMs * (kMinRequiredSamples - 1); + if (videocontenttypehelpers::IsScreenshare(content_type)) { + EXPECT_EQ( + kFreezeDelayMs + kInterFrameDelayMs, + metrics::MinSample("WebRTC.Video.Screenshare.MeanFreezeDurationMs")); + EXPECT_EQ(kExpectedTimeBetweenFreezes, + metrics::MinSample( + "WebRTC.Video.Screenshare.MeanTimeBetweenFreezesMs")); + } else { + EXPECT_EQ(kFreezeDelayMs + kInterFrameDelayMs, + metrics::MinSample("WebRTC.Video.MeanFreezeDurationMs")); + EXPECT_EQ(kExpectedTimeBetweenFreezes, + metrics::MinSample("WebRTC.Video.MeanTimeBetweenFreezesMs")); + } +} + +TEST_P(ReceiveStatisticsProxyTest, PausesAreIgnored) { + const VideoContentType content_type = GetParam(); + const int kInterFrameDelayMs = 33; + const int kPauseDurationMs = 10000; + for (int i = 0; i <= kMinRequiredSamples; ++i) { + statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight, + content_type); + fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); + } + // Add a pause. + fake_clock_.AdvanceTimeMilliseconds(kPauseDurationMs); + statistics_proxy_->OnStreamInactive(); + + // Second playback interval with triple the length. + for (int i = 0; i <= kMinRequiredSamples * 3; ++i) { + statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight, + content_type); + fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); + } + + statistics_proxy_.reset(); + // Average of two playback intervals. + const int kExpectedTimeBetweenFreezes = + kInterFrameDelayMs * kMinRequiredSamples * 2; + if (videocontenttypehelpers::IsScreenshare(content_type)) { + EXPECT_EQ(-1, metrics::MinSample( + "WebRTC.Video.Screenshare.MeanFreezeDurationMs")); + EXPECT_EQ(kExpectedTimeBetweenFreezes, + metrics::MinSample( + "WebRTC.Video.Screenshare.MeanTimeBetweenFreezesMs")); + } else { + EXPECT_EQ(-1, metrics::MinSample("WebRTC.Video.MeanFreezeDurationMs")); + EXPECT_EQ(kExpectedTimeBetweenFreezes, + metrics::MinSample("WebRTC.Video.MeanTimeBetweenFreezesMs")); + } +} + +TEST_P(ReceiveStatisticsProxyTest, TimeInHdReported) { + const VideoContentType content_type = GetParam(); + const int kInterFrameDelayMs = 20; + // HD frames. + for (int i = 0; i < kMinRequiredSamples; ++i) { + statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight, + content_type); + fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); + } + // SD frames. + for (int i = 0; i < 2 * kMinRequiredSamples; ++i) { + statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth / 2, kHeight / 2, + content_type); + fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); + } + // Extra last frame. + statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth / 2, kHeight / 2, + content_type); + statistics_proxy_.reset(); + const int kExpectedTimeInHdPercents = 33; + if (videocontenttypehelpers::IsScreenshare(content_type)) { + EXPECT_EQ( + kExpectedTimeInHdPercents, + metrics::MinSample("WebRTC.Video.Screenshare.TimeInHdPercentage")); + } else { + EXPECT_EQ(kExpectedTimeInHdPercents, + metrics::MinSample("WebRTC.Video.TimeInHdPercentage")); + } +} + +TEST_P(ReceiveStatisticsProxyTest, TimeInBlockyVideoReported) { + const VideoContentType content_type = GetParam(); + const int kInterFrameDelayMs = 20; + const int kHighQp = 80; + const int kLowQp = 30; + // High quality frames. + for (int i = 0; i < kMinRequiredSamples; ++i) { + statistics_proxy_->OnDecodedFrame(kLowQp, kWidth, kHeight, content_type); + fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); + } + // Blocky frames. + for (int i = 0; i < 2 * kMinRequiredSamples; ++i) { + statistics_proxy_->OnDecodedFrame(kHighQp, kWidth, kHeight, content_type); + fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); + } + // Extra last frame. + statistics_proxy_->OnDecodedFrame(kHighQp, kWidth, kHeight, content_type); + statistics_proxy_.reset(); + const int kExpectedTimeInHdPercents = 66; + if (videocontenttypehelpers::IsScreenshare(content_type)) { + EXPECT_EQ(kExpectedTimeInHdPercents, + metrics::MinSample( + "WebRTC.Video.Screenshare.TimeInBlockyVideoPercentage")); + } else { + EXPECT_EQ(kExpectedTimeInHdPercents, + metrics::MinSample("WebRTC.Video.TimeInBlockyVideoPercentage")); + } +} + +TEST_P(ReceiveStatisticsProxyTest, DownscalesReported) { + const VideoContentType content_type = GetParam(); + const int kInterFrameDelayMs = 1000; // To ensure long enough call duration. + const int kLowQp = 30; + + statistics_proxy_->OnDecodedFrame(kLowQp, kWidth / 2, kHeight / 2, + content_type); + fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); + + // Downscale. + statistics_proxy_->OnDecodedFrame(kLowQp, kWidth, kHeight, content_type); + fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); + + statistics_proxy_->OnDecodedFrame(kLowQp, kWidth / 2, kHeight / 2, + content_type); + fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); + + statistics_proxy_->OnDecodedFrame(kLowQp, kWidth / 2, kHeight / 2, + content_type); + fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); + + // Downscale. + statistics_proxy_->OnDecodedFrame(kLowQp, kWidth / 4, kHeight / 4, + content_type); + fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs); + + statistics_proxy_.reset(); + const int kExpectedDownscales = 30; // 2 per 5 seconds = 30 per minute. + if (videocontenttypehelpers::IsScreenshare(content_type)) { + EXPECT_EQ( + kExpectedDownscales, + metrics::MinSample( + "WebRTC.Video.Screenshare.NumberResolutionDownswitchesPerMinute")); + } else { + EXPECT_EQ(kExpectedDownscales, + metrics::MinSample( + "WebRTC.Video.NumberResolutionDownswitchesPerMinute")); + } +} + TEST_P(ReceiveStatisticsProxyTest, StatsAreSlicedOnSimulcastAndExperiment) { VideoContentType content_type = GetParam(); const uint8_t experiment_id = 1; @@ -960,13 +1155,15 @@ TEST_P(ReceiveStatisticsProxyTest, StatsAreSlicedOnSimulcastAndExperiment) { videocontenttypehelpers::SetSimulcastId(&content_type, 1); for (int i = 0; i <= kMinRequiredSamples; ++i) { fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs1); - statistics_proxy_->OnDecodedFrame(rtc::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight, + content_type); } videocontenttypehelpers::SetSimulcastId(&content_type, 2); for (int i = 0; i <= kMinRequiredSamples; ++i) { fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs2); - statistics_proxy_->OnDecodedFrame(rtc::nullopt, content_type); + statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight, + content_type); } statistics_proxy_.reset(); diff --git a/video/video_quality_observer.cc b/video/video_quality_observer.cc new file mode 100644 index 0000000000..bcadb642d3 --- /dev/null +++ b/video/video_quality_observer.cc @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/video_quality_observer.h" + +#include +#include + +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/strings/string_builder.h" +#include "system_wrappers/include/metrics.h" + +namespace webrtc { + +namespace { +const int kMinFrameSamplesToDetectFreeze = 5; +const int kMinCallDurationMs = 3000; +const int kMinRequiredSamples = 1; +const int kMinIncreaseForFreezeMs = 150; +const int kPixelsInHighResolution = 960 * 540; // CPU-adapted HD still counts. +const int kPixelsInMediumResolution = 640 * 360; +const int kBlockyQpThresholdVp8 = 70; +const int kBlockyQpThresholdVp9 = 60; // TODO(ilnik): tune this value. +// TODO(ilnik): Add H264/HEVC thresholds. +} // namespace + +VideoQualityObserver::VideoQualityObserver(VideoContentType content_type) + : last_frame_decoded_ms_(-1), + num_frames_decoded_(0), + first_frame_decoded_ms_(-1), + last_frame_pixels_(0), + last_frame_qp_(0), + last_unfreeze_time_(0), + time_in_resolution_ms_(3, 0), + current_resolution_(Resolution::Low), + num_resolution_downgrades_(0), + time_in_blocky_video_ms_(0), + content_type_(content_type), + is_paused_(false) {} + +VideoQualityObserver::~VideoQualityObserver() { + UpdateHistograms(); +} + +void VideoQualityObserver::UpdateHistograms() { + // Don't report anything on an empty video stream. + if (num_frames_decoded_ == 0) { + return; + } + + char log_stream_buf[2 * 1024]; + rtc::SimpleStringBuilder log_stream(log_stream_buf); + + if (last_frame_decoded_ms_ > last_unfreeze_time_) { + smooth_playback_durations_.Add(last_frame_decoded_ms_ - + last_unfreeze_time_); + } + + std::string uma_prefix = videocontenttypehelpers::IsScreenshare(content_type_) + ? "WebRTC.Video.Screenshare" + : "WebRTC.Video"; + + auto mean_time_between_freezes = + smooth_playback_durations_.Avg(kMinRequiredSamples); + if (mean_time_between_freezes) { + RTC_HISTOGRAM_COUNTS_SPARSE_100000(uma_prefix + ".MeanTimeBetweenFreezesMs", + *mean_time_between_freezes); + log_stream << uma_prefix << ".MeanTimeBetweenFreezesMs " + << *mean_time_between_freezes << "\n"; + } + auto avg_freeze_length = freezes_durations_.Avg(kMinRequiredSamples); + if (avg_freeze_length) { + RTC_HISTOGRAM_COUNTS_SPARSE_100000(uma_prefix + ".MeanFreezeDurationMs", + *avg_freeze_length); + log_stream << uma_prefix << ".MeanFreezeDurationMs " << *avg_freeze_length + << "\n"; + } + + int64_t call_duration_ms = last_frame_decoded_ms_ - first_frame_decoded_ms_; + + if (call_duration_ms >= kMinCallDurationMs) { + int time_spent_in_hd_percentage = static_cast( + time_in_resolution_ms_[Resolution::High] * 100 / call_duration_ms); + int time_with_blocky_video_percentage = + static_cast(time_in_blocky_video_ms_ * 100 / call_duration_ms); + + RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".TimeInHdPercentage", + time_spent_in_hd_percentage); + log_stream << uma_prefix << ".TimeInHdPercentage " + << time_spent_in_hd_percentage << "\n"; + RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".TimeInBlockyVideoPercentage", + time_with_blocky_video_percentage); + log_stream << uma_prefix << ".TimeInBlockyVideoPercentage " + << time_with_blocky_video_percentage << "\n"; + RTC_HISTOGRAM_COUNTS_SPARSE_100( + uma_prefix + ".NumberResolutionDownswitchesPerMinute", + num_resolution_downgrades_ * 60000 / call_duration_ms); + log_stream << uma_prefix << ".NumberResolutionDownswitchesPerMinute " + << num_resolution_downgrades_ * 60000 / call_duration_ms << "\n"; + } + RTC_LOG(LS_INFO) << log_stream.str(); +} + +void VideoQualityObserver::OnDecodedFrame(rtc::Optional qp, + int width, + int height, + int64_t now_ms, + VideoCodecType codec) { + if (num_frames_decoded_ == 0) { + first_frame_decoded_ms_ = now_ms; + last_unfreeze_time_ = now_ms; + } + + ++num_frames_decoded_; + + if (!is_paused_ && num_frames_decoded_ > 1) { + // Process inter-frame delay. + int64_t interframe_delay_ms = now_ms - last_frame_decoded_ms_; + interframe_delays_.Add(interframe_delay_ms); + rtc::Optional avg_interframe_delay = + interframe_delays_.Avg(kMinFrameSamplesToDetectFreeze); + // Check if it was a freeze. + if (num_frames_decoded_ > kMinFrameSamplesToDetectFreeze && + interframe_delay_ms >= + std::max(3 * *avg_interframe_delay, + *avg_interframe_delay + kMinIncreaseForFreezeMs)) { + freezes_durations_.Add(interframe_delay_ms); + smooth_playback_durations_.Add(last_frame_decoded_ms_ - + last_unfreeze_time_); + last_unfreeze_time_ = now_ms; + } else { + // Only count inter-frame delay as playback time if there + // was no freeze. + time_in_resolution_ms_[current_resolution_] += interframe_delay_ms; + rtc::Optional qp_blocky_threshold; + // TODO(ilnik): add other codec types when we have QP for them. + switch (codec) { + case kVideoCodecVP8: + qp_blocky_threshold = kBlockyQpThresholdVp8; + break; + case kVideoCodecVP9: + qp_blocky_threshold = kBlockyQpThresholdVp9; + break; + default: + qp_blocky_threshold = rtc::nullopt; + } + if (qp_blocky_threshold && qp.value_or(0) > *qp_blocky_threshold) { + time_in_blocky_video_ms_ += interframe_delay_ms; + } + } + } + + if (is_paused_) { + // If the stream was paused since the previous frame, do not count the + // pause toward smooth playback. Explicitly count the part before it and + // start the new smooth playback interval from this frame. + is_paused_ = false; + if (last_frame_decoded_ms_ > last_unfreeze_time_) { + smooth_playback_durations_.Add(last_frame_decoded_ms_ - + last_unfreeze_time_); + } + last_unfreeze_time_ = now_ms; + } + + int64_t pixels = width * height; + if (pixels >= kPixelsInHighResolution) { + current_resolution_ = Resolution::High; + } else if (pixels >= kPixelsInMediumResolution) { + current_resolution_ = Resolution::Medium; + } else { + current_resolution_ = Resolution::Low; + } + + if (pixels < last_frame_pixels_) { + ++num_resolution_downgrades_; + } + + last_frame_decoded_ms_ = now_ms; + last_frame_qp_ = qp.value_or(0); + last_frame_pixels_ = pixels; +} + +void VideoQualityObserver::OnStreamInactive() { + is_paused_ = true; +} +} // namespace webrtc diff --git a/video/video_quality_observer.h b/video/video_quality_observer.h new file mode 100644 index 0000000000..cbeab4339e --- /dev/null +++ b/video/video_quality_observer.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_VIDEO_QUALITY_OBSERVER_H_ +#define VIDEO_VIDEO_QUALITY_OBSERVER_H_ + +#include +#include + +#include "api/optional.h" +#include "api/video/video_content_type.h" +#include "common_types.h" // NOLINT(build/include) +#include "rtc_base/numerics/sample_counter.h" + +namespace webrtc { + +// Calculates spatial and temporal quality metrics and reports them to UMA +// stats. +class VideoQualityObserver { + public: + // Use either VideoQualityObserver::kBlockyQpThresholdVp8 or + // VideoQualityObserver::kBlockyQpThresholdVp9. + explicit VideoQualityObserver(VideoContentType content_type); + ~VideoQualityObserver(); + + void OnDecodedFrame(rtc::Optional qp, + int width, + int height, + int64_t now_ms, + VideoCodecType codec); + + void OnStreamInactive(); + + private: + void UpdateHistograms(); + + enum Resolution { + Low = 0, + Medium = 1, + High = 2, + }; + + int64_t last_frame_decoded_ms_; + int64_t num_frames_decoded_; + int64_t first_frame_decoded_ms_; + int64_t last_frame_pixels_; + uint8_t last_frame_qp_; + // Decoded timestamp of the last delayed frame. + int64_t last_unfreeze_time_; + rtc::SampleCounter interframe_delays_; + // An inter-frame delay is counted as a freeze if it's significantly longer + // than average inter-frame delay. + rtc::SampleCounter freezes_durations_; + // Time between freezes. + rtc::SampleCounter smooth_playback_durations_; + // Counters for time spent in different resolutions. Time between each two + // Consecutive frames is counted to bin corresponding to the first frame + // resolution. + std::vector time_in_resolution_ms_; + // Resolution of the last decoded frame. Resolution enum is used as an index. + Resolution current_resolution_; + int num_resolution_downgrades_; + // Similar to resolution, time spent in high-QP video. + int64_t time_in_blocky_video_ms_; + // Content type of the last decoded frame. + VideoContentType content_type_; + bool is_paused_; +}; + +} // namespace webrtc + +#endif // VIDEO_VIDEO_QUALITY_OBSERVER_H_ diff --git a/video/video_stream_decoder.cc b/video/video_stream_decoder.cc index e766767cd0..5557c6cbd6 100644 --- a/video/video_stream_decoder.cc +++ b/video/video_stream_decoder.cc @@ -77,7 +77,8 @@ VideoStreamDecoder::~VideoStreamDecoder() { int32_t VideoStreamDecoder::FrameToRender(VideoFrame& video_frame, rtc::Optional qp, VideoContentType content_type) { - receive_stats_callback_->OnDecodedFrame(qp, content_type); + receive_stats_callback_->OnDecodedFrame(qp, video_frame.width(), + video_frame.height(), content_type); incoming_video_stream_->OnFrame(video_frame); return 0; }