zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement VideoQualityObserver

Browse Source 
This class receives data about video frames from ReceiveStatisticsProxy,
calculates spatial and temporal quality metrics and outputs them to UMA
stats. It is all done in a separate class because it will be further
extended to calculate aggregated quality metrics in the future.

Bug: webrtc:9295
Change-Id: Ie36db83e10c0e8da0b9baa392651cb9a67a54a80
Reviewed-on: https://webrtc-review.googlesource.com/78220
Commit-Queue: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#23387}
This commit is contained in:
Ilya Nikolaevskiy
2018-05-23 11:53:19 +02:00
committed by Commit Bot
parent 95de63b6fc
commit 94150ee487
7 changed files with 550 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
video/BUILD.gn
View File

@ -38,6 +38,8 @@ rtc_static_library("video") {
"stream_synchronization.h",
"transport_adapter.cc",
"transport_adapter.h",
"video_quality_observer.cc",
"video_quality_observer.h",
"video_receive_stream.cc",
"video_receive_stream.h",
"video_send_stream.cc",

32
video/receive_statistics_proxy.cc
View File

@ -106,11 +106,14 @@ ReceiveStatisticsProxy::ReceiveStatisticsProxy(
render_fps_tracker_(100, 10u),
render_pixel_tracker_(100, 10u),
total_byte_tracker_(100, 10u), // bucket_interval_ms, bucket_count
video_quality_observer_(
new VideoQualityObserver(VideoContentType::UNSPECIFIED)),
interframe_delay_max_moving_(kMovingMaxWindowMs),
freq_offset_counter_(clock, nullptr, kFreqOffsetProcessIntervalMs),
first_report_block_time_ms_(-1),
avg_rtt_ms_(0),
last_content_type_(VideoContentType::UNSPECIFIED),
last_codec_type_(kVideoCodecVP8),
timing_frame_info_counter_(kMovingMaxWindowMs) {
decode_thread_.DetachFromThread();
network_thread_.DetachFromThread();
@ -147,13 +150,14 @@ void ReceiveStatisticsProxy::UpdateHistograms() {
<< stream_duration_sec << '\n';
}
log_stream << "Frames decoded " << stats_.frames_decoded;
log_stream << "Frames decoded " << stats_.frames_decoded << '\n';
if (num_unique_frames_) {
int num_dropped_frames = *num_unique_frames_ - stats_.frames_decoded;
RTC_HISTOGRAM_COUNTS_1000("WebRTC.Video.DroppedFrames.Receiver",
num_dropped_frames);
log_stream << "WebRTC.Video.DroppedFrames.Receiver " << num_dropped_frames;
log_stream << "WebRTC.Video.DroppedFrames.Receiver " << num_dropped_frames
<< '\n';
}
if (first_report_block_time_ms_ != -1 &&
@ -680,12 +684,30 @@ void ReceiveStatisticsProxy::DataCountersUpdated(
total_byte_tracker_.AddSamples(total_bytes - last_total_bytes);
}
// Deprecated. TODO(ilnik): remove once all depending projects are updated.
void ReceiveStatisticsProxy::OnDecodedFrame(rtc::Optional<uint8_t> qp,
VideoContentType content_type) {
OnDecodedFrame(qp, 0, 0, content_type);
}
void ReceiveStatisticsProxy::OnDecodedFrame(rtc::Optional<uint8_t> qp,
int width,
int height,
VideoContentType content_type) {
rtc::CritScope lock(&crit_);
uint64_t now = clock_->TimeInMilliseconds();
if (videocontenttypehelpers::IsScreenshare(content_type) !=
videocontenttypehelpers::IsScreenshare(last_content_type_)) {
// Reset the quality observer if content type is switched. This will
// report stats for the previous part of the call.
video_quality_observer_.reset(new VideoQualityObserver(content_type));
}
video_quality_observer_->OnDecodedFrame(qp, width, height, now,
last_codec_type_);
ContentSpecificStats* content_specific_stats =
&content_specific_stats_[content_type];
++stats_.frames_decoded;
@ -809,9 +831,10 @@ void ReceiveStatisticsProxy::OnPreDecode(
if (!codec_specific_info || encoded_image.qp_ == -1) {
return;
}
if (codec_specific_info->codecType == kVideoCodecVP8) {
rtc::CritScope lock(&crit_);
last_codec_type_ = codec_specific_info->codecType;
if (last_codec_type_ == kVideoCodecVP8) {
qp_counters_.vp8.Add(encoded_image.qp_);
rtc::CritScope lock(&crit_);
qp_sample_.Add(encoded_image.qp_);
}
}
@ -822,6 +845,7 @@ void ReceiveStatisticsProxy::OnStreamInactive() {
rtc::CritScope lock(&crit_);
// Don't report inter-frame delay if stream was paused.
last_decoded_frame_time_ms_.reset();
video_quality_observer_->OnStreamInactive();
}
void ReceiveStatisticsProxy::OnRttUpdate(int64_t avg_rtt_ms,

14
video/receive_statistics_proxy.h
View File

@ -12,6 +12,7 @@
#define VIDEO_RECEIVE_STATISTICS_PROXY_H_
#include <map>
#include <memory>
#include <string>
#include <vector>
@ -31,6 +32,7 @@
#include "video/quality_threshold.h"
#include "video/report_block_stats.h"
#include "video/stats_counter.h"
#include "video/video_quality_observer.h"
#include "video/video_stream_decoder.h"
namespace webrtc {
@ -50,7 +52,14 @@ class ReceiveStatisticsProxy : public VCMReceiveStatisticsCallback,
VideoReceiveStream::Stats GetStats() const;
void OnDecodedFrame(rtc::Optional<uint8_t> qp, VideoContentType content_type);
// Deprecated. TODO(ilnik): remove once all depending projects are updated.
RTC_DEPRECATED void OnDecodedFrame(rtc::Optional<uint8_t> qp,
VideoContentType content_type);
void OnDecodedFrame(rtc::Optional<uint8_t> qp,
int width,
int height,
VideoContentType content_type);
void OnSyncOffsetUpdated(int64_t sync_offset_ms, double estimated_freq_khz);
void OnRenderedFrame(const VideoFrame& frame);
void OnIncomingPayloadType(int payload_type);
@ -163,6 +172,8 @@ class ReceiveStatisticsProxy : public VCMReceiveStatisticsCallback,
rtc::SampleCounter target_delay_counter_ RTC_GUARDED_BY(crit_);
rtc::SampleCounter current_delay_counter_ RTC_GUARDED_BY(crit_);
rtc::SampleCounter delay_counter_ RTC_GUARDED_BY(crit_);
std::unique_ptr<VideoQualityObserver> video_quality_observer_
RTC_GUARDED_BY(crit_);
mutable rtc::MovingMaxCounter<int> interframe_delay_max_moving_
RTC_GUARDED_BY(crit_);
std::map<VideoContentType, ContentSpecificStats> content_specific_stats_
@ -175,6 +186,7 @@ class ReceiveStatisticsProxy : public VCMReceiveStatisticsCallback,
int64_t avg_rtt_ms_ RTC_GUARDED_BY(crit_);
mutable std::map<int64_t, size_t> frame_window_ RTC_GUARDED_BY(&crit_);
VideoContentType last_content_type_ RTC_GUARDED_BY(&crit_);
VideoCodecType last_codec_type_ RTC_GUARDED_BY(&crit_);
rtc::Optional<int64_t> first_decoded_frame_time_ms_ RTC_GUARDED_BY(&crit_);
rtc::Optional<int64_t> last_decoded_frame_time_ms_ RTC_GUARDED_BY(&crit_);
// Mutable because calling Max() on MovingMaxCounter is not const. Yet it is

267
video/receive_statistics_proxy_unittest.cc
View File

@ -28,6 +28,10 @@ const int64_t kFreqOffsetProcessIntervalInMs = 40000;
const uint32_t kLocalSsrc = 123;
const uint32_t kRemoteSsrc = 456;
const int kMinRequiredSamples = 200;
const int kWidth = 1280;
const int kHeight = 720;
} // namespace
// TODO(sakal): ReceiveStatisticsProxy is lacking unittesting.
@ -70,7 +74,7 @@ class ReceiveStatisticsProxyTest
TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameIncreasesFramesDecoded) {
EXPECT_EQ(0u, statistics_proxy_->GetStats().frames_decoded);
for (uint32_t i = 1; i <= 3; ++i) {
statistics_proxy_->OnDecodedFrame(rtc::nullopt,
statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight,
VideoContentType::UNSPECIFIED);
EXPECT_EQ(i, statistics_proxy_->GetStats().frames_decoded);
}
@ -80,7 +84,7 @@ TEST_F(ReceiveStatisticsProxyTest, DecodedFpsIsReported) {
const int kFps = 20;
const int kRequiredSamples = metrics::kMinRunTimeInSeconds * kFps;
for (int i = 0; i < kRequiredSamples; ++i) {
statistics_proxy_->OnDecodedFrame(rtc::Optional<uint8_t>(),
statistics_proxy_->OnDecodedFrame(rtc::Optional<uint8_t>(), kWidth, kHeight,
VideoContentType::UNSPECIFIED);
fake_clock_.AdvanceTimeMilliseconds(1000 / kFps);
}
@ -93,7 +97,7 @@ TEST_F(ReceiveStatisticsProxyTest, DecodedFpsIsNotReportedForTooFewSamples) {
const int kFps = 20;
const int kRequiredSamples = metrics::kMinRunTimeInSeconds * kFps;
for (int i = 0; i < kRequiredSamples - 1; ++i) {
statistics_proxy_->OnDecodedFrame(rtc::Optional<uint8_t>(),
statistics_proxy_->OnDecodedFrame(rtc::Optional<uint8_t>(), kWidth, kHeight,
VideoContentType::UNSPECIFIED);
fake_clock_.AdvanceTimeMilliseconds(1000 / kFps);
}
@ -104,15 +108,15 @@ TEST_F(ReceiveStatisticsProxyTest, DecodedFpsIsNotReportedForTooFewSamples) {
TEST_F(ReceiveStatisticsProxyTest, DecodedFpsIsReportedWithQpReset) {
const int kFps1 = 10;
for (int i = 0; i < metrics::kMinRunTimeInSeconds * kFps1; ++i) {
statistics_proxy_->OnDecodedFrame(rtc::Optional<uint8_t>(),
statistics_proxy_->OnDecodedFrame(rtc::Optional<uint8_t>(), kWidth, kHeight,
VideoContentType::UNSPECIFIED);
fake_clock_.AdvanceTimeMilliseconds(1000 / kFps1);
}
// First QP value received, resets frames decoded.
const int kFps2 = 20;
for (int i = 0; i < metrics::kMinRunTimeInSeconds * kFps2; ++i) {
statistics_proxy_->OnDecodedFrame(rtc::Optional<uint8_t>(1u),
VideoContentType::UNSPECIFIED);
statistics_proxy_->OnDecodedFrame(rtc::Optional<uint8_t>(1u), kWidth,
kHeight, VideoContentType::UNSPECIFIED);
fake_clock_.AdvanceTimeMilliseconds(1000 / kFps2);
}
statistics_proxy_.reset();
@ -124,19 +128,22 @@ TEST_F(ReceiveStatisticsProxyTest, DecodedFpsIsReportedWithQpReset) {
TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameWithQpResetsFramesDecoded) {
EXPECT_EQ(0u, statistics_proxy_->GetStats().frames_decoded);
for (uint32_t i = 1; i <= 3; ++i) {
statistics_proxy_->OnDecodedFrame(rtc::nullopt,
statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight,
VideoContentType::UNSPECIFIED);
EXPECT_EQ(i, statistics_proxy_->GetStats().frames_decoded);
}
statistics_proxy_->OnDecodedFrame(1u, VideoContentType::UNSPECIFIED);
statistics_proxy_->OnDecodedFrame(1u, kWidth, kHeight,
VideoContentType::UNSPECIFIED);
EXPECT_EQ(1u, statistics_proxy_->GetStats().frames_decoded);
}
TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameIncreasesQpSum) {
EXPECT_EQ(rtc::nullopt, statistics_proxy_->GetStats().qp_sum);
statistics_proxy_->OnDecodedFrame(3u, VideoContentType::UNSPECIFIED);
statistics_proxy_->OnDecodedFrame(3u, kWidth, kHeight,
VideoContentType::UNSPECIFIED);
EXPECT_EQ(3u, statistics_proxy_->GetStats().qp_sum);
statistics_proxy_->OnDecodedFrame(127u, VideoContentType::UNSPECIFIED);
statistics_proxy_->OnDecodedFrame(127u, kWidth, kHeight,
VideoContentType::UNSPECIFIED);
EXPECT_EQ(130u, statistics_proxy_->GetStats().qp_sum);
}
@ -145,10 +152,12 @@ TEST_F(ReceiveStatisticsProxyTest, ReportsContentType) {
const std::string kScreenshareString("screen");
EXPECT_EQ(kRealtimeString, videocontenttypehelpers::ToString(
statistics_proxy_->GetStats().content_type));
statistics_proxy_->OnDecodedFrame(3u, VideoContentType::SCREENSHARE);
statistics_proxy_->OnDecodedFrame(3u, kWidth, kHeight,
VideoContentType::SCREENSHARE);
EXPECT_EQ(kScreenshareString, videocontenttypehelpers::ToString(
statistics_proxy_->GetStats().content_type));
statistics_proxy_->OnDecodedFrame(3u, VideoContentType::UNSPECIFIED);
statistics_proxy_->OnDecodedFrame(3u, kWidth, kHeight,
VideoContentType::UNSPECIFIED);
EXPECT_EQ(kRealtimeString, videocontenttypehelpers::ToString(
statistics_proxy_->GetStats().content_type));
}
@ -158,21 +167,25 @@ TEST_F(ReceiveStatisticsProxyTest, ReportsMaxInterframeDelay) {
const int64_t kInterframeDelayMs2 = 200;
const int64_t kInterframeDelayMs3 = 100;
EXPECT_EQ(-1, statistics_proxy_->GetStats().interframe_delay_max_ms);
statistics_proxy_->OnDecodedFrame(3u, VideoContentType::UNSPECIFIED);
statistics_proxy_->OnDecodedFrame(3u, kWidth, kHeight,
VideoContentType::UNSPECIFIED);
EXPECT_EQ(-1, statistics_proxy_->GetStats().interframe_delay_max_ms);
fake_clock_.AdvanceTimeMilliseconds(kInterframeDelayMs1);
statistics_proxy_->OnDecodedFrame(127u, VideoContentType::UNSPECIFIED);
statistics_proxy_->OnDecodedFrame(127u, kWidth, kHeight,
VideoContentType::UNSPECIFIED);
EXPECT_EQ(kInterframeDelayMs1,
statistics_proxy_->GetStats().interframe_delay_max_ms);
fake_clock_.AdvanceTimeMilliseconds(kInterframeDelayMs2);
statistics_proxy_->OnDecodedFrame(127u, VideoContentType::UNSPECIFIED);
statistics_proxy_->OnDecodedFrame(127u, kWidth, kHeight,
VideoContentType::UNSPECIFIED);
EXPECT_EQ(kInterframeDelayMs2,
statistics_proxy_->GetStats().interframe_delay_max_ms);
fake_clock_.AdvanceTimeMilliseconds(kInterframeDelayMs3);
statistics_proxy_->OnDecodedFrame(127u, VideoContentType::UNSPECIFIED);
statistics_proxy_->OnDecodedFrame(127u, kWidth, kHeight,
VideoContentType::UNSPECIFIED);
// kInterframeDelayMs3 is smaller than kInterframeDelayMs2.
EXPECT_EQ(kInterframeDelayMs2,
statistics_proxy_->GetStats().interframe_delay_max_ms);
@ -183,22 +196,26 @@ TEST_F(ReceiveStatisticsProxyTest, ReportInterframeDelayInWindow) {
const int64_t kInterframeDelayMs2 = 750;
const int64_t kInterframeDelayMs3 = 700;
EXPECT_EQ(-1, statistics_proxy_->GetStats().interframe_delay_max_ms);
statistics_proxy_->OnDecodedFrame(3u, VideoContentType::UNSPECIFIED);
statistics_proxy_->OnDecodedFrame(3u, kWidth, kHeight,
VideoContentType::UNSPECIFIED);
EXPECT_EQ(-1, statistics_proxy_->GetStats().interframe_delay_max_ms);
fake_clock_.AdvanceTimeMilliseconds(kInterframeDelayMs1);
statistics_proxy_->OnDecodedFrame(127u, VideoContentType::UNSPECIFIED);
statistics_proxy_->OnDecodedFrame(127u, kWidth, kHeight,
VideoContentType::UNSPECIFIED);
EXPECT_EQ(kInterframeDelayMs1,
statistics_proxy_->GetStats().interframe_delay_max_ms);
fake_clock_.AdvanceTimeMilliseconds(kInterframeDelayMs2);
statistics_proxy_->OnDecodedFrame(127u, VideoContentType::UNSPECIFIED);
statistics_proxy_->OnDecodedFrame(127u, kWidth, kHeight,
VideoContentType::UNSPECIFIED);
// Still first delay is the maximum
EXPECT_EQ(kInterframeDelayMs1,
statistics_proxy_->GetStats().interframe_delay_max_ms);
fake_clock_.AdvanceTimeMilliseconds(kInterframeDelayMs3);
statistics_proxy_->OnDecodedFrame(127u, VideoContentType::UNSPECIFIED);
statistics_proxy_->OnDecodedFrame(127u, kWidth, kHeight,
VideoContentType::UNSPECIFIED);
// Now the first sample is out of the window, so the second is the maximum.
EXPECT_EQ(kInterframeDelayMs2,
statistics_proxy_->GetStats().interframe_delay_max_ms);
@ -206,16 +223,17 @@ TEST_F(ReceiveStatisticsProxyTest, ReportInterframeDelayInWindow) {
TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameWithoutQpQpSumWontExist) {
EXPECT_EQ(rtc::nullopt, statistics_proxy_->GetStats().qp_sum);
statistics_proxy_->OnDecodedFrame(rtc::nullopt,
statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight,
VideoContentType::UNSPECIFIED);
EXPECT_EQ(rtc::nullopt, statistics_proxy_->GetStats().qp_sum);
}
TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameWithoutQpResetsQpSum) {
EXPECT_EQ(rtc::nullopt, statistics_proxy_->GetStats().qp_sum);
statistics_proxy_->OnDecodedFrame(3u, VideoContentType::UNSPECIFIED);
statistics_proxy_->OnDecodedFrame(3u, kWidth, kHeight,
VideoContentType::UNSPECIFIED);
EXPECT_EQ(3u, statistics_proxy_->GetStats().qp_sum);
statistics_proxy_->OnDecodedFrame(rtc::nullopt,
statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight,
VideoContentType::UNSPECIFIED);
EXPECT_EQ(rtc::nullopt, statistics_proxy_->GetStats().qp_sum);
}
@ -718,7 +736,7 @@ TEST_F(ReceiveStatisticsProxyTest, DoesNotReportStaleFramerates) {
// Since OnRenderedFrame is never called the fps in each sample will be 0,
// i.e. bad
frame.set_ntp_time_ms(fake_clock_.CurrentNtpInMilliseconds());
statistics_proxy_->OnDecodedFrame(rtc::nullopt,
statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight,
VideoContentType::UNSPECIFIED);
statistics_proxy_->OnRenderedFrame(frame);
fake_clock_.AdvanceTimeMilliseconds(1000 / kDefaultFps);
@ -833,12 +851,14 @@ TEST_P(ReceiveStatisticsProxyTest, InterFrameDelaysAreReported) {
const VideoContentType content_type = GetParam();
const int kInterFrameDelayMs = 33;
for (int i = 0; i < kMinRequiredSamples; ++i) {
statistics_proxy_->OnDecodedFrame(rtc::nullopt, content_type);
statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight,
content_type);
fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs);
}
// One extra with double the interval.
fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs);
statistics_proxy_->OnDecodedFrame(rtc::nullopt, content_type);
statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight,
content_type);
statistics_proxy_.reset();
const int kExpectedInterFrame =
@ -866,16 +886,19 @@ TEST_P(ReceiveStatisticsProxyTest, InterFrameDelaysPercentilesAreReported) {
const int kLastFivePercentsSamples = kMinRequiredSamples * 5 / 100;
for (int i = 0; i <= kMinRequiredSamples - kLastFivePercentsSamples; ++i) {
fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs);
statistics_proxy_->OnDecodedFrame(rtc::nullopt, content_type);
statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight,
content_type);
}
// Last 5% of intervals are double in size.
for (int i = 0; i < kLastFivePercentsSamples; ++i) {
fake_clock_.AdvanceTimeMilliseconds(2 * kInterFrameDelayMs);
statistics_proxy_->OnDecodedFrame(rtc::nullopt, content_type);
statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight,
content_type);
}
// Final sample is outlier and 10 times as big.
fake_clock_.AdvanceTimeMilliseconds(10 * kInterFrameDelayMs);
statistics_proxy_->OnDecodedFrame(rtc::nullopt, content_type);
statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight,
content_type);
statistics_proxy_.reset();
const int kExpectedInterFrame = kInterFrameDelayMs * 2;
@ -894,7 +917,8 @@ TEST_P(ReceiveStatisticsProxyTest, MaxInterFrameDelayOnlyWithValidAverage) {
const VideoContentType content_type = GetParam();
const int kInterFrameDelayMs = 33;
for (int i = 0; i < kMinRequiredSamples; ++i) {
statistics_proxy_->OnDecodedFrame(rtc::nullopt, content_type);
statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight,
content_type);
fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs);
}
@ -913,7 +937,8 @@ TEST_P(ReceiveStatisticsProxyTest, MaxInterFrameDelayOnlyWithPause) {
const VideoContentType content_type = GetParam();
const int kInterFrameDelayMs = 33;
for (int i = 0; i <= kMinRequiredSamples; ++i) {
statistics_proxy_->OnDecodedFrame(rtc::nullopt, content_type);
statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight,
content_type);
fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs);
}
@ -924,9 +949,11 @@ TEST_P(ReceiveStatisticsProxyTest, MaxInterFrameDelayOnlyWithPause) {
// Insert two more frames. The interval during the pause should be disregarded
// in the stats.
statistics_proxy_->OnDecodedFrame(rtc::nullopt, content_type);
statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight,
content_type);
fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs);
statistics_proxy_->OnDecodedFrame(rtc::nullopt, content_type);
statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight,
content_type);
statistics_proxy_.reset();
if (videocontenttypehelpers::IsScreenshare(content_type)) {
@ -950,6 +977,174 @@ TEST_P(ReceiveStatisticsProxyTest, MaxInterFrameDelayOnlyWithPause) {
}
}
TEST_P(ReceiveStatisticsProxyTest, FreezesAreReported) {
const VideoContentType content_type = GetParam();
const int kInterFrameDelayMs = 33;
const int kFreezeDelayMs = 200;
for (int i = 0; i < kMinRequiredSamples; ++i) {
statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight,
content_type);
fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs);
}
// Add extra freeze.
fake_clock_.AdvanceTimeMilliseconds(kFreezeDelayMs);
statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight,
content_type);
statistics_proxy_.reset();
const int kExpectedTimeBetweenFreezes =
kInterFrameDelayMs * (kMinRequiredSamples - 1);
if (videocontenttypehelpers::IsScreenshare(content_type)) {
EXPECT_EQ(
kFreezeDelayMs + kInterFrameDelayMs,
metrics::MinSample("WebRTC.Video.Screenshare.MeanFreezeDurationMs"));
EXPECT_EQ(kExpectedTimeBetweenFreezes,
metrics::MinSample(
"WebRTC.Video.Screenshare.MeanTimeBetweenFreezesMs"));
} else {
EXPECT_EQ(kFreezeDelayMs + kInterFrameDelayMs,
metrics::MinSample("WebRTC.Video.MeanFreezeDurationMs"));
EXPECT_EQ(kExpectedTimeBetweenFreezes,
metrics::MinSample("WebRTC.Video.MeanTimeBetweenFreezesMs"));
}
}
TEST_P(ReceiveStatisticsProxyTest, PausesAreIgnored) {
const VideoContentType content_type = GetParam();
const int kInterFrameDelayMs = 33;
const int kPauseDurationMs = 10000;
for (int i = 0; i <= kMinRequiredSamples; ++i) {
statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight,
content_type);
fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs);
}
// Add a pause.
fake_clock_.AdvanceTimeMilliseconds(kPauseDurationMs);
statistics_proxy_->OnStreamInactive();
// Second playback interval with triple the length.
for (int i = 0; i <= kMinRequiredSamples * 3; ++i) {
statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight,
content_type);
fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs);
}
statistics_proxy_.reset();
// Average of two playback intervals.
const int kExpectedTimeBetweenFreezes =
kInterFrameDelayMs * kMinRequiredSamples * 2;
if (videocontenttypehelpers::IsScreenshare(content_type)) {
EXPECT_EQ(-1, metrics::MinSample(
"WebRTC.Video.Screenshare.MeanFreezeDurationMs"));
EXPECT_EQ(kExpectedTimeBetweenFreezes,
metrics::MinSample(
"WebRTC.Video.Screenshare.MeanTimeBetweenFreezesMs"));
} else {
EXPECT_EQ(-1, metrics::MinSample("WebRTC.Video.MeanFreezeDurationMs"));
EXPECT_EQ(kExpectedTimeBetweenFreezes,
metrics::MinSample("WebRTC.Video.MeanTimeBetweenFreezesMs"));
}
}
TEST_P(ReceiveStatisticsProxyTest, TimeInHdReported) {
const VideoContentType content_type = GetParam();
const int kInterFrameDelayMs = 20;
// HD frames.
for (int i = 0; i < kMinRequiredSamples; ++i) {
statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight,
content_type);
fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs);
}
// SD frames.
for (int i = 0; i < 2 * kMinRequiredSamples; ++i) {
statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth / 2, kHeight / 2,
content_type);
fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs);
}
// Extra last frame.
statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth / 2, kHeight / 2,
content_type);
statistics_proxy_.reset();
const int kExpectedTimeInHdPercents = 33;
if (videocontenttypehelpers::IsScreenshare(content_type)) {
EXPECT_EQ(
kExpectedTimeInHdPercents,
metrics::MinSample("WebRTC.Video.Screenshare.TimeInHdPercentage"));
} else {
EXPECT_EQ(kExpectedTimeInHdPercents,
metrics::MinSample("WebRTC.Video.TimeInHdPercentage"));
}
}
TEST_P(ReceiveStatisticsProxyTest, TimeInBlockyVideoReported) {
const VideoContentType content_type = GetParam();
const int kInterFrameDelayMs = 20;
const int kHighQp = 80;
const int kLowQp = 30;
// High quality frames.
for (int i = 0; i < kMinRequiredSamples; ++i) {
statistics_proxy_->OnDecodedFrame(kLowQp, kWidth, kHeight, content_type);
fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs);
}
// Blocky frames.
for (int i = 0; i < 2 * kMinRequiredSamples; ++i) {
statistics_proxy_->OnDecodedFrame(kHighQp, kWidth, kHeight, content_type);
fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs);
}
// Extra last frame.
statistics_proxy_->OnDecodedFrame(kHighQp, kWidth, kHeight, content_type);
statistics_proxy_.reset();
const int kExpectedTimeInHdPercents = 66;
if (videocontenttypehelpers::IsScreenshare(content_type)) {
EXPECT_EQ(kExpectedTimeInHdPercents,
metrics::MinSample(
"WebRTC.Video.Screenshare.TimeInBlockyVideoPercentage"));
} else {
EXPECT_EQ(kExpectedTimeInHdPercents,
metrics::MinSample("WebRTC.Video.TimeInBlockyVideoPercentage"));
}
}
TEST_P(ReceiveStatisticsProxyTest, DownscalesReported) {
const VideoContentType content_type = GetParam();
const int kInterFrameDelayMs = 1000; // To ensure long enough call duration.
const int kLowQp = 30;
statistics_proxy_->OnDecodedFrame(kLowQp, kWidth / 2, kHeight / 2,
content_type);
fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs);
// Downscale.
statistics_proxy_->OnDecodedFrame(kLowQp, kWidth, kHeight, content_type);
fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs);
statistics_proxy_->OnDecodedFrame(kLowQp, kWidth / 2, kHeight / 2,
content_type);
fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs);
statistics_proxy_->OnDecodedFrame(kLowQp, kWidth / 2, kHeight / 2,
content_type);
fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs);
// Downscale.
statistics_proxy_->OnDecodedFrame(kLowQp, kWidth / 4, kHeight / 4,
content_type);
fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs);
statistics_proxy_.reset();
const int kExpectedDownscales = 30; // 2 per 5 seconds = 30 per minute.
if (videocontenttypehelpers::IsScreenshare(content_type)) {
EXPECT_EQ(
kExpectedDownscales,
metrics::MinSample(
"WebRTC.Video.Screenshare.NumberResolutionDownswitchesPerMinute"));
} else {
EXPECT_EQ(kExpectedDownscales,
metrics::MinSample(
"WebRTC.Video.NumberResolutionDownswitchesPerMinute"));
}
}
TEST_P(ReceiveStatisticsProxyTest, StatsAreSlicedOnSimulcastAndExperiment) {
VideoContentType content_type = GetParam();
const uint8_t experiment_id = 1;
@ -960,13 +1155,15 @@ TEST_P(ReceiveStatisticsProxyTest, StatsAreSlicedOnSimulcastAndExperiment) {
videocontenttypehelpers::SetSimulcastId(&content_type, 1);
for (int i = 0; i <= kMinRequiredSamples; ++i) {
fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs1);
statistics_proxy_->OnDecodedFrame(rtc::nullopt, content_type);
statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight,
content_type);
}
videocontenttypehelpers::SetSimulcastId(&content_type, 2);
for (int i = 0; i <= kMinRequiredSamples; ++i) {
fake_clock_.AdvanceTimeMilliseconds(kInterFrameDelayMs2);
statistics_proxy_->OnDecodedFrame(rtc::nullopt, content_type);
statistics_proxy_->OnDecodedFrame(rtc::nullopt, kWidth, kHeight,
content_type);
}
statistics_proxy_.reset();

194
video/video_quality_observer.cc Normal file
View File

@ -0,0 +1,194 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "video/video_quality_observer.h"
#include <algorithm>
#include <string>
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/strings/string_builder.h"
#include "system_wrappers/include/metrics.h"
namespace webrtc {
namespace {
const int kMinFrameSamplesToDetectFreeze = 5;
const int kMinCallDurationMs = 3000;
const int kMinRequiredSamples = 1;
const int kMinIncreaseForFreezeMs = 150;
const int kPixelsInHighResolution = 960 * 540; // CPU-adapted HD still counts.
const int kPixelsInMediumResolution = 640 * 360;
const int kBlockyQpThresholdVp8 = 70;
const int kBlockyQpThresholdVp9 = 60; // TODO(ilnik): tune this value.
// TODO(ilnik): Add H264/HEVC thresholds.
} // namespace
VideoQualityObserver::VideoQualityObserver(VideoContentType content_type)
: last_frame_decoded_ms_(-1),
num_frames_decoded_(0),
first_frame_decoded_ms_(-1),
last_frame_pixels_(0),
last_frame_qp_(0),
last_unfreeze_time_(0),
time_in_resolution_ms_(3, 0),
current_resolution_(Resolution::Low),
num_resolution_downgrades_(0),
time_in_blocky_video_ms_(0),
content_type_(content_type),
is_paused_(false) {}
VideoQualityObserver::~VideoQualityObserver() {
UpdateHistograms();
}
void VideoQualityObserver::UpdateHistograms() {
// Don't report anything on an empty video stream.
if (num_frames_decoded_ == 0) {
return;
}
char log_stream_buf[2 * 1024];
rtc::SimpleStringBuilder log_stream(log_stream_buf);
if (last_frame_decoded_ms_ > last_unfreeze_time_) {
smooth_playback_durations_.Add(last_frame_decoded_ms_ -
last_unfreeze_time_);
}
std::string uma_prefix = videocontenttypehelpers::IsScreenshare(content_type_)
? "WebRTC.Video.Screenshare"
: "WebRTC.Video";
auto mean_time_between_freezes =
smooth_playback_durations_.Avg(kMinRequiredSamples);
if (mean_time_between_freezes) {
RTC_HISTOGRAM_COUNTS_SPARSE_100000(uma_prefix + ".MeanTimeBetweenFreezesMs",
*mean_time_between_freezes);
log_stream << uma_prefix << ".MeanTimeBetweenFreezesMs "
<< *mean_time_between_freezes << "\n";
}
auto avg_freeze_length = freezes_durations_.Avg(kMinRequiredSamples);
if (avg_freeze_length) {
RTC_HISTOGRAM_COUNTS_SPARSE_100000(uma_prefix + ".MeanFreezeDurationMs",
*avg_freeze_length);
log_stream << uma_prefix << ".MeanFreezeDurationMs " << *avg_freeze_length
<< "\n";
}
int64_t call_duration_ms = last_frame_decoded_ms_ - first_frame_decoded_ms_;
if (call_duration_ms >= kMinCallDurationMs) {
int time_spent_in_hd_percentage = static_cast<int>(
time_in_resolution_ms_[Resolution::High] * 100 / call_duration_ms);
int time_with_blocky_video_percentage =
static_cast<int>(time_in_blocky_video_ms_ * 100 / call_duration_ms);
RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".TimeInHdPercentage",
time_spent_in_hd_percentage);
log_stream << uma_prefix << ".TimeInHdPercentage "
<< time_spent_in_hd_percentage << "\n";
RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".TimeInBlockyVideoPercentage",
time_with_blocky_video_percentage);
log_stream << uma_prefix << ".TimeInBlockyVideoPercentage "
<< time_with_blocky_video_percentage << "\n";
RTC_HISTOGRAM_COUNTS_SPARSE_100(
uma_prefix + ".NumberResolutionDownswitchesPerMinute",
num_resolution_downgrades_ * 60000 / call_duration_ms);
log_stream << uma_prefix << ".NumberResolutionDownswitchesPerMinute "
<< num_resolution_downgrades_ * 60000 / call_duration_ms << "\n";
}
RTC_LOG(LS_INFO) << log_stream.str();
}
void VideoQualityObserver::OnDecodedFrame(rtc::Optional<uint8_t> qp,
int width,
int height,
int64_t now_ms,
VideoCodecType codec) {
if (num_frames_decoded_ == 0) {
first_frame_decoded_ms_ = now_ms;
last_unfreeze_time_ = now_ms;
}
++num_frames_decoded_;
if (!is_paused_ && num_frames_decoded_ > 1) {
// Process inter-frame delay.
int64_t interframe_delay_ms = now_ms - last_frame_decoded_ms_;
interframe_delays_.Add(interframe_delay_ms);
rtc::Optional<int> avg_interframe_delay =
interframe_delays_.Avg(kMinFrameSamplesToDetectFreeze);
// Check if it was a freeze.
if (num_frames_decoded_ > kMinFrameSamplesToDetectFreeze &&
interframe_delay_ms >=
std::max(3 * *avg_interframe_delay,
*avg_interframe_delay + kMinIncreaseForFreezeMs)) {
freezes_durations_.Add(interframe_delay_ms);
smooth_playback_durations_.Add(last_frame_decoded_ms_ -
last_unfreeze_time_);
last_unfreeze_time_ = now_ms;
} else {
// Only count inter-frame delay as playback time if there
// was no freeze.
time_in_resolution_ms_[current_resolution_] += interframe_delay_ms;
rtc::Optional<int> qp_blocky_threshold;
// TODO(ilnik): add other codec types when we have QP for them.
switch (codec) {
case kVideoCodecVP8:
qp_blocky_threshold = kBlockyQpThresholdVp8;
break;
case kVideoCodecVP9:
qp_blocky_threshold = kBlockyQpThresholdVp9;
break;
default:
qp_blocky_threshold = rtc::nullopt;
}
if (qp_blocky_threshold && qp.value_or(0) > *qp_blocky_threshold) {
time_in_blocky_video_ms_ += interframe_delay_ms;
}
}
}
if (is_paused_) {
// If the stream was paused since the previous frame, do not count the
// pause toward smooth playback. Explicitly count the part before it and
// start the new smooth playback interval from this frame.
is_paused_ = false;
if (last_frame_decoded_ms_ > last_unfreeze_time_) {
smooth_playback_durations_.Add(last_frame_decoded_ms_ -
last_unfreeze_time_);
}
last_unfreeze_time_ = now_ms;
}
int64_t pixels = width * height;
if (pixels >= kPixelsInHighResolution) {
current_resolution_ = Resolution::High;
} else if (pixels >= kPixelsInMediumResolution) {
current_resolution_ = Resolution::Medium;
} else {
current_resolution_ = Resolution::Low;
}
if (pixels < last_frame_pixels_) {
++num_resolution_downgrades_;
}
last_frame_decoded_ms_ = now_ms;
last_frame_qp_ = qp.value_or(0);
last_frame_pixels_ = pixels;
}
void VideoQualityObserver::OnStreamInactive() {
is_paused_ = true;
}
} // namespace webrtc

79
video/video_quality_observer.h Normal file
View File

@ -0,0 +1,79 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef VIDEO_VIDEO_QUALITY_OBSERVER_H_
#define VIDEO_VIDEO_QUALITY_OBSERVER_H_
#include <stdint.h>
#include <vector>
#include "api/optional.h"
#include "api/video/video_content_type.h"
#include "common_types.h" // NOLINT(build/include)
#include "rtc_base/numerics/sample_counter.h"
namespace webrtc {
// Calculates spatial and temporal quality metrics and reports them to UMA
// stats.
class VideoQualityObserver {
public:
// Use either VideoQualityObserver::kBlockyQpThresholdVp8 or
// VideoQualityObserver::kBlockyQpThresholdVp9.
explicit VideoQualityObserver(VideoContentType content_type);
~VideoQualityObserver();
void OnDecodedFrame(rtc::Optional<uint8_t> qp,
int width,
int height,
int64_t now_ms,
VideoCodecType codec);
void OnStreamInactive();
private:
void UpdateHistograms();
enum Resolution {
Low = 0,
Medium = 1,
High = 2,
};
int64_t last_frame_decoded_ms_;
int64_t num_frames_decoded_;
int64_t first_frame_decoded_ms_;
int64_t last_frame_pixels_;
uint8_t last_frame_qp_;
// Decoded timestamp of the last delayed frame.
int64_t last_unfreeze_time_;
rtc::SampleCounter interframe_delays_;
// An inter-frame delay is counted as a freeze if it's significantly longer
// than average inter-frame delay.
rtc::SampleCounter freezes_durations_;
// Time between freezes.
rtc::SampleCounter smooth_playback_durations_;
// Counters for time spent in different resolutions. Time between each two
// Consecutive frames is counted to bin corresponding to the first frame
// resolution.
std::vector<int64_t> time_in_resolution_ms_;
// Resolution of the last decoded frame. Resolution enum is used as an index.
Resolution current_resolution_;
int num_resolution_downgrades_;
// Similar to resolution, time spent in high-QP video.
int64_t time_in_blocky_video_ms_;
// Content type of the last decoded frame.
VideoContentType content_type_;
bool is_paused_;
};
} // namespace webrtc
#endif // VIDEO_VIDEO_QUALITY_OBSERVER_H_

3
video/video_stream_decoder.cc
View File

@ -77,7 +77,8 @@ VideoStreamDecoder::~VideoStreamDecoder() {
int32_t VideoStreamDecoder::FrameToRender(VideoFrame& video_frame,
rtc::Optional<uint8_t> qp,
VideoContentType content_type) {
receive_stats_callback_->OnDecodedFrame(qp, content_type);
receive_stats_callback_->OnDecodedFrame(qp, video_frame.width(),
video_frame.height(), content_type);
incoming_video_stream_->OnFrame(video_frame);
return 0;
}