zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Extract frames comparator out from DVQA

Browse Source 
Bug: b/196229820
Change-Id: Iaea04feadf0ed9cd734dd31e7ccca915fb7c585a
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/228645
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#34776}
This commit is contained in:
Artem Titov
2021-08-16 23:19:43 +02:00
committed by WebRTC LUCI CQ
parent be9281b92b
commit 82c3a6f3a7
8 changed files with 801 additions and 349 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
test/pc/e2e/BUILD.gn
View File

@ -34,6 +34,7 @@ if (!build_with_chromium) {
testonly = true
deps = [
":default_video_quality_analyzer_frames_comparator_test",
":default_video_quality_analyzer_test",
":multi_head_queue_test",
":peer_connection_e2e_smoke_test",
@ -517,6 +518,19 @@ if (!build_with_chromium) {
]
}
rtc_library("default_video_quality_analyzer_frames_comparator_test") {
testonly = true
sources = [ "analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc" ]
deps = [
":default_video_quality_analyzer_internal",
":default_video_quality_analyzer_shared",
"../..:test_support",
"../../../api:create_frame_generator",
"../../../api/units:timestamp",
"../../../system_wrappers",
]
}
rtc_library("multi_head_queue_test") {
testonly = true
sources = [ "analyzer/video/multi_head_queue_test.cc" ]
@ -651,25 +665,38 @@ if (!build_with_chromium) {
# This target contains implementation details of DefaultVideoQualityAnalyzer,
# so headers exported by it shouldn't be used in other places.
rtc_library("default_video_quality_analyzer_internal") {
visibility = [ ":default_video_quality_analyzer" ]
visibility = [
":default_video_quality_analyzer",
":default_video_quality_analyzer_frames_comparator_test",
]
testonly = true
sources = [
"analyzer/video/default_video_quality_analyzer_cpu_measurer.cc",
"analyzer/video/default_video_quality_analyzer_cpu_measurer.h",
"analyzer/video/default_video_quality_analyzer_frames_comparator.cc",
"analyzer/video/default_video_quality_analyzer_frames_comparator.h",
"analyzer/video/default_video_quality_analyzer_internal_shared_objects.cc",
"analyzer/video/default_video_quality_analyzer_internal_shared_objects.h",
]
deps = [
":default_video_quality_analyzer_shared",
"../../../api:array_view",
"../../../api:scoped_refptr",
"../../../api/numerics:numerics",
"../../../api/units:timestamp",
"../../../api/video:video_frame",
"../../../common_video",
"../../../rtc_base:checks",
"../../../rtc_base:rtc_base_approved",
"../../../rtc_base:rtc_base_tests_utils",
"../../../rtc_base:rtc_event",
"../../../rtc_base:stringutils",
"../../../rtc_base:timeutils",
"../../../rtc_base/synchronization:mutex",
"../../../rtc_tools:video_quality_analysis",
"../../../system_wrappers:system_wrappers",
]
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
}

355
test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
View File

@ -24,6 +24,7 @@
#include "rtc_base/strings/string_builder.h"
#include "rtc_base/time_utils.h"
#include "rtc_tools/frame_analyzer/video_geometry_aligner.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
@ -31,8 +32,6 @@ namespace webrtc {
namespace webrtc_pc_e2e {
namespace {
constexpr int kMaxActiveComparisons = 10;
constexpr int kFreezeThresholdMs = 150;
constexpr int kMicrosPerSecond = 1000000;
constexpr int kBitsInByte = 8;
@ -108,7 +107,9 @@ SamplesStatsCounter::StatsSample StatsSample(double value,
DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer(
webrtc::Clock* clock,
DefaultVideoQualityAnalyzerOptions options)
: options_(options), clock_(clock) {}
: options_(options),
clock_(clock),
frames_comparator_(clock, cpu_measurer_, options) {}
DefaultVideoQualityAnalyzer::~DefaultVideoQualityAnalyzer() {
Stop();
}
@ -118,20 +119,15 @@ void DefaultVideoQualityAnalyzer::Start(
rtc::ArrayView<const std::string> peer_names,
int max_threads_count) {
test_label_ = std::move(test_case_name);
for (int i = 0; i < max_threads_count; i++) {
thread_pool_.push_back(rtc::PlatformThread::SpawnJoinable(
[this] { ProcessComparisons(); },
"DefaultVideoQualityAnalyzerWorker-" + std::to_string(i)));
}
frames_comparator_.Start(max_threads_count);
{
MutexLock lock(&lock_);
MutexLock lock(&mutex_);
peers_ = std::make_unique<NamesCollection>(peer_names);
RTC_CHECK(start_time_.IsMinusInfinity());
state_ = State::kActive;
start_time_ = Now();
}
cpu_measurer_.StartMeasuringCpuProcessTime();
}
uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured(
@ -146,40 +142,20 @@ uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured(
size_t peers_count = -1;
size_t stream_index;
{
MutexLock lock(&lock_);
MutexLock lock(&mutex_);
// Create a local copy of `start_time_`, peer's index and total peers count
// to access it under `comparison_lock_` without holding a `lock_`
// to access it without holding a `mutex_` during access to
// `frames_comparator_`.
start_time = start_time_;
peer_index = peers_->index(peer_name);
peers_count = peers_->size();
stream_index = streams_.AddIfAbsent(stream_label);
}
// Ensure stats for this stream exists.
frames_comparator_.EnsureStatsForStream(stream_index, peer_index, peers_count,
captured_time, start_time);
{
// Ensure stats for this stream exists.
MutexLock lock(&comparison_lock_);
for (size_t i = 0; i < peers_count; ++i) {
if (i == peer_index && !options_.enable_receive_own_stream) {
continue;
}
InternalStatsKey stats_key(stream_index, peer_index, i);
if (stream_stats_.find(stats_key) == stream_stats_.end()) {
stream_stats_.insert({stats_key, StreamStats(captured_time)});
// Assume that the first freeze was before first stream frame captured.
// This way time before the first freeze would be counted as time
// between freezes.
stream_last_freeze_end_time_.insert({stats_key, start_time});
} else {
// When we see some `stream_label` for the first time we need to create
// stream stats object for it and set up some states, but we need to do
// it only once and for all receivers, so on the next frame on the same
// `stream_label` we can be sure, that it's already done and we needn't
// to scan though all peers again.
break;
}
}
}
{
MutexLock lock(&lock_);
MutexLock lock(&mutex_);
stream_to_sender_[stream_index] = peer_index;
frame_counters_.captured++;
for (size_t i = 0; i < peers_->size(); ++i) {
@ -215,12 +191,11 @@ uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured(
InternalStatsKey key(stream_index, peer_index, i);
stream_frame_counters_.at(key).dropped++;
MutexLock lock1(&comparison_lock_);
analyzer_stats_.frames_in_flight_left_count.AddSample(
StatsSample(captured_frames_in_flight_.size(), Now()));
AddComparison(InternalStatsKey(stream_index, peer_index, i),
it->second.frame(), absl::nullopt, true,
it->second.GetStatsForPeer(i));
frames_comparator_.AddComparison(
InternalStatsKey(stream_index, peer_index, i), it->second.frame(),
absl::nullopt, true, it->second.GetStatsForPeer(i));
}
captured_frames_in_flight_.erase(it);
@ -260,7 +235,7 @@ uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured(
void DefaultVideoQualityAnalyzer::OnFramePreEncode(
absl::string_view peer_name,
const webrtc::VideoFrame& frame) {
MutexLock lock(&lock_);
MutexLock lock(&mutex_);
auto it = captured_frames_in_flight_.find(frame.id());
RTC_DCHECK(it != captured_frames_in_flight_.end())
<< "Frame id=" << frame.id() << " not found";
@ -280,7 +255,7 @@ void DefaultVideoQualityAnalyzer::OnFrameEncoded(
uint16_t frame_id,
const webrtc::EncodedImage& encoded_image,
const EncoderStats& stats) {
MutexLock lock(&lock_);
MutexLock lock(&mutex_);
auto it = captured_frames_in_flight_.find(frame_id);
if (it == captured_frames_in_flight_.end()) {
RTC_LOG(WARNING)
@ -325,7 +300,7 @@ void DefaultVideoQualityAnalyzer::OnFramePreDecode(
absl::string_view peer_name,
uint16_t frame_id,
const webrtc::EncodedImage& input_image) {
MutexLock lock(&lock_);
MutexLock lock(&mutex_);
size_t peer_index = peers_->index(peer_name);
auto it = captured_frames_in_flight_.find(frame_id);
@ -361,7 +336,7 @@ void DefaultVideoQualityAnalyzer::OnFrameDecoded(
absl::string_view peer_name,
const webrtc::VideoFrame& frame,
const DecoderStats& stats) {
MutexLock lock(&lock_);
MutexLock lock(&mutex_);
size_t peer_index = peers_->index(peer_name);
auto it = captured_frames_in_flight_.find(frame.id());
@ -391,7 +366,7 @@ void DefaultVideoQualityAnalyzer::OnFrameDecoded(
void DefaultVideoQualityAnalyzer::OnFrameRendered(
absl::string_view peer_name,
const webrtc::VideoFrame& frame) {
MutexLock lock(&lock_);
MutexLock lock(&mutex_);
size_t peer_index = peers_->index(peer_name);
auto frame_it = captured_frames_in_flight_.find(frame.id());
@ -442,13 +417,11 @@ void DefaultVideoQualityAnalyzer::OnFrameRendered(
absl::optional<VideoFrame> dropped_frame = dropped_frame_it->second.frame();
dropped_frame_it->second.MarkDropped(peer_index);
{
MutexLock lock1(&comparison_lock_);
analyzer_stats_.frames_in_flight_left_count.AddSample(
StatsSample(captured_frames_in_flight_.size(), Now()));
AddComparison(stats_key, dropped_frame, absl::nullopt, true,
dropped_frame_it->second.GetStatsForPeer(peer_index));
}
analyzer_stats_.frames_in_flight_left_count.AddSample(
StatsSample(captured_frames_in_flight_.size(), Now()));
frames_comparator_.AddComparison(
stats_key, dropped_frame, absl::nullopt, true,
dropped_frame_it->second.GetStatsForPeer(peer_index));
if (dropped_frame_it->second.HaveAllPeersReceived()) {
captured_frames_in_flight_.erase(dropped_frame_it);
@ -463,15 +436,11 @@ void DefaultVideoQualityAnalyzer::OnFrameRendered(
}
state->SetLastRenderedFrameTime(peer_index,
frame_in_flight->rendered_time(peer_index));
{
MutexLock lock(&comparison_lock_);
stream_stats_.at(stats_key).skipped_between_rendered.AddSample(
StatsSample(dropped_count, Now()));
analyzer_stats_.frames_in_flight_left_count.AddSample(
StatsSample(captured_frames_in_flight_.size(), Now()));
AddComparison(stats_key, captured_frame, frame, false,
frame_in_flight->GetStatsForPeer(peer_index));
}
analyzer_stats_.frames_in_flight_left_count.AddSample(
StatsSample(captured_frames_in_flight_.size(), Now()));
frames_comparator_.AddComparison(
stats_key, dropped_count, captured_frame, frame, /*dropped=*/false,
frame_in_flight->GetStatsForPeer(peer_index));
if (frame_it->second.HaveAllPeersReceived()) {
captured_frames_in_flight_.erase(frame_it);
@ -495,8 +464,7 @@ void DefaultVideoQualityAnalyzer::OnDecoderError(absl::string_view peer_name,
void DefaultVideoQualityAnalyzer::RegisterParticipantInCall(
absl::string_view peer_name) {
MutexLock lock1(&lock_);
MutexLock lock2(&comparison_lock_);
MutexLock lock(&mutex_);
RTC_CHECK(!peers_->HasName(peer_name));
size_t new_peer_index = peers_->AddIfAbsent(peer_name);
@ -504,6 +472,7 @@ void DefaultVideoQualityAnalyzer::RegisterParticipantInCall(
// streams exists. Since in flight frames will be sent to the new peer
// as well. Sending stats (from this peer to others) will be added by
// DefaultVideoQualityAnalyzer::OnFrameCaptured.
std::vector<std::pair<InternalStatsKey, Timestamp>> stream_started_time;
for (auto& key_val : stream_to_sender_) {
size_t stream_index = key_val.first;
size_t sender_peer_index = key_val.second;
@ -528,11 +497,11 @@ void DefaultVideoQualityAnalyzer::RegisterParticipantInCall(
// then `counters` will be empty. In such case empty `counters` are ok.
stream_frame_counters_.insert({key, std::move(counters)});
stream_stats_.insert(
{key,
StreamStats(stream_states_.at(stream_index).stream_started_time())});
stream_last_freeze_end_time_.insert({key, start_time_});
stream_started_time.push_back(
{key, stream_states_.at(stream_index).stream_started_time()});
}
frames_comparator_.RegisterParticipantInCall(stream_started_time,
start_time_);
// Ensure, that frames states are handled correctly
// (e.g. dropped frames tracking).
for (auto& key_val : stream_states_) {
@ -549,30 +518,19 @@ void DefaultVideoQualityAnalyzer::RegisterParticipantInCall(
}
void DefaultVideoQualityAnalyzer::Stop() {
std::map<InternalStatsKey, Timestamp> last_rendered_frame_times;
{
MutexLock lock(&lock_);
MutexLock lock(&mutex_);
if (state_ == State::kStopped) {
return;
}
state_ = State::kStopped;
}
cpu_measurer_.StopMeasuringCpuProcessTime();
comparison_available_event_.Set();
thread_pool_.clear();
// Perform final Metrics update. On this place analyzer is stopped and no one
// holds any locks.
{
// Time between freezes.
// Count time since the last freeze to the end of the call as time
// between freezes.
MutexLock lock1(&lock_);
MutexLock lock2(&comparison_lock_);
for (auto& state_entry : stream_states_) {
const size_t stream_index = state_entry.first;
const StreamState& stream_state = state_entry.second;
for (size_t i = 0; i < peers_->size(); ++i) {
if (i == static_cast<size_t>(stream_state.owner())) {
if (i == stream_state.owner() && !options_.enable_receive_own_stream) {
continue;
}
@ -580,18 +538,31 @@ void DefaultVideoQualityAnalyzer::Stop() {
// If there are no freezes in the call we have to report
// time_between_freezes_ms as call duration and in such case
// `stream_last_freeze_end_time_` for this stream will be `start_time_`.
// `stream_last_freeze_end_time` for this stream will be `start_time_`.
// If there is freeze, then we need add time from last rendered frame
// to last freeze end as time between freezes.
if (stream_state.last_rendered_frame_time(i)) {
stream_stats_.at(stats_key).time_between_freezes_ms.AddSample(
StatsSample(
stream_state.last_rendered_frame_time(i).value().ms() -
stream_last_freeze_end_time_.at(stats_key).ms(),
Now()));
last_rendered_frame_times.emplace(
stats_key, stream_state.last_rendered_frame_time(i).value());
}
}
}
}
frames_comparator_.Stop(last_rendered_frame_times);
// Perform final Metrics update. On this place analyzer is stopped and no one
// holds any locks.
{
MutexLock lock(&mutex_);
FramesComparatorStats frames_comparator_stats =
frames_comparator_.frames_comparator_stats();
analyzer_stats_.comparisons_queue_size =
frames_comparator_stats.comparisons_queue_size;
analyzer_stats_.comparisons_done = frames_comparator_stats.comparisons_done;
analyzer_stats_.cpu_overloaded_comparisons_done =
frames_comparator_stats.cpu_overloaded_comparisons_done;
analyzer_stats_.memory_overloaded_comparisons_done =
frames_comparator_stats.memory_overloaded_comparisons_done;
analyzer_stats_.frames_in_flight_left_count.AddSample(
StatsSample(captured_frames_in_flight_.size(), Now()));
}
@ -599,7 +570,7 @@ void DefaultVideoQualityAnalyzer::Stop() {
}
std::string DefaultVideoQualityAnalyzer::GetStreamLabel(uint16_t frame_id) {
MutexLock lock1(&lock_);
MutexLock lock1(&mutex_);
auto it = captured_frames_in_flight_.find(frame_id);
if (it != captured_frames_in_flight_.end()) {
return streams_.name(it->second.stream());
@ -615,10 +586,9 @@ std::string DefaultVideoQualityAnalyzer::GetStreamLabel(uint16_t frame_id) {
}
std::set<StatsKey> DefaultVideoQualityAnalyzer::GetKnownVideoStreams() const {
MutexLock lock1(&lock_);
MutexLock lock2(&comparison_lock_);
MutexLock lock(&mutex_);
std::set<StatsKey> out;
for (auto& item : stream_stats_) {
for (auto& item : frames_comparator_.stream_stats()) {
RTC_LOG(INFO) << item.first.ToString() << " ==> "
<< ToStatsKey(item.first).ToString();
out.insert(ToStatsKey(item.first));
@ -627,13 +597,13 @@ std::set<StatsKey> DefaultVideoQualityAnalyzer::GetKnownVideoStreams() const {
}
const FrameCounters& DefaultVideoQualityAnalyzer::GetGlobalCounters() const {
MutexLock lock(&lock_);
MutexLock lock(&mutex_);
return frame_counters_;
}
std::map<StatsKey, FrameCounters>
DefaultVideoQualityAnalyzer::GetPerStreamCounters() const {
MutexLock lock(&lock_);
MutexLock lock(&mutex_);
std::map<StatsKey, FrameCounters> out;
for (auto& item : stream_frame_counters_) {
out.emplace(ToStatsKey(item.first), item.second);
@ -642,216 +612,31 @@ DefaultVideoQualityAnalyzer::GetPerStreamCounters() const {
}
std::map<StatsKey, StreamStats> DefaultVideoQualityAnalyzer::GetStats() const {
MutexLock lock1(&lock_);
MutexLock lock2(&comparison_lock_);
MutexLock lock1(&mutex_);
std::map<StatsKey, StreamStats> out;
for (auto& item : stream_stats_) {
for (auto& item : frames_comparator_.stream_stats()) {
out.emplace(ToStatsKey(item.first), item.second);
}
return out;
}
AnalyzerStats DefaultVideoQualityAnalyzer::GetAnalyzerStats() const {
MutexLock lock(&comparison_lock_);
MutexLock lock(&mutex_);
return analyzer_stats_;
}
void DefaultVideoQualityAnalyzer::AddComparison(
InternalStatsKey stats_key,
absl::optional<VideoFrame> captured,
absl::optional<VideoFrame> rendered,
bool dropped,
FrameStats frame_stats) {
cpu_measurer_.StartExcludingCpuThreadTime();
analyzer_stats_.comparisons_queue_size.AddSample(
StatsSample(comparisons_.size(), Now()));
// If there too many computations waiting in the queue, we won't provide
// frames itself to make future computations lighter.
if (comparisons_.size() >= kMaxActiveComparisons) {
comparisons_.emplace_back(std::move(stats_key), absl::nullopt,
absl::nullopt, dropped, std::move(frame_stats),
OverloadReason::kCpu);
} else {
OverloadReason overload_reason = OverloadReason::kNone;
if (!captured && !dropped) {
overload_reason = OverloadReason::kMemory;
}
comparisons_.emplace_back(std::move(stats_key), std::move(captured),
std::move(rendered), dropped,
std::move(frame_stats), overload_reason);
}
comparison_available_event_.Set();
cpu_measurer_.StopExcludingCpuThreadTime();
}
void DefaultVideoQualityAnalyzer::ProcessComparisons() {
while (true) {
// Try to pick next comparison to perform from the queue.
absl::optional<FrameComparison> comparison = absl::nullopt;
{
MutexLock lock(&comparison_lock_);
if (!comparisons_.empty()) {
comparison = comparisons_.front();
comparisons_.pop_front();
if (!comparisons_.empty()) {
comparison_available_event_.Set();
}
}
}
if (!comparison) {
bool more_frames_expected;
{
// If there are no comparisons and state is stopped =>
// no more frames expected.
MutexLock lock(&lock_);
more_frames_expected = state_ != State::kStopped;
}
if (!more_frames_expected) {
comparison_available_event_.Set();
return;
}
comparison_available_event_.Wait(1000);
continue;
}
cpu_measurer_.StartExcludingCpuThreadTime();
ProcessComparison(comparison.value());
cpu_measurer_.StopExcludingCpuThreadTime();
}
}
void DefaultVideoQualityAnalyzer::ProcessComparison(
const FrameComparison& comparison) {
// Perform expensive psnr and ssim calculations while not holding lock.
double psnr = -1.0;
double ssim = -1.0;
if (options_.heavy_metrics_computation_enabled && comparison.captured &&
!comparison.dropped) {
rtc::scoped_refptr<I420BufferInterface> reference_buffer =
comparison.captured->video_frame_buffer()->ToI420();
rtc::scoped_refptr<I420BufferInterface> test_buffer =
comparison.rendered->video_frame_buffer()->ToI420();
if (options_.adjust_cropping_before_comparing_frames) {
test_buffer =
ScaleVideoFrameBuffer(*test_buffer.get(), reference_buffer->width(),
reference_buffer->height());
reference_buffer = test::AdjustCropping(reference_buffer, test_buffer);
}
psnr = I420PSNR(*reference_buffer.get(), *test_buffer.get());
ssim = I420SSIM(*reference_buffer.get(), *test_buffer.get());
}
const FrameStats& frame_stats = comparison.frame_stats;
MutexLock lock(&comparison_lock_);
auto stats_it = stream_stats_.find(comparison.stats_key);
RTC_CHECK(stats_it != stream_stats_.end()) << comparison.stats_key.ToString();
StreamStats* stats = &stats_it->second;
analyzer_stats_.comparisons_done++;
if (comparison.overload_reason == OverloadReason::kCpu) {
analyzer_stats_.cpu_overloaded_comparisons_done++;
} else if (comparison.overload_reason == OverloadReason::kMemory) {
analyzer_stats_.memory_overloaded_comparisons_done++;
}
if (psnr > 0) {
stats->psnr.AddSample(StatsSample(psnr, frame_stats.rendered_time));
}
if (ssim > 0) {
stats->ssim.AddSample(StatsSample(ssim, frame_stats.received_time));
}
if (frame_stats.encoded_time.IsFinite()) {
stats->encode_time_ms.AddSample(StatsSample(
(frame_stats.encoded_time - frame_stats.pre_encode_time).ms(),
frame_stats.encoded_time));
stats->encode_frame_rate.AddEvent(frame_stats.encoded_time);
stats->total_encoded_images_payload += frame_stats.encoded_image_size;
stats->target_encode_bitrate.AddSample(StatsSample(
frame_stats.target_encode_bitrate, frame_stats.encoded_time));
} else {
if (frame_stats.pre_encode_time.IsFinite()) {
stats->dropped_by_encoder++;
} else {
stats->dropped_before_encoder++;
}
}
// Next stats can be calculated only if frame was received on remote side.
if (!comparison.dropped) {
stats->resolution_of_rendered_frame.AddSample(
StatsSample(*comparison.frame_stats.rendered_frame_width *
*comparison.frame_stats.rendered_frame_height,
frame_stats.rendered_time));
stats->transport_time_ms.AddSample(StatsSample(
(frame_stats.decode_start_time - frame_stats.encoded_time).ms(),
frame_stats.received_time));
stats->total_delay_incl_transport_ms.AddSample(StatsSample(
(frame_stats.rendered_time - frame_stats.captured_time).ms(),
frame_stats.received_time));
stats->decode_time_ms.AddSample(StatsSample(
(frame_stats.decode_end_time - frame_stats.decode_start_time).ms(),
frame_stats.decode_end_time));
stats->receive_to_render_time_ms.AddSample(StatsSample(
(frame_stats.rendered_time - frame_stats.received_time).ms(),
frame_stats.rendered_time));
if (frame_stats.prev_frame_rendered_time.IsFinite()) {
TimeDelta time_between_rendered_frames =
frame_stats.rendered_time - frame_stats.prev_frame_rendered_time;
stats->time_between_rendered_frames_ms.AddSample(StatsSample(
time_between_rendered_frames.ms(), frame_stats.rendered_time));
double average_time_between_rendered_frames_ms =
stats->time_between_rendered_frames_ms.GetAverage();
if (time_between_rendered_frames.ms() >
std::max(kFreezeThresholdMs + average_time_between_rendered_frames_ms,
3 * average_time_between_rendered_frames_ms)) {
stats->freeze_time_ms.AddSample(StatsSample(
time_between_rendered_frames.ms(), frame_stats.rendered_time));
auto freeze_end_it =
stream_last_freeze_end_time_.find(comparison.stats_key);
RTC_DCHECK(freeze_end_it != stream_last_freeze_end_time_.end());
stats->time_between_freezes_ms.AddSample(StatsSample(
(frame_stats.prev_frame_rendered_time - freeze_end_it->second).ms(),
frame_stats.rendered_time));
freeze_end_it->second = frame_stats.rendered_time;
}
}
}
// Compute stream codec info.
if (frame_stats.used_encoder.has_value()) {
if (stats->encoders.empty() || stats->encoders.back().codec_name !=
frame_stats.used_encoder->codec_name) {
stats->encoders.push_back(*frame_stats.used_encoder);
}
stats->encoders.back().last_frame_id =
frame_stats.used_encoder->last_frame_id;
stats->encoders.back().switched_from_at =
frame_stats.used_encoder->switched_from_at;
}
if (frame_stats.used_decoder.has_value()) {
if (stats->decoders.empty() || stats->decoders.back().codec_name !=
frame_stats.used_decoder->codec_name) {
stats->decoders.push_back(*frame_stats.used_decoder);
}
stats->decoders.back().last_frame_id =
frame_stats.used_decoder->last_frame_id;
stats->decoders.back().switched_from_at =
frame_stats.used_decoder->switched_from_at;
}
}
void DefaultVideoQualityAnalyzer::ReportResults() {
using ::webrtc::test::ImproveDirection;
MutexLock lock1(&lock_);
MutexLock lock2(&comparison_lock_);
for (auto& item : stream_stats_) {
MutexLock lock(&mutex_);
for (auto& item : frames_comparator_.stream_stats()) {
ReportResults(GetTestCaseName(StatsKeyToMetricName(ToStatsKey(item.first))),
item.second, stream_frame_counters_.at(item.first));
}
test::PrintResult("cpu_usage", "", test_label_.c_str(), GetCpuUsagePercent(),
"%", false, ImproveDirection::kSmallerIsBetter);
LogFrameCounters("Global", frame_counters_);
for (auto& item : stream_stats_) {
for (auto& item : frames_comparator_.stream_stats()) {
LogFrameCounters(ToStatsKey(item.first).ToString(),
stream_frame_counters_.at(item.first));
LogStreamInternalStats(ToStatsKey(item.first).ToString(), item.second,

80
test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
View File

@ -30,6 +30,7 @@
#include "rtc_base/synchronization/mutex.h"
#include "system_wrappers/include/clock.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
#include "test/pc/e2e/analyzer/video/multi_head_queue.h"
@ -38,33 +39,11 @@
namespace webrtc {
namespace webrtc_pc_e2e {
struct DefaultVideoQualityAnalyzerOptions {
// Tells DefaultVideoQualityAnalyzer if heavy metrics like PSNR and SSIM have
// to be computed or not.
bool heavy_metrics_computation_enabled = true;
// If true DefaultVideoQualityAnalyzer will try to adjust frames before
// computing PSNR and SSIM for them. In some cases picture may be shifted by
// a few pixels after the encode/decode step. Those difference is invisible
// for a human eye, but it affects the metrics. So the adjustment is used to
// get metrics that are closer to how human persepts the video. This feature
// significantly slows down the comparison, so turn it on only when it is
// needed.
bool adjust_cropping_before_comparing_frames = false;
// Amount of frames that are queued in the DefaultVideoQualityAnalyzer from
// the point they were captured to the point they were rendered on all
// receivers per stream.
size_t max_frames_in_flight_per_stream_count =
kDefaultMaxFramesInFlightPerStream;
// If true, the analyzer will expect peers to receive their own video streams.
bool enable_receive_own_stream = false;
};
class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
public:
explicit DefaultVideoQualityAnalyzer(
webrtc::Clock* clock,
DefaultVideoQualityAnalyzerOptions options =
DefaultVideoQualityAnalyzerOptions());
DefaultVideoQualityAnalyzerOptions options = {});
~DefaultVideoQualityAnalyzer() override;
void Start(std::string test_case_name,
@ -307,21 +286,12 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
std::map<absl::string_view, size_t> index_;
};
void AddComparison(InternalStatsKey stats_key,
absl::optional<VideoFrame> captured,
absl::optional<VideoFrame> rendered,
bool dropped,
FrameStats frame_stats)
RTC_EXCLUSIVE_LOCKS_REQUIRED(comparison_lock_);
static void ProcessComparisonsThread(void* obj);
void ProcessComparisons();
void ProcessComparison(const FrameComparison& comparison);
// Report results for all metrics for all streams.
void ReportResults();
void ReportResults(const std::string& test_case_name,
const StreamStats& stats,
const FrameCounters& frame_counters)
RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_);
RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
// Report result for single metric for specified stream.
static void ReportResult(const std::string& metric_name,
const std::string& test_case_name,
@ -333,28 +303,27 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
std::string GetTestCaseName(const std::string& stream_label) const;
Timestamp Now();
StatsKey ToStatsKey(const InternalStatsKey& key) const
RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_);
RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
// Returns string representation of stats key for metrics naming. Used for
// backward compatibility by metrics naming for 2 peers cases.
std::string StatsKeyToMetricName(const StatsKey& key) const
RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_);
RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
// TODO(titovartem) restore const when old constructor will be removed.
DefaultVideoQualityAnalyzerOptions options_;
const DefaultVideoQualityAnalyzerOptions options_;
webrtc::Clock* const clock_;
std::atomic<uint16_t> next_frame_id_{0};
std::string test_label_;
mutable Mutex lock_;
std::unique_ptr<NamesCollection> peers_ RTC_GUARDED_BY(lock_);
State state_ RTC_GUARDED_BY(lock_) = State::kNew;
Timestamp start_time_ RTC_GUARDED_BY(lock_) = Timestamp::MinusInfinity();
mutable Mutex mutex_;
std::unique_ptr<NamesCollection> peers_ RTC_GUARDED_BY(mutex_);
State state_ RTC_GUARDED_BY(mutex_) = State::kNew;
Timestamp start_time_ RTC_GUARDED_BY(mutex_) = Timestamp::MinusInfinity();
// Mapping from stream label to unique size_t value to use in stats and avoid
// extra string copying.
NamesCollection streams_ RTC_GUARDED_BY(lock_);
NamesCollection streams_ RTC_GUARDED_BY(mutex_);
// Frames that were captured by all streams and still aren't rendered on
// receviers or deemed dropped. Frame with id X can be removed from this map
// receivers or deemed dropped. Frame with id X can be removed from this map
// if:
// 1. The frame with id X was received in OnFrameRendered by all expected
// receivers.
@ -365,36 +334,27 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
// oldest frame id in this stream. In such case only the frame content
// will be removed, but the map entry will be preserved.
std::map<uint16_t, FrameInFlight> captured_frames_in_flight_
RTC_GUARDED_BY(lock_);
RTC_GUARDED_BY(mutex_);
// Global frames count for all video streams.
FrameCounters frame_counters_ RTC_GUARDED_BY(lock_);
FrameCounters frame_counters_ RTC_GUARDED_BY(mutex_);
// Frame counters per each stream per each receiver.
std::map<InternalStatsKey, FrameCounters> stream_frame_counters_
RTC_GUARDED_BY(lock_);
RTC_GUARDED_BY(mutex_);
// Map from stream index in `streams_` to its StreamState.
std::map<size_t, StreamState> stream_states_ RTC_GUARDED_BY(lock_);
std::map<size_t, StreamState> stream_states_ RTC_GUARDED_BY(mutex_);
// Map from stream index in `streams_` to sender peer index in `peers_`.
std::map<size_t, size_t> stream_to_sender_ RTC_GUARDED_BY(lock_);
std::map<size_t, size_t> stream_to_sender_ RTC_GUARDED_BY(mutex_);
// Stores history mapping between stream index in `streams_` and frame ids.
// Updated when frame id overlap. It required to properly return stream label
// after 1st frame from simulcast streams was already rendered and last is
// still encoding.
std::map<size_t, std::set<uint16_t>> stream_to_frame_id_history_
RTC_GUARDED_BY(lock_);
mutable Mutex comparison_lock_;
std::map<InternalStatsKey, StreamStats> stream_stats_
RTC_GUARDED_BY(comparison_lock_);
std::map<InternalStatsKey, Timestamp> stream_last_freeze_end_time_
RTC_GUARDED_BY(comparison_lock_);
std::deque<FrameComparison> comparisons_ RTC_GUARDED_BY(comparison_lock_);
AnalyzerStats analyzer_stats_ RTC_GUARDED_BY(comparison_lock_);
std::vector<rtc::PlatformThread> thread_pool_;
rtc::Event comparison_available_event_;
RTC_GUARDED_BY(mutex_);
AnalyzerStats analyzer_stats_ RTC_GUARDED_BY(mutex_);
DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer_;
DefaultVideoQualityAnalyzerFramesComparator frames_comparator_;
};
} // namespace webrtc_pc_e2e

359
test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc Normal file
View File

@ -0,0 +1,359 @@
/*
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h"
#include <algorithm>
#include <utility>
#include <vector>
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "api/scoped_refptr.h"
#include "api/video/i420_buffer.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "rtc_base/checks.h"
#include "rtc_base/platform_thread.h"
#include "rtc_base/synchronization/mutex.h"
#include "rtc_tools/frame_analyzer/video_geometry_aligner.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
namespace webrtc {
namespace {
constexpr int kFreezeThresholdMs = 150;
constexpr int kMaxActiveComparisons = 10;
SamplesStatsCounter::StatsSample StatsSample(double value,
Timestamp sampling_time) {
return SamplesStatsCounter::StatsSample{value, sampling_time};
}
} // namespace
void DefaultVideoQualityAnalyzerFramesComparator::Start(int max_threads_count) {
for (int i = 0; i < max_threads_count; i++) {
thread_pool_.push_back(rtc::PlatformThread::SpawnJoinable(
[this] { ProcessComparisons(); },
"DefaultVideoQualityAnalyzerFramesComparator-" + std::to_string(i)));
}
{
MutexLock lock(&mutex_);
RTC_CHECK_EQ(state_, State::kNew) << "Frames comparator is already started";
state_ = State::kActive;
}
cpu_measurer_.StartMeasuringCpuProcessTime();
}
void DefaultVideoQualityAnalyzerFramesComparator::Stop(
const std::map<InternalStatsKey, Timestamp>& last_rendered_frame_times) {
{
MutexLock lock(&mutex_);
if (state_ == State::kStopped) {
return;
}
RTC_CHECK_EQ(state_, State::kActive)
<< "Frames comparator has to be started before it will be used";
state_ = State::kStopped;
}
cpu_measurer_.StopMeasuringCpuProcessTime();
comparison_available_event_.Set();
thread_pool_.clear();
{
MutexLock lock(&mutex_);
// Perform final Metrics update. On this place analyzer is stopped and no
// one holds any locks.
// Time between freezes.
// Count time since the last freeze to the end of the call as time
// between freezes.
for (auto& entry : last_rendered_frame_times) {
const InternalStatsKey& stats_key = entry.first;
const Timestamp& last_rendered_frame_time = entry.second;
// If there are no freezes in the call we have to report
// time_between_freezes_ms as call duration and in such case
// `last_rendered_frame_time` for this stream will be stream start time.
// If there is freeze, then we need add time from last rendered frame
// to last freeze end as time between freezes.
stream_stats_.at(stats_key).time_between_freezes_ms.AddSample(
StatsSample(last_rendered_frame_time.ms() -
stream_last_freeze_end_time_.at(stats_key).ms(),
Now()));
}
}
}
void DefaultVideoQualityAnalyzerFramesComparator::EnsureStatsForStream(
size_t stream_index,
size_t sender_peer_index,
size_t peers_count,
Timestamp captured_time,
Timestamp start_time) {
MutexLock lock(&mutex_);
RTC_CHECK_EQ(state_, State::kActive)
<< "Frames comparator has to be started before it will be used";
for (size_t i = 0; i < peers_count; ++i) {
if (i == sender_peer_index && !options_.enable_receive_own_stream) {
continue;
}
InternalStatsKey stats_key(stream_index, sender_peer_index, i);
if (stream_stats_.find(stats_key) == stream_stats_.end()) {
stream_stats_.insert(
{stats_key, webrtc_pc_e2e::StreamStats(captured_time)});
// Assume that the first freeze was before first stream frame captured.
// This way time before the first freeze would be counted as time
// between freezes.
stream_last_freeze_end_time_.insert({stats_key, start_time});
} else {
// When we see some `stream_label` for the first time we need to create
// stream stats object for it and set up some states, but we need to do
// it only once and for all receivers, so on the next frame on the same
// `stream_label` we can be sure, that it's already done and we needn't
// to scan though all peers again.
break;
}
}
}
void DefaultVideoQualityAnalyzerFramesComparator::RegisterParticipantInCall(
rtc::ArrayView<std::pair<InternalStatsKey, Timestamp>> stream_started_time,
Timestamp start_time) {
MutexLock lock(&mutex_);
RTC_CHECK_EQ(state_, State::kActive)
<< "Frames comparator has to be started before it will be used";
for (const std::pair<InternalStatsKey, Timestamp>& pair :
stream_started_time) {
stream_stats_.insert({pair.first, webrtc_pc_e2e::StreamStats(pair.second)});
stream_last_freeze_end_time_.insert({pair.first, start_time});
}
}
void DefaultVideoQualityAnalyzerFramesComparator::AddComparison(
InternalStatsKey stats_key,
absl::optional<VideoFrame> captured,
absl::optional<VideoFrame> rendered,
bool dropped,
FrameStats frame_stats) {
MutexLock lock(&mutex_);
RTC_CHECK_EQ(state_, State::kActive)
<< "Frames comparator has to be started before it will be used";
AddComparisonInternal(std::move(stats_key), std::move(captured),
std::move(rendered), dropped, std::move(frame_stats));
}
void DefaultVideoQualityAnalyzerFramesComparator::AddComparison(
InternalStatsKey stats_key,
int skipped_between_rendered,
absl::optional<VideoFrame> captured,
absl::optional<VideoFrame> rendered,
bool dropped,
FrameStats frame_stats) {
MutexLock lock(&mutex_);
RTC_CHECK_EQ(state_, State::kActive)
<< "Frames comparator has to be started before it will be used";
stream_stats_.at(stats_key).skipped_between_rendered.AddSample(
StatsSample(skipped_between_rendered, Now()));
AddComparisonInternal(std::move(stats_key), std::move(captured),
std::move(rendered), dropped, std::move(frame_stats));
}
void DefaultVideoQualityAnalyzerFramesComparator::AddComparisonInternal(
InternalStatsKey stats_key,
absl::optional<VideoFrame> captured,
absl::optional<VideoFrame> rendered,
bool dropped,
FrameStats frame_stats) {
cpu_measurer_.StartExcludingCpuThreadTime();
frames_comparator_stats_.comparisons_queue_size.AddSample(
StatsSample(comparisons_.size(), Now()));
// If there too many computations waiting in the queue, we won't provide
// frames itself to make future computations lighter.
if (comparisons_.size() >= kMaxActiveComparisons) {
comparisons_.emplace_back(std::move(stats_key), absl::nullopt,
absl::nullopt, dropped, std::move(frame_stats),
OverloadReason::kCpu);
} else {
OverloadReason overload_reason = OverloadReason::kNone;
if (!captured && !dropped) {
overload_reason = OverloadReason::kMemory;
}
comparisons_.emplace_back(std::move(stats_key), std::move(captured),
std::move(rendered), dropped,
std::move(frame_stats), overload_reason);
}
comparison_available_event_.Set();
cpu_measurer_.StopExcludingCpuThreadTime();
}
void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparisons() {
while (true) {
// Try to pick next comparison to perform from the queue.
absl::optional<FrameComparison> comparison = absl::nullopt;
{
MutexLock lock(&mutex_);
if (!comparisons_.empty()) {
comparison = comparisons_.front();
comparisons_.pop_front();
if (!comparisons_.empty()) {
comparison_available_event_.Set();
}
}
}
if (!comparison) {
bool more_frames_expected;
{
// If there are no comparisons and state is stopped =>
// no more frames expected.
MutexLock lock(&mutex_);
more_frames_expected = state_ != State::kStopped;
}
if (!more_frames_expected) {
comparison_available_event_.Set();
return;
}
comparison_available_event_.Wait(1000);
continue;
}
cpu_measurer_.StartExcludingCpuThreadTime();
ProcessComparison(comparison.value());
cpu_measurer_.StopExcludingCpuThreadTime();
}
}
void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparison(
const FrameComparison& comparison) {
// Perform expensive psnr and ssim calculations while not holding lock.
double psnr = -1.0;
double ssim = -1.0;
if (options_.heavy_metrics_computation_enabled && comparison.captured &&
!comparison.dropped) {
rtc::scoped_refptr<I420BufferInterface> reference_buffer =
comparison.captured->video_frame_buffer()->ToI420();
rtc::scoped_refptr<I420BufferInterface> test_buffer =
comparison.rendered->video_frame_buffer()->ToI420();
if (options_.adjust_cropping_before_comparing_frames) {
test_buffer =
ScaleVideoFrameBuffer(*test_buffer.get(), reference_buffer->width(),
reference_buffer->height());
reference_buffer = test::AdjustCropping(reference_buffer, test_buffer);
}
psnr = I420PSNR(*reference_buffer.get(), *test_buffer.get());
ssim = I420SSIM(*reference_buffer.get(), *test_buffer.get());
}
const FrameStats& frame_stats = comparison.frame_stats;
MutexLock lock(&mutex_);
auto stats_it = stream_stats_.find(comparison.stats_key);
RTC_CHECK(stats_it != stream_stats_.end()) << comparison.stats_key.ToString();
webrtc_pc_e2e::StreamStats* stats = &stats_it->second;
frames_comparator_stats_.comparisons_done++;
if (comparison.overload_reason == OverloadReason::kCpu) {
frames_comparator_stats_.cpu_overloaded_comparisons_done++;
} else if (comparison.overload_reason == OverloadReason::kMemory) {
frames_comparator_stats_.memory_overloaded_comparisons_done++;
}
if (psnr > 0) {
stats->psnr.AddSample(StatsSample(psnr, frame_stats.rendered_time));
}
if (ssim > 0) {
stats->ssim.AddSample(StatsSample(ssim, frame_stats.received_time));
}
if (frame_stats.encoded_time.IsFinite()) {
stats->encode_time_ms.AddSample(StatsSample(
(frame_stats.encoded_time - frame_stats.pre_encode_time).ms(),
frame_stats.encoded_time));
stats->encode_frame_rate.AddEvent(frame_stats.encoded_time);
stats->total_encoded_images_payload += frame_stats.encoded_image_size;
stats->target_encode_bitrate.AddSample(StatsSample(
frame_stats.target_encode_bitrate, frame_stats.encoded_time));
} else {
if (frame_stats.pre_encode_time.IsFinite()) {
stats->dropped_by_encoder++;
} else {
stats->dropped_before_encoder++;
}
}
// Next stats can be calculated only if frame was received on remote side.
if (!comparison.dropped) {
stats->resolution_of_rendered_frame.AddSample(
StatsSample(*comparison.frame_stats.rendered_frame_width *
*comparison.frame_stats.rendered_frame_height,
frame_stats.rendered_time));
stats->transport_time_ms.AddSample(StatsSample(
(frame_stats.decode_start_time - frame_stats.encoded_time).ms(),
frame_stats.received_time));
stats->total_delay_incl_transport_ms.AddSample(StatsSample(
(frame_stats.rendered_time - frame_stats.captured_time).ms(),
frame_stats.received_time));
stats->decode_time_ms.AddSample(StatsSample(
(frame_stats.decode_end_time - frame_stats.decode_start_time).ms(),
frame_stats.decode_end_time));
stats->receive_to_render_time_ms.AddSample(StatsSample(
(frame_stats.rendered_time - frame_stats.received_time).ms(),
frame_stats.rendered_time));
if (frame_stats.prev_frame_rendered_time.IsFinite()) {
TimeDelta time_between_rendered_frames =
frame_stats.rendered_time - frame_stats.prev_frame_rendered_time;
stats->time_between_rendered_frames_ms.AddSample(StatsSample(
time_between_rendered_frames.ms(), frame_stats.rendered_time));
double average_time_between_rendered_frames_ms =
stats->time_between_rendered_frames_ms.GetAverage();
if (time_between_rendered_frames.ms() >
std::max(kFreezeThresholdMs + average_time_between_rendered_frames_ms,
3 * average_time_between_rendered_frames_ms)) {
stats->freeze_time_ms.AddSample(StatsSample(
time_between_rendered_frames.ms(), frame_stats.rendered_time));
auto freeze_end_it =
stream_last_freeze_end_time_.find(comparison.stats_key);
RTC_DCHECK(freeze_end_it != stream_last_freeze_end_time_.end());
stats->time_between_freezes_ms.AddSample(StatsSample(
(frame_stats.prev_frame_rendered_time - freeze_end_it->second).ms(),
frame_stats.rendered_time));
freeze_end_it->second = frame_stats.rendered_time;
}
}
}
// Compute stream codec info.
if (frame_stats.used_encoder.has_value()) {
if (stats->encoders.empty() || stats->encoders.back().codec_name !=
frame_stats.used_encoder->codec_name) {
stats->encoders.push_back(*frame_stats.used_encoder);
}
stats->encoders.back().last_frame_id =
frame_stats.used_encoder->last_frame_id;
stats->encoders.back().switched_from_at =
frame_stats.used_encoder->switched_from_at;
}
if (frame_stats.used_decoder.has_value()) {
if (stats->decoders.empty() || stats->decoders.back().codec_name !=
frame_stats.used_decoder->codec_name) {
stats->decoders.push_back(*frame_stats.used_decoder);
}
stats->decoders.back().last_frame_id =
frame_stats.used_decoder->last_frame_id;
stats->decoders.back().switched_from_at =
frame_stats.used_decoder->switched_from_at;
}
}
Timestamp DefaultVideoQualityAnalyzerFramesComparator::Now() {
return clock_->CurrentTime();
}
} // namespace webrtc

150
test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h Normal file
View File

@ -0,0 +1,150 @@
/*
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAMES_COMPARATOR_H_
#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAMES_COMPARATOR_H_
#include <deque>
#include <map>
#include <utility>
#include <vector>
#include "api/array_view.h"
#include "rtc_base/event.h"
#include "rtc_base/platform_thread.h"
#include "rtc_base/synchronization/mutex.h"
#include "system_wrappers/include/clock.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
namespace webrtc {
struct FramesComparatorStats {
// Size of analyzer internal comparisons queue, measured when new element
// id added to the queue.
SamplesStatsCounter comparisons_queue_size;
// Number of performed comparisons of 2 video frames from captured and
// rendered streams.
int64_t comparisons_done = 0;
// Number of cpu overloaded comparisons. Comparison is cpu overloaded if it is
// queued when there are too many not processed comparisons in the queue.
// Overloaded comparison doesn't include metrics like SSIM and PSNR that
// require heavy computations.
int64_t cpu_overloaded_comparisons_done = 0;
// Number of memory overloaded comparisons. Comparison is memory overloaded if
// it is queued when its captured frame was already removed due to high memory
// usage for that video stream.
int64_t memory_overloaded_comparisons_done = 0;
};
// Performs comparisons of added frames and tracks frames related statistics.
// This class is thread safe.
class DefaultVideoQualityAnalyzerFramesComparator {
public:
// Creates frames comparator.
// Frames comparator doesn't use `options.enable_receive_own_stream` for any
// purposes, because it's unrelated to its functionality.
DefaultVideoQualityAnalyzerFramesComparator(
webrtc::Clock* clock,
DefaultVideoQualityAnalyzerCpuMeasurer& cpu_measurer,
webrtc_pc_e2e::DefaultVideoQualityAnalyzerOptions options = {})
: options_(options), clock_(clock), cpu_measurer_(cpu_measurer) {}
~DefaultVideoQualityAnalyzerFramesComparator() { Stop({}); }
// Starts frames comparator. This method must be invoked before calling
// any other method on this object.
void Start(int max_threads_count);
// Stops frames comparator. This method will block until all added frame
// comparisons will be processed. After `Stop()` is invoked no more new
// comparisons can be added to this frames comparator.
//
// `last_rendered_frame_time` contains timestamps of last rendered frame for
// each (stream, sender, receiver) tuple to properly update time between
// freezes: it has include time from the last freeze until and of call.
void Stop(
const std::map<InternalStatsKey, Timestamp>& last_rendered_frame_times);
// Ensures that stream `stream_index` has stats objects created for all
// potential receivers. This method must be called before adding any
// frames comparison for that stream.
void EnsureStatsForStream(size_t stream_index,
size_t sender_peer_index,
size_t peers_count,
Timestamp captured_time,
Timestamp start_time);
// Ensures that newly added participant will have stream stats objects created
// for all streams which they can receive. This method must be called before
// any frames comparison will be added for the newly added participant.
//
// `stream_started_time` - start time of each stream for which stats object
// has to be created.
// `start_time` - call start time.
void RegisterParticipantInCall(
rtc::ArrayView<std::pair<InternalStatsKey, Timestamp>>
stream_started_time,
Timestamp start_time);
void AddComparison(InternalStatsKey stats_key,
absl::optional<VideoFrame> captured,
absl::optional<VideoFrame> rendered,
bool dropped,
FrameStats frame_stats);
// `skipped_between_rendered` - amount of frames dropped on this stream before
// last received frame and current frame.
void AddComparison(InternalStatsKey stats_key,
int skipped_between_rendered,
absl::optional<VideoFrame> captured,
absl::optional<VideoFrame> rendered,
bool dropped,
FrameStats frame_stats);
std::map<InternalStatsKey, webrtc_pc_e2e::StreamStats> stream_stats() const {
MutexLock lock(&mutex_);
return stream_stats_;
}
FramesComparatorStats frames_comparator_stats() const {
MutexLock lock(&mutex_);
return frames_comparator_stats_;
}
private:
enum State { kNew, kActive, kStopped };
void AddComparisonInternal(InternalStatsKey stats_key,
absl::optional<VideoFrame> captured,
absl::optional<VideoFrame> rendered,
bool dropped,
FrameStats frame_stats)
RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
void ProcessComparisons();
void ProcessComparison(const FrameComparison& comparison);
Timestamp Now();
const webrtc_pc_e2e::DefaultVideoQualityAnalyzerOptions options_;
webrtc::Clock* const clock_;
DefaultVideoQualityAnalyzerCpuMeasurer& cpu_measurer_;
mutable Mutex mutex_;
State state_ RTC_GUARDED_BY(mutex_) = State::kNew;
std::map<InternalStatsKey, webrtc_pc_e2e::StreamStats> stream_stats_
RTC_GUARDED_BY(mutex_);
std::map<InternalStatsKey, Timestamp> stream_last_freeze_end_time_
RTC_GUARDED_BY(mutex_);
std::deque<FrameComparison> comparisons_ RTC_GUARDED_BY(mutex_);
FramesComparatorStats frames_comparator_stats_ RTC_GUARDED_BY(mutex_);
std::vector<rtc::PlatformThread> thread_pool_;
rtc::Event comparison_available_event_;
};
} // namespace webrtc
#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAMES_COMPARATOR_H_

150
test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc Normal file
View File

@ -0,0 +1,150 @@
/*
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h"
#include <map>
#include "api/test/create_frame_generator.h"
#include "api/units/timestamp.h"
#include "system_wrappers/include/clock.h"
#include "system_wrappers/include/sleep.h"
#include "test/gtest.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
namespace webrtc {
namespace {
using StatsSample = ::webrtc::SamplesStatsCounter::StatsSample;
constexpr int kMaxFramesInFlightPerStream = 10;
webrtc_pc_e2e::DefaultVideoQualityAnalyzerOptions AnalyzerOptionsForTest() {
webrtc_pc_e2e::DefaultVideoQualityAnalyzerOptions options;
options.heavy_metrics_computation_enabled = false;
options.adjust_cropping_before_comparing_frames = false;
options.max_frames_in_flight_per_stream_count = kMaxFramesInFlightPerStream;
return options;
}
webrtc_pc_e2e::StreamCodecInfo Vp8CodecForOneFrame(uint16_t frame_id,
Timestamp time) {
webrtc_pc_e2e::StreamCodecInfo info;
info.codec_name = "VP8";
info.first_frame_id = frame_id;
info.last_frame_id = frame_id;
info.switched_on_at = time;
info.switched_from_at = time;
return info;
}
FrameStats FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame(
Timestamp captured_time) {
FrameStats frame_stats(captured_time);
frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
frame_stats.encoded_time = captured_time + TimeDelta::Millis(20);
frame_stats.received_time = captured_time + TimeDelta::Millis(30);
frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40);
frame_stats.decode_end_time = captured_time + TimeDelta::Millis(50);
frame_stats.rendered_time = captured_time + TimeDelta::Millis(60);
frame_stats.used_encoder = Vp8CodecForOneFrame(1, frame_stats.encoded_time);
frame_stats.used_encoder =
Vp8CodecForOneFrame(1, frame_stats.decode_end_time);
frame_stats.rendered_frame_width = 10;
frame_stats.rendered_frame_height = 10;
return frame_stats;
}
double GetFirstOrDie(const SamplesStatsCounter& counter) {
EXPECT_TRUE(!counter.IsEmpty()) << "Counter has to be not empty";
return counter.GetSamples()[0];
}
TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
StatsPresentedAfterAddingOneComparison) {
DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
DefaultVideoQualityAnalyzerFramesComparator comparator(
Clock::GetRealTimeClock(), cpu_measurer, AnalyzerOptionsForTest());
Timestamp stream_start_time = Clock::GetRealTimeClock()->CurrentTime();
size_t stream = 0;
size_t sender = 0;
size_t receiver = 1;
size_t peers_count = 2;
InternalStatsKey stats_key(stream, sender, receiver);
FrameStats frame_stats =
FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame(stream_start_time);
comparator.Start(1);
comparator.EnsureStatsForStream(stream, sender, peers_count,
stream_start_time, stream_start_time);
comparator.AddComparison(stats_key,
/*captured=*/absl::nullopt,
/*rendered=*/absl::nullopt, /*dropped=*/false,
frame_stats);
comparator.Stop({});
std::map<InternalStatsKey, webrtc_pc_e2e::StreamStats> stats =
comparator.stream_stats();
EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.at(stats_key).transport_time_ms), 20.0);
EXPECT_DOUBLE_EQ(
GetFirstOrDie(stats.at(stats_key).total_delay_incl_transport_ms), 60.0);
EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.at(stats_key).encode_time_ms), 10.0);
EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.at(stats_key).decode_time_ms), 10.0);
EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.at(stats_key).receive_to_render_time_ms),
30.0);
EXPECT_DOUBLE_EQ(
GetFirstOrDie(stats.at(stats_key).resolution_of_rendered_frame), 100.0);
}
TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
MultiFrameStatsPresentedAfterAddingTwoComparisonWith10msDelay) {
DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
DefaultVideoQualityAnalyzerFramesComparator comparator(
Clock::GetRealTimeClock(), cpu_measurer, AnalyzerOptionsForTest());
Timestamp stream_start_time = Clock::GetRealTimeClock()->CurrentTime();
size_t stream = 0;
size_t sender = 0;
size_t receiver = 1;
size_t peers_count = 2;
InternalStatsKey stats_key(stream, sender, receiver);
FrameStats frame_stats1 =
FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame(stream_start_time);
FrameStats frame_stats2 = FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame(
stream_start_time + TimeDelta::Millis(15));
frame_stats2.prev_frame_rendered_time = frame_stats1.rendered_time;
comparator.Start(1);
comparator.EnsureStatsForStream(stream, sender, peers_count,
stream_start_time, stream_start_time);
comparator.AddComparison(stats_key,
/*captured=*/absl::nullopt,
/*rendered=*/absl::nullopt, /*dropped=*/false,
frame_stats1);
comparator.AddComparison(stats_key,
/*captured=*/absl::nullopt,
/*rendered=*/absl::nullopt, /*dropped=*/false,
frame_stats2);
comparator.Stop({});
std::map<InternalStatsKey, webrtc_pc_e2e::StreamStats> stats =
comparator.stream_stats();
EXPECT_DOUBLE_EQ(
GetFirstOrDie(stats.at(stats_key).time_between_rendered_frames_ms), 15.0);
EXPECT_DOUBLE_EQ(stats.at(stats_key).encode_frame_rate.GetEventsPerSecond(),
2.0 / 15 * 1000);
}
} // namespace
} // namespace webrtc

4
test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.cc
View File

@ -26,7 +26,7 @@ void RateCounter::AddEvent(Timestamp event_time) {
event_first_time_ = event_time;
}
event_last_time_ = event_time;
event_count_++;
events_count_++;
}
double RateCounter::GetEventsPerSecond() const {
@ -34,7 +34,7 @@ double RateCounter::GetEventsPerSecond() const {
// Divide on us and multiply on kMicrosPerSecond to correctly process cases
// where there were too small amount of events, so difference is less then 1
// sec. We can use us here, because Timestamp has us resolution.
return static_cast<double>(event_count_) /
return static_cast<double>(events_count_) /
(event_last_time_ - event_first_time_).us() * kMicrosPerSecond;
}

23
test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h
View File

@ -39,7 +39,7 @@ class RateCounter {
private:
Timestamp event_first_time_ = Timestamp::MinusInfinity();
Timestamp event_last_time_ = Timestamp::MinusInfinity();
int64_t event_count_ = 0;
int64_t events_count_ = 0;
};
struct FrameCounters {
@ -165,6 +165,27 @@ struct StatsKey {
bool operator<(const StatsKey& a, const StatsKey& b);
bool operator==(const StatsKey& a, const StatsKey& b);
struct DefaultVideoQualityAnalyzerOptions {
// Tells DefaultVideoQualityAnalyzer if heavy metrics like PSNR and SSIM have
// to be computed or not.
bool heavy_metrics_computation_enabled = true;
// If true DefaultVideoQualityAnalyzer will try to adjust frames before
// computing PSNR and SSIM for them. In some cases picture may be shifted by
// a few pixels after the encode/decode step. Those difference is invisible
// for a human eye, but it affects the metrics. So the adjustment is used to
// get metrics that are closer to how human perceive the video. This feature
// significantly slows down the comparison, so turn it on only when it is
// needed.
bool adjust_cropping_before_comparing_frames = false;
// Amount of frames that are queued in the DefaultVideoQualityAnalyzer from
// the point they were captured to the point they were rendered on all
// receivers per stream.
size_t max_frames_in_flight_per_stream_count =
kDefaultMaxFramesInFlightPerStream;
// If true, the analyzer will expect peers to receive their own video streams.
bool enable_receive_own_stream = false;
};
} // namespace webrtc_pc_e2e
} // namespace webrtc