diff --git a/test/pc/e2e/BUILD.gn b/test/pc/e2e/BUILD.gn index ff3b701c42..67a84ac734 100644 --- a/test/pc/e2e/BUILD.gn +++ b/test/pc/e2e/BUILD.gn @@ -34,6 +34,7 @@ if (!build_with_chromium) { testonly = true deps = [ + ":default_video_quality_analyzer_frames_comparator_test", ":default_video_quality_analyzer_test", ":multi_head_queue_test", ":peer_connection_e2e_smoke_test", @@ -517,6 +518,19 @@ if (!build_with_chromium) { ] } + rtc_library("default_video_quality_analyzer_frames_comparator_test") { + testonly = true + sources = [ "analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc" ] + deps = [ + ":default_video_quality_analyzer_internal", + ":default_video_quality_analyzer_shared", + "../..:test_support", + "../../../api:create_frame_generator", + "../../../api/units:timestamp", + "../../../system_wrappers", + ] + } + rtc_library("multi_head_queue_test") { testonly = true sources = [ "analyzer/video/multi_head_queue_test.cc" ] @@ -651,25 +665,38 @@ if (!build_with_chromium) { # This target contains implementation details of DefaultVideoQualityAnalyzer, # so headers exported by it shouldn't be used in other places. rtc_library("default_video_quality_analyzer_internal") { - visibility = [ ":default_video_quality_analyzer" ] + visibility = [ + ":default_video_quality_analyzer", + ":default_video_quality_analyzer_frames_comparator_test", + ] testonly = true sources = [ "analyzer/video/default_video_quality_analyzer_cpu_measurer.cc", "analyzer/video/default_video_quality_analyzer_cpu_measurer.h", + "analyzer/video/default_video_quality_analyzer_frames_comparator.cc", + "analyzer/video/default_video_quality_analyzer_frames_comparator.h", "analyzer/video/default_video_quality_analyzer_internal_shared_objects.cc", "analyzer/video/default_video_quality_analyzer_internal_shared_objects.h", ] deps = [ ":default_video_quality_analyzer_shared", + "../../../api:array_view", + "../../../api:scoped_refptr", "../../../api/numerics:numerics", "../../../api/units:timestamp", "../../../api/video:video_frame", + "../../../common_video", + "../../../rtc_base:checks", + "../../../rtc_base:rtc_base_approved", "../../../rtc_base:rtc_base_tests_utils", + "../../../rtc_base:rtc_event", "../../../rtc_base:stringutils", "../../../rtc_base:timeutils", "../../../rtc_base/synchronization:mutex", + "../../../rtc_tools:video_quality_analysis", + "../../../system_wrappers:system_wrappers", ] absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc index f3e2344433..04fdaca770 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc @@ -24,6 +24,7 @@ #include "rtc_base/strings/string_builder.h" #include "rtc_base/time_utils.h" #include "rtc_tools/frame_analyzer/video_geometry_aligner.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h" #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" @@ -31,8 +32,6 @@ namespace webrtc { namespace webrtc_pc_e2e { namespace { -constexpr int kMaxActiveComparisons = 10; -constexpr int kFreezeThresholdMs = 150; constexpr int kMicrosPerSecond = 1000000; constexpr int kBitsInByte = 8; @@ -108,7 +107,9 @@ SamplesStatsCounter::StatsSample StatsSample(double value, DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer( webrtc::Clock* clock, DefaultVideoQualityAnalyzerOptions options) - : options_(options), clock_(clock) {} + : options_(options), + clock_(clock), + frames_comparator_(clock, cpu_measurer_, options) {} DefaultVideoQualityAnalyzer::~DefaultVideoQualityAnalyzer() { Stop(); } @@ -118,20 +119,15 @@ void DefaultVideoQualityAnalyzer::Start( rtc::ArrayView peer_names, int max_threads_count) { test_label_ = std::move(test_case_name); - for (int i = 0; i < max_threads_count; i++) { - thread_pool_.push_back(rtc::PlatformThread::SpawnJoinable( - [this] { ProcessComparisons(); }, - "DefaultVideoQualityAnalyzerWorker-" + std::to_string(i))); - } + frames_comparator_.Start(max_threads_count); { - MutexLock lock(&lock_); + MutexLock lock(&mutex_); peers_ = std::make_unique(peer_names); RTC_CHECK(start_time_.IsMinusInfinity()); state_ = State::kActive; start_time_ = Now(); } - cpu_measurer_.StartMeasuringCpuProcessTime(); } uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured( @@ -146,40 +142,20 @@ uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured( size_t peers_count = -1; size_t stream_index; { - MutexLock lock(&lock_); + MutexLock lock(&mutex_); // Create a local copy of `start_time_`, peer's index and total peers count - // to access it under `comparison_lock_` without holding a `lock_` + // to access it without holding a `mutex_` during access to + // `frames_comparator_`. start_time = start_time_; peer_index = peers_->index(peer_name); peers_count = peers_->size(); stream_index = streams_.AddIfAbsent(stream_label); } + // Ensure stats for this stream exists. + frames_comparator_.EnsureStatsForStream(stream_index, peer_index, peers_count, + captured_time, start_time); { - // Ensure stats for this stream exists. - MutexLock lock(&comparison_lock_); - for (size_t i = 0; i < peers_count; ++i) { - if (i == peer_index && !options_.enable_receive_own_stream) { - continue; - } - InternalStatsKey stats_key(stream_index, peer_index, i); - if (stream_stats_.find(stats_key) == stream_stats_.end()) { - stream_stats_.insert({stats_key, StreamStats(captured_time)}); - // Assume that the first freeze was before first stream frame captured. - // This way time before the first freeze would be counted as time - // between freezes. - stream_last_freeze_end_time_.insert({stats_key, start_time}); - } else { - // When we see some `stream_label` for the first time we need to create - // stream stats object for it and set up some states, but we need to do - // it only once and for all receivers, so on the next frame on the same - // `stream_label` we can be sure, that it's already done and we needn't - // to scan though all peers again. - break; - } - } - } - { - MutexLock lock(&lock_); + MutexLock lock(&mutex_); stream_to_sender_[stream_index] = peer_index; frame_counters_.captured++; for (size_t i = 0; i < peers_->size(); ++i) { @@ -215,12 +191,11 @@ uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured( InternalStatsKey key(stream_index, peer_index, i); stream_frame_counters_.at(key).dropped++; - MutexLock lock1(&comparison_lock_); analyzer_stats_.frames_in_flight_left_count.AddSample( StatsSample(captured_frames_in_flight_.size(), Now())); - AddComparison(InternalStatsKey(stream_index, peer_index, i), - it->second.frame(), absl::nullopt, true, - it->second.GetStatsForPeer(i)); + frames_comparator_.AddComparison( + InternalStatsKey(stream_index, peer_index, i), it->second.frame(), + absl::nullopt, true, it->second.GetStatsForPeer(i)); } captured_frames_in_flight_.erase(it); @@ -260,7 +235,7 @@ uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured( void DefaultVideoQualityAnalyzer::OnFramePreEncode( absl::string_view peer_name, const webrtc::VideoFrame& frame) { - MutexLock lock(&lock_); + MutexLock lock(&mutex_); auto it = captured_frames_in_flight_.find(frame.id()); RTC_DCHECK(it != captured_frames_in_flight_.end()) << "Frame id=" << frame.id() << " not found"; @@ -280,7 +255,7 @@ void DefaultVideoQualityAnalyzer::OnFrameEncoded( uint16_t frame_id, const webrtc::EncodedImage& encoded_image, const EncoderStats& stats) { - MutexLock lock(&lock_); + MutexLock lock(&mutex_); auto it = captured_frames_in_flight_.find(frame_id); if (it == captured_frames_in_flight_.end()) { RTC_LOG(WARNING) @@ -325,7 +300,7 @@ void DefaultVideoQualityAnalyzer::OnFramePreDecode( absl::string_view peer_name, uint16_t frame_id, const webrtc::EncodedImage& input_image) { - MutexLock lock(&lock_); + MutexLock lock(&mutex_); size_t peer_index = peers_->index(peer_name); auto it = captured_frames_in_flight_.find(frame_id); @@ -361,7 +336,7 @@ void DefaultVideoQualityAnalyzer::OnFrameDecoded( absl::string_view peer_name, const webrtc::VideoFrame& frame, const DecoderStats& stats) { - MutexLock lock(&lock_); + MutexLock lock(&mutex_); size_t peer_index = peers_->index(peer_name); auto it = captured_frames_in_flight_.find(frame.id()); @@ -391,7 +366,7 @@ void DefaultVideoQualityAnalyzer::OnFrameDecoded( void DefaultVideoQualityAnalyzer::OnFrameRendered( absl::string_view peer_name, const webrtc::VideoFrame& frame) { - MutexLock lock(&lock_); + MutexLock lock(&mutex_); size_t peer_index = peers_->index(peer_name); auto frame_it = captured_frames_in_flight_.find(frame.id()); @@ -442,13 +417,11 @@ void DefaultVideoQualityAnalyzer::OnFrameRendered( absl::optional dropped_frame = dropped_frame_it->second.frame(); dropped_frame_it->second.MarkDropped(peer_index); - { - MutexLock lock1(&comparison_lock_); - analyzer_stats_.frames_in_flight_left_count.AddSample( - StatsSample(captured_frames_in_flight_.size(), Now())); - AddComparison(stats_key, dropped_frame, absl::nullopt, true, - dropped_frame_it->second.GetStatsForPeer(peer_index)); - } + analyzer_stats_.frames_in_flight_left_count.AddSample( + StatsSample(captured_frames_in_flight_.size(), Now())); + frames_comparator_.AddComparison( + stats_key, dropped_frame, absl::nullopt, true, + dropped_frame_it->second.GetStatsForPeer(peer_index)); if (dropped_frame_it->second.HaveAllPeersReceived()) { captured_frames_in_flight_.erase(dropped_frame_it); @@ -463,15 +436,11 @@ void DefaultVideoQualityAnalyzer::OnFrameRendered( } state->SetLastRenderedFrameTime(peer_index, frame_in_flight->rendered_time(peer_index)); - { - MutexLock lock(&comparison_lock_); - stream_stats_.at(stats_key).skipped_between_rendered.AddSample( - StatsSample(dropped_count, Now())); - analyzer_stats_.frames_in_flight_left_count.AddSample( - StatsSample(captured_frames_in_flight_.size(), Now())); - AddComparison(stats_key, captured_frame, frame, false, - frame_in_flight->GetStatsForPeer(peer_index)); - } + analyzer_stats_.frames_in_flight_left_count.AddSample( + StatsSample(captured_frames_in_flight_.size(), Now())); + frames_comparator_.AddComparison( + stats_key, dropped_count, captured_frame, frame, /*dropped=*/false, + frame_in_flight->GetStatsForPeer(peer_index)); if (frame_it->second.HaveAllPeersReceived()) { captured_frames_in_flight_.erase(frame_it); @@ -495,8 +464,7 @@ void DefaultVideoQualityAnalyzer::OnDecoderError(absl::string_view peer_name, void DefaultVideoQualityAnalyzer::RegisterParticipantInCall( absl::string_view peer_name) { - MutexLock lock1(&lock_); - MutexLock lock2(&comparison_lock_); + MutexLock lock(&mutex_); RTC_CHECK(!peers_->HasName(peer_name)); size_t new_peer_index = peers_->AddIfAbsent(peer_name); @@ -504,6 +472,7 @@ void DefaultVideoQualityAnalyzer::RegisterParticipantInCall( // streams exists. Since in flight frames will be sent to the new peer // as well. Sending stats (from this peer to others) will be added by // DefaultVideoQualityAnalyzer::OnFrameCaptured. + std::vector> stream_started_time; for (auto& key_val : stream_to_sender_) { size_t stream_index = key_val.first; size_t sender_peer_index = key_val.second; @@ -528,11 +497,11 @@ void DefaultVideoQualityAnalyzer::RegisterParticipantInCall( // then `counters` will be empty. In such case empty `counters` are ok. stream_frame_counters_.insert({key, std::move(counters)}); - stream_stats_.insert( - {key, - StreamStats(stream_states_.at(stream_index).stream_started_time())}); - stream_last_freeze_end_time_.insert({key, start_time_}); + stream_started_time.push_back( + {key, stream_states_.at(stream_index).stream_started_time()}); } + frames_comparator_.RegisterParticipantInCall(stream_started_time, + start_time_); // Ensure, that frames states are handled correctly // (e.g. dropped frames tracking). for (auto& key_val : stream_states_) { @@ -549,30 +518,19 @@ void DefaultVideoQualityAnalyzer::RegisterParticipantInCall( } void DefaultVideoQualityAnalyzer::Stop() { + std::map last_rendered_frame_times; { - MutexLock lock(&lock_); + MutexLock lock(&mutex_); if (state_ == State::kStopped) { return; } state_ = State::kStopped; - } - cpu_measurer_.StopMeasuringCpuProcessTime(); - comparison_available_event_.Set(); - thread_pool_.clear(); - // Perform final Metrics update. On this place analyzer is stopped and no one - // holds any locks. - { - // Time between freezes. - // Count time since the last freeze to the end of the call as time - // between freezes. - MutexLock lock1(&lock_); - MutexLock lock2(&comparison_lock_); for (auto& state_entry : stream_states_) { const size_t stream_index = state_entry.first; const StreamState& stream_state = state_entry.second; for (size_t i = 0; i < peers_->size(); ++i) { - if (i == static_cast(stream_state.owner())) { + if (i == stream_state.owner() && !options_.enable_receive_own_stream) { continue; } @@ -580,18 +538,31 @@ void DefaultVideoQualityAnalyzer::Stop() { // If there are no freezes in the call we have to report // time_between_freezes_ms as call duration and in such case - // `stream_last_freeze_end_time_` for this stream will be `start_time_`. + // `stream_last_freeze_end_time` for this stream will be `start_time_`. // If there is freeze, then we need add time from last rendered frame // to last freeze end as time between freezes. if (stream_state.last_rendered_frame_time(i)) { - stream_stats_.at(stats_key).time_between_freezes_ms.AddSample( - StatsSample( - stream_state.last_rendered_frame_time(i).value().ms() - - stream_last_freeze_end_time_.at(stats_key).ms(), - Now())); + last_rendered_frame_times.emplace( + stats_key, stream_state.last_rendered_frame_time(i).value()); } } } + } + frames_comparator_.Stop(last_rendered_frame_times); + + // Perform final Metrics update. On this place analyzer is stopped and no one + // holds any locks. + { + MutexLock lock(&mutex_); + FramesComparatorStats frames_comparator_stats = + frames_comparator_.frames_comparator_stats(); + analyzer_stats_.comparisons_queue_size = + frames_comparator_stats.comparisons_queue_size; + analyzer_stats_.comparisons_done = frames_comparator_stats.comparisons_done; + analyzer_stats_.cpu_overloaded_comparisons_done = + frames_comparator_stats.cpu_overloaded_comparisons_done; + analyzer_stats_.memory_overloaded_comparisons_done = + frames_comparator_stats.memory_overloaded_comparisons_done; analyzer_stats_.frames_in_flight_left_count.AddSample( StatsSample(captured_frames_in_flight_.size(), Now())); } @@ -599,7 +570,7 @@ void DefaultVideoQualityAnalyzer::Stop() { } std::string DefaultVideoQualityAnalyzer::GetStreamLabel(uint16_t frame_id) { - MutexLock lock1(&lock_); + MutexLock lock1(&mutex_); auto it = captured_frames_in_flight_.find(frame_id); if (it != captured_frames_in_flight_.end()) { return streams_.name(it->second.stream()); @@ -615,10 +586,9 @@ std::string DefaultVideoQualityAnalyzer::GetStreamLabel(uint16_t frame_id) { } std::set DefaultVideoQualityAnalyzer::GetKnownVideoStreams() const { - MutexLock lock1(&lock_); - MutexLock lock2(&comparison_lock_); + MutexLock lock(&mutex_); std::set out; - for (auto& item : stream_stats_) { + for (auto& item : frames_comparator_.stream_stats()) { RTC_LOG(INFO) << item.first.ToString() << " ==> " << ToStatsKey(item.first).ToString(); out.insert(ToStatsKey(item.first)); @@ -627,13 +597,13 @@ std::set DefaultVideoQualityAnalyzer::GetKnownVideoStreams() const { } const FrameCounters& DefaultVideoQualityAnalyzer::GetGlobalCounters() const { - MutexLock lock(&lock_); + MutexLock lock(&mutex_); return frame_counters_; } std::map DefaultVideoQualityAnalyzer::GetPerStreamCounters() const { - MutexLock lock(&lock_); + MutexLock lock(&mutex_); std::map out; for (auto& item : stream_frame_counters_) { out.emplace(ToStatsKey(item.first), item.second); @@ -642,216 +612,31 @@ DefaultVideoQualityAnalyzer::GetPerStreamCounters() const { } std::map DefaultVideoQualityAnalyzer::GetStats() const { - MutexLock lock1(&lock_); - MutexLock lock2(&comparison_lock_); + MutexLock lock1(&mutex_); std::map out; - for (auto& item : stream_stats_) { + for (auto& item : frames_comparator_.stream_stats()) { out.emplace(ToStatsKey(item.first), item.second); } return out; } AnalyzerStats DefaultVideoQualityAnalyzer::GetAnalyzerStats() const { - MutexLock lock(&comparison_lock_); + MutexLock lock(&mutex_); return analyzer_stats_; } -void DefaultVideoQualityAnalyzer::AddComparison( - InternalStatsKey stats_key, - absl::optional captured, - absl::optional rendered, - bool dropped, - FrameStats frame_stats) { - cpu_measurer_.StartExcludingCpuThreadTime(); - analyzer_stats_.comparisons_queue_size.AddSample( - StatsSample(comparisons_.size(), Now())); - // If there too many computations waiting in the queue, we won't provide - // frames itself to make future computations lighter. - if (comparisons_.size() >= kMaxActiveComparisons) { - comparisons_.emplace_back(std::move(stats_key), absl::nullopt, - absl::nullopt, dropped, std::move(frame_stats), - OverloadReason::kCpu); - } else { - OverloadReason overload_reason = OverloadReason::kNone; - if (!captured && !dropped) { - overload_reason = OverloadReason::kMemory; - } - comparisons_.emplace_back(std::move(stats_key), std::move(captured), - std::move(rendered), dropped, - std::move(frame_stats), overload_reason); - } - comparison_available_event_.Set(); - cpu_measurer_.StopExcludingCpuThreadTime(); -} - -void DefaultVideoQualityAnalyzer::ProcessComparisons() { - while (true) { - // Try to pick next comparison to perform from the queue. - absl::optional comparison = absl::nullopt; - { - MutexLock lock(&comparison_lock_); - if (!comparisons_.empty()) { - comparison = comparisons_.front(); - comparisons_.pop_front(); - if (!comparisons_.empty()) { - comparison_available_event_.Set(); - } - } - } - if (!comparison) { - bool more_frames_expected; - { - // If there are no comparisons and state is stopped => - // no more frames expected. - MutexLock lock(&lock_); - more_frames_expected = state_ != State::kStopped; - } - if (!more_frames_expected) { - comparison_available_event_.Set(); - return; - } - comparison_available_event_.Wait(1000); - continue; - } - - cpu_measurer_.StartExcludingCpuThreadTime(); - ProcessComparison(comparison.value()); - cpu_measurer_.StopExcludingCpuThreadTime(); - } -} - -void DefaultVideoQualityAnalyzer::ProcessComparison( - const FrameComparison& comparison) { - // Perform expensive psnr and ssim calculations while not holding lock. - double psnr = -1.0; - double ssim = -1.0; - if (options_.heavy_metrics_computation_enabled && comparison.captured && - !comparison.dropped) { - rtc::scoped_refptr reference_buffer = - comparison.captured->video_frame_buffer()->ToI420(); - rtc::scoped_refptr test_buffer = - comparison.rendered->video_frame_buffer()->ToI420(); - if (options_.adjust_cropping_before_comparing_frames) { - test_buffer = - ScaleVideoFrameBuffer(*test_buffer.get(), reference_buffer->width(), - reference_buffer->height()); - reference_buffer = test::AdjustCropping(reference_buffer, test_buffer); - } - psnr = I420PSNR(*reference_buffer.get(), *test_buffer.get()); - ssim = I420SSIM(*reference_buffer.get(), *test_buffer.get()); - } - - const FrameStats& frame_stats = comparison.frame_stats; - - MutexLock lock(&comparison_lock_); - auto stats_it = stream_stats_.find(comparison.stats_key); - RTC_CHECK(stats_it != stream_stats_.end()) << comparison.stats_key.ToString(); - StreamStats* stats = &stats_it->second; - analyzer_stats_.comparisons_done++; - if (comparison.overload_reason == OverloadReason::kCpu) { - analyzer_stats_.cpu_overloaded_comparisons_done++; - } else if (comparison.overload_reason == OverloadReason::kMemory) { - analyzer_stats_.memory_overloaded_comparisons_done++; - } - if (psnr > 0) { - stats->psnr.AddSample(StatsSample(psnr, frame_stats.rendered_time)); - } - if (ssim > 0) { - stats->ssim.AddSample(StatsSample(ssim, frame_stats.received_time)); - } - if (frame_stats.encoded_time.IsFinite()) { - stats->encode_time_ms.AddSample(StatsSample( - (frame_stats.encoded_time - frame_stats.pre_encode_time).ms(), - frame_stats.encoded_time)); - stats->encode_frame_rate.AddEvent(frame_stats.encoded_time); - stats->total_encoded_images_payload += frame_stats.encoded_image_size; - stats->target_encode_bitrate.AddSample(StatsSample( - frame_stats.target_encode_bitrate, frame_stats.encoded_time)); - } else { - if (frame_stats.pre_encode_time.IsFinite()) { - stats->dropped_by_encoder++; - } else { - stats->dropped_before_encoder++; - } - } - // Next stats can be calculated only if frame was received on remote side. - if (!comparison.dropped) { - stats->resolution_of_rendered_frame.AddSample( - StatsSample(*comparison.frame_stats.rendered_frame_width * - *comparison.frame_stats.rendered_frame_height, - frame_stats.rendered_time)); - stats->transport_time_ms.AddSample(StatsSample( - (frame_stats.decode_start_time - frame_stats.encoded_time).ms(), - frame_stats.received_time)); - stats->total_delay_incl_transport_ms.AddSample(StatsSample( - (frame_stats.rendered_time - frame_stats.captured_time).ms(), - frame_stats.received_time)); - stats->decode_time_ms.AddSample(StatsSample( - (frame_stats.decode_end_time - frame_stats.decode_start_time).ms(), - frame_stats.decode_end_time)); - stats->receive_to_render_time_ms.AddSample(StatsSample( - (frame_stats.rendered_time - frame_stats.received_time).ms(), - frame_stats.rendered_time)); - - if (frame_stats.prev_frame_rendered_time.IsFinite()) { - TimeDelta time_between_rendered_frames = - frame_stats.rendered_time - frame_stats.prev_frame_rendered_time; - stats->time_between_rendered_frames_ms.AddSample(StatsSample( - time_between_rendered_frames.ms(), frame_stats.rendered_time)); - double average_time_between_rendered_frames_ms = - stats->time_between_rendered_frames_ms.GetAverage(); - if (time_between_rendered_frames.ms() > - std::max(kFreezeThresholdMs + average_time_between_rendered_frames_ms, - 3 * average_time_between_rendered_frames_ms)) { - stats->freeze_time_ms.AddSample(StatsSample( - time_between_rendered_frames.ms(), frame_stats.rendered_time)); - auto freeze_end_it = - stream_last_freeze_end_time_.find(comparison.stats_key); - RTC_DCHECK(freeze_end_it != stream_last_freeze_end_time_.end()); - stats->time_between_freezes_ms.AddSample(StatsSample( - (frame_stats.prev_frame_rendered_time - freeze_end_it->second).ms(), - frame_stats.rendered_time)); - freeze_end_it->second = frame_stats.rendered_time; - } - } - } - // Compute stream codec info. - if (frame_stats.used_encoder.has_value()) { - if (stats->encoders.empty() || stats->encoders.back().codec_name != - frame_stats.used_encoder->codec_name) { - stats->encoders.push_back(*frame_stats.used_encoder); - } - stats->encoders.back().last_frame_id = - frame_stats.used_encoder->last_frame_id; - stats->encoders.back().switched_from_at = - frame_stats.used_encoder->switched_from_at; - } - - if (frame_stats.used_decoder.has_value()) { - if (stats->decoders.empty() || stats->decoders.back().codec_name != - frame_stats.used_decoder->codec_name) { - stats->decoders.push_back(*frame_stats.used_decoder); - } - stats->decoders.back().last_frame_id = - frame_stats.used_decoder->last_frame_id; - stats->decoders.back().switched_from_at = - frame_stats.used_decoder->switched_from_at; - } -} - void DefaultVideoQualityAnalyzer::ReportResults() { using ::webrtc::test::ImproveDirection; - MutexLock lock1(&lock_); - MutexLock lock2(&comparison_lock_); - for (auto& item : stream_stats_) { + MutexLock lock(&mutex_); + for (auto& item : frames_comparator_.stream_stats()) { ReportResults(GetTestCaseName(StatsKeyToMetricName(ToStatsKey(item.first))), item.second, stream_frame_counters_.at(item.first)); } test::PrintResult("cpu_usage", "", test_label_.c_str(), GetCpuUsagePercent(), "%", false, ImproveDirection::kSmallerIsBetter); LogFrameCounters("Global", frame_counters_); - for (auto& item : stream_stats_) { + for (auto& item : frames_comparator_.stream_stats()) { LogFrameCounters(ToStatsKey(item.first).ToString(), stream_frame_counters_.at(item.first)); LogStreamInternalStats(ToStatsKey(item.first).ToString(), item.second, diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h index d388b1b187..57b202e894 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h @@ -30,6 +30,7 @@ #include "rtc_base/synchronization/mutex.h" #include "system_wrappers/include/clock.h" #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h" #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" #include "test/pc/e2e/analyzer/video/multi_head_queue.h" @@ -38,33 +39,11 @@ namespace webrtc { namespace webrtc_pc_e2e { -struct DefaultVideoQualityAnalyzerOptions { - // Tells DefaultVideoQualityAnalyzer if heavy metrics like PSNR and SSIM have - // to be computed or not. - bool heavy_metrics_computation_enabled = true; - // If true DefaultVideoQualityAnalyzer will try to adjust frames before - // computing PSNR and SSIM for them. In some cases picture may be shifted by - // a few pixels after the encode/decode step. Those difference is invisible - // for a human eye, but it affects the metrics. So the adjustment is used to - // get metrics that are closer to how human persepts the video. This feature - // significantly slows down the comparison, so turn it on only when it is - // needed. - bool adjust_cropping_before_comparing_frames = false; - // Amount of frames that are queued in the DefaultVideoQualityAnalyzer from - // the point they were captured to the point they were rendered on all - // receivers per stream. - size_t max_frames_in_flight_per_stream_count = - kDefaultMaxFramesInFlightPerStream; - // If true, the analyzer will expect peers to receive their own video streams. - bool enable_receive_own_stream = false; -}; - class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { public: explicit DefaultVideoQualityAnalyzer( webrtc::Clock* clock, - DefaultVideoQualityAnalyzerOptions options = - DefaultVideoQualityAnalyzerOptions()); + DefaultVideoQualityAnalyzerOptions options = {}); ~DefaultVideoQualityAnalyzer() override; void Start(std::string test_case_name, @@ -307,21 +286,12 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { std::map index_; }; - void AddComparison(InternalStatsKey stats_key, - absl::optional captured, - absl::optional rendered, - bool dropped, - FrameStats frame_stats) - RTC_EXCLUSIVE_LOCKS_REQUIRED(comparison_lock_); - static void ProcessComparisonsThread(void* obj); - void ProcessComparisons(); - void ProcessComparison(const FrameComparison& comparison); // Report results for all metrics for all streams. void ReportResults(); void ReportResults(const std::string& test_case_name, const StreamStats& stats, const FrameCounters& frame_counters) - RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_); + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); // Report result for single metric for specified stream. static void ReportResult(const std::string& metric_name, const std::string& test_case_name, @@ -333,28 +303,27 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { std::string GetTestCaseName(const std::string& stream_label) const; Timestamp Now(); StatsKey ToStatsKey(const InternalStatsKey& key) const - RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_); + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); // Returns string representation of stats key for metrics naming. Used for // backward compatibility by metrics naming for 2 peers cases. std::string StatsKeyToMetricName(const StatsKey& key) const - RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_); + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); - // TODO(titovartem) restore const when old constructor will be removed. - DefaultVideoQualityAnalyzerOptions options_; + const DefaultVideoQualityAnalyzerOptions options_; webrtc::Clock* const clock_; std::atomic next_frame_id_{0}; std::string test_label_; - mutable Mutex lock_; - std::unique_ptr peers_ RTC_GUARDED_BY(lock_); - State state_ RTC_GUARDED_BY(lock_) = State::kNew; - Timestamp start_time_ RTC_GUARDED_BY(lock_) = Timestamp::MinusInfinity(); + mutable Mutex mutex_; + std::unique_ptr peers_ RTC_GUARDED_BY(mutex_); + State state_ RTC_GUARDED_BY(mutex_) = State::kNew; + Timestamp start_time_ RTC_GUARDED_BY(mutex_) = Timestamp::MinusInfinity(); // Mapping from stream label to unique size_t value to use in stats and avoid // extra string copying. - NamesCollection streams_ RTC_GUARDED_BY(lock_); + NamesCollection streams_ RTC_GUARDED_BY(mutex_); // Frames that were captured by all streams and still aren't rendered on - // receviers or deemed dropped. Frame with id X can be removed from this map + // receivers or deemed dropped. Frame with id X can be removed from this map // if: // 1. The frame with id X was received in OnFrameRendered by all expected // receivers. @@ -365,36 +334,27 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { // oldest frame id in this stream. In such case only the frame content // will be removed, but the map entry will be preserved. std::map captured_frames_in_flight_ - RTC_GUARDED_BY(lock_); + RTC_GUARDED_BY(mutex_); // Global frames count for all video streams. - FrameCounters frame_counters_ RTC_GUARDED_BY(lock_); + FrameCounters frame_counters_ RTC_GUARDED_BY(mutex_); // Frame counters per each stream per each receiver. std::map stream_frame_counters_ - RTC_GUARDED_BY(lock_); + RTC_GUARDED_BY(mutex_); // Map from stream index in `streams_` to its StreamState. - std::map stream_states_ RTC_GUARDED_BY(lock_); + std::map stream_states_ RTC_GUARDED_BY(mutex_); // Map from stream index in `streams_` to sender peer index in `peers_`. - std::map stream_to_sender_ RTC_GUARDED_BY(lock_); + std::map stream_to_sender_ RTC_GUARDED_BY(mutex_); // Stores history mapping between stream index in `streams_` and frame ids. // Updated when frame id overlap. It required to properly return stream label // after 1st frame from simulcast streams was already rendered and last is // still encoding. std::map> stream_to_frame_id_history_ - RTC_GUARDED_BY(lock_); - - mutable Mutex comparison_lock_; - std::map stream_stats_ - RTC_GUARDED_BY(comparison_lock_); - std::map stream_last_freeze_end_time_ - RTC_GUARDED_BY(comparison_lock_); - std::deque comparisons_ RTC_GUARDED_BY(comparison_lock_); - AnalyzerStats analyzer_stats_ RTC_GUARDED_BY(comparison_lock_); - - std::vector thread_pool_; - rtc::Event comparison_available_event_; + RTC_GUARDED_BY(mutex_); + AnalyzerStats analyzer_stats_ RTC_GUARDED_BY(mutex_); DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer_; + DefaultVideoQualityAnalyzerFramesComparator frames_comparator_; }; } // namespace webrtc_pc_e2e diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc new file mode 100644 index 0000000000..0f5f3bff54 --- /dev/null +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc @@ -0,0 +1,359 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h" + +#include +#include +#include + +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/scoped_refptr.h" +#include "api/video/i420_buffer.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "rtc_base/checks.h" +#include "rtc_base/platform_thread.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_tools/frame_analyzer/video_geometry_aligner.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" + +namespace webrtc { +namespace { + +constexpr int kFreezeThresholdMs = 150; +constexpr int kMaxActiveComparisons = 10; + +SamplesStatsCounter::StatsSample StatsSample(double value, + Timestamp sampling_time) { + return SamplesStatsCounter::StatsSample{value, sampling_time}; +} + +} // namespace + +void DefaultVideoQualityAnalyzerFramesComparator::Start(int max_threads_count) { + for (int i = 0; i < max_threads_count; i++) { + thread_pool_.push_back(rtc::PlatformThread::SpawnJoinable( + [this] { ProcessComparisons(); }, + "DefaultVideoQualityAnalyzerFramesComparator-" + std::to_string(i))); + } + { + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kNew) << "Frames comparator is already started"; + state_ = State::kActive; + } + cpu_measurer_.StartMeasuringCpuProcessTime(); +} + +void DefaultVideoQualityAnalyzerFramesComparator::Stop( + const std::map& last_rendered_frame_times) { + { + MutexLock lock(&mutex_); + if (state_ == State::kStopped) { + return; + } + RTC_CHECK_EQ(state_, State::kActive) + << "Frames comparator has to be started before it will be used"; + state_ = State::kStopped; + } + cpu_measurer_.StopMeasuringCpuProcessTime(); + comparison_available_event_.Set(); + thread_pool_.clear(); + + { + MutexLock lock(&mutex_); + // Perform final Metrics update. On this place analyzer is stopped and no + // one holds any locks. + + // Time between freezes. + // Count time since the last freeze to the end of the call as time + // between freezes. + for (auto& entry : last_rendered_frame_times) { + const InternalStatsKey& stats_key = entry.first; + const Timestamp& last_rendered_frame_time = entry.second; + + // If there are no freezes in the call we have to report + // time_between_freezes_ms as call duration and in such case + // `last_rendered_frame_time` for this stream will be stream start time. + // If there is freeze, then we need add time from last rendered frame + // to last freeze end as time between freezes. + stream_stats_.at(stats_key).time_between_freezes_ms.AddSample( + StatsSample(last_rendered_frame_time.ms() - + stream_last_freeze_end_time_.at(stats_key).ms(), + Now())); + } + } +} + +void DefaultVideoQualityAnalyzerFramesComparator::EnsureStatsForStream( + size_t stream_index, + size_t sender_peer_index, + size_t peers_count, + Timestamp captured_time, + Timestamp start_time) { + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "Frames comparator has to be started before it will be used"; + + for (size_t i = 0; i < peers_count; ++i) { + if (i == sender_peer_index && !options_.enable_receive_own_stream) { + continue; + } + InternalStatsKey stats_key(stream_index, sender_peer_index, i); + if (stream_stats_.find(stats_key) == stream_stats_.end()) { + stream_stats_.insert( + {stats_key, webrtc_pc_e2e::StreamStats(captured_time)}); + // Assume that the first freeze was before first stream frame captured. + // This way time before the first freeze would be counted as time + // between freezes. + stream_last_freeze_end_time_.insert({stats_key, start_time}); + } else { + // When we see some `stream_label` for the first time we need to create + // stream stats object for it and set up some states, but we need to do + // it only once and for all receivers, so on the next frame on the same + // `stream_label` we can be sure, that it's already done and we needn't + // to scan though all peers again. + break; + } + } +} + +void DefaultVideoQualityAnalyzerFramesComparator::RegisterParticipantInCall( + rtc::ArrayView> stream_started_time, + Timestamp start_time) { + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "Frames comparator has to be started before it will be used"; + + for (const std::pair& pair : + stream_started_time) { + stream_stats_.insert({pair.first, webrtc_pc_e2e::StreamStats(pair.second)}); + stream_last_freeze_end_time_.insert({pair.first, start_time}); + } +} + +void DefaultVideoQualityAnalyzerFramesComparator::AddComparison( + InternalStatsKey stats_key, + absl::optional captured, + absl::optional rendered, + bool dropped, + FrameStats frame_stats) { + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "Frames comparator has to be started before it will be used"; + AddComparisonInternal(std::move(stats_key), std::move(captured), + std::move(rendered), dropped, std::move(frame_stats)); +} + +void DefaultVideoQualityAnalyzerFramesComparator::AddComparison( + InternalStatsKey stats_key, + int skipped_between_rendered, + absl::optional captured, + absl::optional rendered, + bool dropped, + FrameStats frame_stats) { + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "Frames comparator has to be started before it will be used"; + stream_stats_.at(stats_key).skipped_between_rendered.AddSample( + StatsSample(skipped_between_rendered, Now())); + AddComparisonInternal(std::move(stats_key), std::move(captured), + std::move(rendered), dropped, std::move(frame_stats)); +} + +void DefaultVideoQualityAnalyzerFramesComparator::AddComparisonInternal( + InternalStatsKey stats_key, + absl::optional captured, + absl::optional rendered, + bool dropped, + FrameStats frame_stats) { + cpu_measurer_.StartExcludingCpuThreadTime(); + frames_comparator_stats_.comparisons_queue_size.AddSample( + StatsSample(comparisons_.size(), Now())); + // If there too many computations waiting in the queue, we won't provide + // frames itself to make future computations lighter. + if (comparisons_.size() >= kMaxActiveComparisons) { + comparisons_.emplace_back(std::move(stats_key), absl::nullopt, + absl::nullopt, dropped, std::move(frame_stats), + OverloadReason::kCpu); + } else { + OverloadReason overload_reason = OverloadReason::kNone; + if (!captured && !dropped) { + overload_reason = OverloadReason::kMemory; + } + comparisons_.emplace_back(std::move(stats_key), std::move(captured), + std::move(rendered), dropped, + std::move(frame_stats), overload_reason); + } + comparison_available_event_.Set(); + cpu_measurer_.StopExcludingCpuThreadTime(); +} + +void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparisons() { + while (true) { + // Try to pick next comparison to perform from the queue. + absl::optional comparison = absl::nullopt; + { + MutexLock lock(&mutex_); + if (!comparisons_.empty()) { + comparison = comparisons_.front(); + comparisons_.pop_front(); + if (!comparisons_.empty()) { + comparison_available_event_.Set(); + } + } + } + if (!comparison) { + bool more_frames_expected; + { + // If there are no comparisons and state is stopped => + // no more frames expected. + MutexLock lock(&mutex_); + more_frames_expected = state_ != State::kStopped; + } + if (!more_frames_expected) { + comparison_available_event_.Set(); + return; + } + comparison_available_event_.Wait(1000); + continue; + } + + cpu_measurer_.StartExcludingCpuThreadTime(); + ProcessComparison(comparison.value()); + cpu_measurer_.StopExcludingCpuThreadTime(); + } +} + +void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparison( + const FrameComparison& comparison) { + // Perform expensive psnr and ssim calculations while not holding lock. + double psnr = -1.0; + double ssim = -1.0; + if (options_.heavy_metrics_computation_enabled && comparison.captured && + !comparison.dropped) { + rtc::scoped_refptr reference_buffer = + comparison.captured->video_frame_buffer()->ToI420(); + rtc::scoped_refptr test_buffer = + comparison.rendered->video_frame_buffer()->ToI420(); + if (options_.adjust_cropping_before_comparing_frames) { + test_buffer = + ScaleVideoFrameBuffer(*test_buffer.get(), reference_buffer->width(), + reference_buffer->height()); + reference_buffer = test::AdjustCropping(reference_buffer, test_buffer); + } + psnr = I420PSNR(*reference_buffer.get(), *test_buffer.get()); + ssim = I420SSIM(*reference_buffer.get(), *test_buffer.get()); + } + + const FrameStats& frame_stats = comparison.frame_stats; + + MutexLock lock(&mutex_); + auto stats_it = stream_stats_.find(comparison.stats_key); + RTC_CHECK(stats_it != stream_stats_.end()) << comparison.stats_key.ToString(); + webrtc_pc_e2e::StreamStats* stats = &stats_it->second; + frames_comparator_stats_.comparisons_done++; + if (comparison.overload_reason == OverloadReason::kCpu) { + frames_comparator_stats_.cpu_overloaded_comparisons_done++; + } else if (comparison.overload_reason == OverloadReason::kMemory) { + frames_comparator_stats_.memory_overloaded_comparisons_done++; + } + if (psnr > 0) { + stats->psnr.AddSample(StatsSample(psnr, frame_stats.rendered_time)); + } + if (ssim > 0) { + stats->ssim.AddSample(StatsSample(ssim, frame_stats.received_time)); + } + if (frame_stats.encoded_time.IsFinite()) { + stats->encode_time_ms.AddSample(StatsSample( + (frame_stats.encoded_time - frame_stats.pre_encode_time).ms(), + frame_stats.encoded_time)); + stats->encode_frame_rate.AddEvent(frame_stats.encoded_time); + stats->total_encoded_images_payload += frame_stats.encoded_image_size; + stats->target_encode_bitrate.AddSample(StatsSample( + frame_stats.target_encode_bitrate, frame_stats.encoded_time)); + } else { + if (frame_stats.pre_encode_time.IsFinite()) { + stats->dropped_by_encoder++; + } else { + stats->dropped_before_encoder++; + } + } + // Next stats can be calculated only if frame was received on remote side. + if (!comparison.dropped) { + stats->resolution_of_rendered_frame.AddSample( + StatsSample(*comparison.frame_stats.rendered_frame_width * + *comparison.frame_stats.rendered_frame_height, + frame_stats.rendered_time)); + stats->transport_time_ms.AddSample(StatsSample( + (frame_stats.decode_start_time - frame_stats.encoded_time).ms(), + frame_stats.received_time)); + stats->total_delay_incl_transport_ms.AddSample(StatsSample( + (frame_stats.rendered_time - frame_stats.captured_time).ms(), + frame_stats.received_time)); + stats->decode_time_ms.AddSample(StatsSample( + (frame_stats.decode_end_time - frame_stats.decode_start_time).ms(), + frame_stats.decode_end_time)); + stats->receive_to_render_time_ms.AddSample(StatsSample( + (frame_stats.rendered_time - frame_stats.received_time).ms(), + frame_stats.rendered_time)); + + if (frame_stats.prev_frame_rendered_time.IsFinite()) { + TimeDelta time_between_rendered_frames = + frame_stats.rendered_time - frame_stats.prev_frame_rendered_time; + stats->time_between_rendered_frames_ms.AddSample(StatsSample( + time_between_rendered_frames.ms(), frame_stats.rendered_time)); + double average_time_between_rendered_frames_ms = + stats->time_between_rendered_frames_ms.GetAverage(); + if (time_between_rendered_frames.ms() > + std::max(kFreezeThresholdMs + average_time_between_rendered_frames_ms, + 3 * average_time_between_rendered_frames_ms)) { + stats->freeze_time_ms.AddSample(StatsSample( + time_between_rendered_frames.ms(), frame_stats.rendered_time)); + auto freeze_end_it = + stream_last_freeze_end_time_.find(comparison.stats_key); + RTC_DCHECK(freeze_end_it != stream_last_freeze_end_time_.end()); + stats->time_between_freezes_ms.AddSample(StatsSample( + (frame_stats.prev_frame_rendered_time - freeze_end_it->second).ms(), + frame_stats.rendered_time)); + freeze_end_it->second = frame_stats.rendered_time; + } + } + } + // Compute stream codec info. + if (frame_stats.used_encoder.has_value()) { + if (stats->encoders.empty() || stats->encoders.back().codec_name != + frame_stats.used_encoder->codec_name) { + stats->encoders.push_back(*frame_stats.used_encoder); + } + stats->encoders.back().last_frame_id = + frame_stats.used_encoder->last_frame_id; + stats->encoders.back().switched_from_at = + frame_stats.used_encoder->switched_from_at; + } + + if (frame_stats.used_decoder.has_value()) { + if (stats->decoders.empty() || stats->decoders.back().codec_name != + frame_stats.used_decoder->codec_name) { + stats->decoders.push_back(*frame_stats.used_decoder); + } + stats->decoders.back().last_frame_id = + frame_stats.used_decoder->last_frame_id; + stats->decoders.back().switched_from_at = + frame_stats.used_decoder->switched_from_at; + } +} + +Timestamp DefaultVideoQualityAnalyzerFramesComparator::Now() { + return clock_->CurrentTime(); +} + +} // namespace webrtc diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h new file mode 100644 index 0000000000..4a2ac9dfe2 --- /dev/null +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAMES_COMPARATOR_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAMES_COMPARATOR_H_ + +#include +#include +#include +#include + +#include "api/array_view.h" +#include "rtc_base/event.h" +#include "rtc_base/platform_thread.h" +#include "rtc_base/synchronization/mutex.h" +#include "system_wrappers/include/clock.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" + +namespace webrtc { + +struct FramesComparatorStats { + // Size of analyzer internal comparisons queue, measured when new element + // id added to the queue. + SamplesStatsCounter comparisons_queue_size; + // Number of performed comparisons of 2 video frames from captured and + // rendered streams. + int64_t comparisons_done = 0; + // Number of cpu overloaded comparisons. Comparison is cpu overloaded if it is + // queued when there are too many not processed comparisons in the queue. + // Overloaded comparison doesn't include metrics like SSIM and PSNR that + // require heavy computations. + int64_t cpu_overloaded_comparisons_done = 0; + // Number of memory overloaded comparisons. Comparison is memory overloaded if + // it is queued when its captured frame was already removed due to high memory + // usage for that video stream. + int64_t memory_overloaded_comparisons_done = 0; +}; + +// Performs comparisons of added frames and tracks frames related statistics. +// This class is thread safe. +class DefaultVideoQualityAnalyzerFramesComparator { + public: + // Creates frames comparator. + // Frames comparator doesn't use `options.enable_receive_own_stream` for any + // purposes, because it's unrelated to its functionality. + DefaultVideoQualityAnalyzerFramesComparator( + webrtc::Clock* clock, + DefaultVideoQualityAnalyzerCpuMeasurer& cpu_measurer, + webrtc_pc_e2e::DefaultVideoQualityAnalyzerOptions options = {}) + : options_(options), clock_(clock), cpu_measurer_(cpu_measurer) {} + ~DefaultVideoQualityAnalyzerFramesComparator() { Stop({}); } + + // Starts frames comparator. This method must be invoked before calling + // any other method on this object. + void Start(int max_threads_count); + // Stops frames comparator. This method will block until all added frame + // comparisons will be processed. After `Stop()` is invoked no more new + // comparisons can be added to this frames comparator. + // + // `last_rendered_frame_time` contains timestamps of last rendered frame for + // each (stream, sender, receiver) tuple to properly update time between + // freezes: it has include time from the last freeze until and of call. + void Stop( + const std::map& last_rendered_frame_times); + + // Ensures that stream `stream_index` has stats objects created for all + // potential receivers. This method must be called before adding any + // frames comparison for that stream. + void EnsureStatsForStream(size_t stream_index, + size_t sender_peer_index, + size_t peers_count, + Timestamp captured_time, + Timestamp start_time); + // Ensures that newly added participant will have stream stats objects created + // for all streams which they can receive. This method must be called before + // any frames comparison will be added for the newly added participant. + // + // `stream_started_time` - start time of each stream for which stats object + // has to be created. + // `start_time` - call start time. + void RegisterParticipantInCall( + rtc::ArrayView> + stream_started_time, + Timestamp start_time); + + void AddComparison(InternalStatsKey stats_key, + absl::optional captured, + absl::optional rendered, + bool dropped, + FrameStats frame_stats); + // `skipped_between_rendered` - amount of frames dropped on this stream before + // last received frame and current frame. + void AddComparison(InternalStatsKey stats_key, + int skipped_between_rendered, + absl::optional captured, + absl::optional rendered, + bool dropped, + FrameStats frame_stats); + + std::map stream_stats() const { + MutexLock lock(&mutex_); + return stream_stats_; + } + FramesComparatorStats frames_comparator_stats() const { + MutexLock lock(&mutex_); + return frames_comparator_stats_; + } + + private: + enum State { kNew, kActive, kStopped }; + + void AddComparisonInternal(InternalStatsKey stats_key, + absl::optional captured, + absl::optional rendered, + bool dropped, + FrameStats frame_stats) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + void ProcessComparisons(); + void ProcessComparison(const FrameComparison& comparison); + Timestamp Now(); + + const webrtc_pc_e2e::DefaultVideoQualityAnalyzerOptions options_; + webrtc::Clock* const clock_; + DefaultVideoQualityAnalyzerCpuMeasurer& cpu_measurer_; + + mutable Mutex mutex_; + State state_ RTC_GUARDED_BY(mutex_) = State::kNew; + std::map stream_stats_ + RTC_GUARDED_BY(mutex_); + std::map stream_last_freeze_end_time_ + RTC_GUARDED_BY(mutex_); + std::deque comparisons_ RTC_GUARDED_BY(mutex_); + FramesComparatorStats frames_comparator_stats_ RTC_GUARDED_BY(mutex_); + + std::vector thread_pool_; + rtc::Event comparison_available_event_; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAMES_COMPARATOR_H_ diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc new file mode 100644 index 0000000000..9a63cd0b20 --- /dev/null +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h" + +#include + +#include "api/test/create_frame_generator.h" +#include "api/units/timestamp.h" +#include "system_wrappers/include/clock.h" +#include "system_wrappers/include/sleep.h" +#include "test/gtest.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" + +namespace webrtc { +namespace { + +using StatsSample = ::webrtc::SamplesStatsCounter::StatsSample; + +constexpr int kMaxFramesInFlightPerStream = 10; + +webrtc_pc_e2e::DefaultVideoQualityAnalyzerOptions AnalyzerOptionsForTest() { + webrtc_pc_e2e::DefaultVideoQualityAnalyzerOptions options; + options.heavy_metrics_computation_enabled = false; + options.adjust_cropping_before_comparing_frames = false; + options.max_frames_in_flight_per_stream_count = kMaxFramesInFlightPerStream; + return options; +} + +webrtc_pc_e2e::StreamCodecInfo Vp8CodecForOneFrame(uint16_t frame_id, + Timestamp time) { + webrtc_pc_e2e::StreamCodecInfo info; + info.codec_name = "VP8"; + info.first_frame_id = frame_id; + info.last_frame_id = frame_id; + info.switched_on_at = time; + info.switched_from_at = time; + return info; +} + +FrameStats FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame( + Timestamp captured_time) { + FrameStats frame_stats(captured_time); + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); + frame_stats.received_time = captured_time + TimeDelta::Millis(30); + frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40); + frame_stats.decode_end_time = captured_time + TimeDelta::Millis(50); + frame_stats.rendered_time = captured_time + TimeDelta::Millis(60); + frame_stats.used_encoder = Vp8CodecForOneFrame(1, frame_stats.encoded_time); + frame_stats.used_encoder = + Vp8CodecForOneFrame(1, frame_stats.decode_end_time); + frame_stats.rendered_frame_width = 10; + frame_stats.rendered_frame_height = 10; + return frame_stats; +} + +double GetFirstOrDie(const SamplesStatsCounter& counter) { + EXPECT_TRUE(!counter.IsEmpty()) << "Counter has to be not empty"; + return counter.GetSamples()[0]; +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + StatsPresentedAfterAddingOneComparison) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, AnalyzerOptionsForTest()); + + Timestamp stream_start_time = Clock::GetRealTimeClock()->CurrentTime(); + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + size_t peers_count = 2; + InternalStatsKey stats_key(stream, sender, receiver); + + FrameStats frame_stats = + FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame(stream_start_time); + + comparator.Start(1); + comparator.EnsureStatsForStream(stream, sender, peers_count, + stream_start_time, stream_start_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, /*dropped=*/false, + frame_stats); + comparator.Stop({}); + + std::map stats = + comparator.stream_stats(); + EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.at(stats_key).transport_time_ms), 20.0); + EXPECT_DOUBLE_EQ( + GetFirstOrDie(stats.at(stats_key).total_delay_incl_transport_ms), 60.0); + EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.at(stats_key).encode_time_ms), 10.0); + EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.at(stats_key).decode_time_ms), 10.0); + EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.at(stats_key).receive_to_render_time_ms), + 30.0); + EXPECT_DOUBLE_EQ( + GetFirstOrDie(stats.at(stats_key).resolution_of_rendered_frame), 100.0); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + MultiFrameStatsPresentedAfterAddingTwoComparisonWith10msDelay) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, AnalyzerOptionsForTest()); + + Timestamp stream_start_time = Clock::GetRealTimeClock()->CurrentTime(); + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + size_t peers_count = 2; + InternalStatsKey stats_key(stream, sender, receiver); + + FrameStats frame_stats1 = + FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame(stream_start_time); + FrameStats frame_stats2 = FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame( + stream_start_time + TimeDelta::Millis(15)); + frame_stats2.prev_frame_rendered_time = frame_stats1.rendered_time; + + comparator.Start(1); + comparator.EnsureStatsForStream(stream, sender, peers_count, + stream_start_time, stream_start_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, /*dropped=*/false, + frame_stats1); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, /*dropped=*/false, + frame_stats2); + comparator.Stop({}); + + std::map stats = + comparator.stream_stats(); + EXPECT_DOUBLE_EQ( + GetFirstOrDie(stats.at(stats_key).time_between_rendered_frames_ms), 15.0); + EXPECT_DOUBLE_EQ(stats.at(stats_key).encode_frame_rate.GetEventsPerSecond(), + 2.0 / 15 * 1000); +} + +} // namespace +} // namespace webrtc diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.cc index d82309174f..1149ac1fdc 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.cc @@ -26,7 +26,7 @@ void RateCounter::AddEvent(Timestamp event_time) { event_first_time_ = event_time; } event_last_time_ = event_time; - event_count_++; + events_count_++; } double RateCounter::GetEventsPerSecond() const { @@ -34,7 +34,7 @@ double RateCounter::GetEventsPerSecond() const { // Divide on us and multiply on kMicrosPerSecond to correctly process cases // where there were too small amount of events, so difference is less then 1 // sec. We can use us here, because Timestamp has us resolution. - return static_cast(event_count_) / + return static_cast(events_count_) / (event_last_time_ - event_first_time_).us() * kMicrosPerSecond; } diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h index b5fdedd629..4c90961a6f 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h @@ -39,7 +39,7 @@ class RateCounter { private: Timestamp event_first_time_ = Timestamp::MinusInfinity(); Timestamp event_last_time_ = Timestamp::MinusInfinity(); - int64_t event_count_ = 0; + int64_t events_count_ = 0; }; struct FrameCounters { @@ -165,6 +165,27 @@ struct StatsKey { bool operator<(const StatsKey& a, const StatsKey& b); bool operator==(const StatsKey& a, const StatsKey& b); +struct DefaultVideoQualityAnalyzerOptions { + // Tells DefaultVideoQualityAnalyzer if heavy metrics like PSNR and SSIM have + // to be computed or not. + bool heavy_metrics_computation_enabled = true; + // If true DefaultVideoQualityAnalyzer will try to adjust frames before + // computing PSNR and SSIM for them. In some cases picture may be shifted by + // a few pixels after the encode/decode step. Those difference is invisible + // for a human eye, but it affects the metrics. So the adjustment is used to + // get metrics that are closer to how human perceive the video. This feature + // significantly slows down the comparison, so turn it on only when it is + // needed. + bool adjust_cropping_before_comparing_frames = false; + // Amount of frames that are queued in the DefaultVideoQualityAnalyzer from + // the point they were captured to the point they were rendered on all + // receivers per stream. + size_t max_frames_in_flight_per_stream_count = + kDefaultMaxFramesInFlightPerStream; + // If true, the analyzer will expect peers to receive their own video streams. + bool enable_receive_own_stream = false; +}; + } // namespace webrtc_pc_e2e } // namespace webrtc