From c028df05acdfcd8a3cde91206c4e49afd29aa937 Mon Sep 17 00:00:00 2001 From: Artem Titov Date: Fri, 28 Feb 2020 16:24:49 +0100 Subject: [PATCH] Extract BWE stats collection. Extract collection of BWE stats from DefaultVideoQualityAnalyzer to separate class to prepare for migration on new GetStats API and simplify quality analyzer. Bug: webrtc:11381 Change-Id: I0e7e2d7e40b467d7a42633a72a7ffc49ebcb0237 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/169444 Commit-Queue: Artem Titov Reviewed-by: Mirko Bonadei Cr-Commit-Position: refs/heads/master@{#30650} --- test/pc/e2e/BUILD.gn | 17 +++ .../video/default_video_quality_analyzer.cc | 80 ------------ .../video/default_video_quality_analyzer.h | 25 +--- .../video/video_quality_metrics_reporter.cc | 115 ++++++++++++++++++ .../video/video_quality_metrics_reporter.h | 68 +++++++++++ test/pc/e2e/peer_connection_quality_test.cc | 3 + 6 files changed, 205 insertions(+), 103 deletions(-) create mode 100644 test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc create mode 100644 test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h diff --git a/test/pc/e2e/BUILD.gn b/test/pc/e2e/BUILD.gn index 3855ad0141..dd4ef211fc 100644 --- a/test/pc/e2e/BUILD.gn +++ b/test/pc/e2e/BUILD.gn @@ -272,6 +272,7 @@ if (rtc_include_tests) { ":stats_poller", ":test_peer", ":video_quality_analyzer_injection_helper", + ":video_quality_metrics_reporter", "../..:field_trial", "../..:platform_video_capturer", "../..:video_test_common", @@ -466,6 +467,22 @@ rtc_library("example_video_quality_analyzer") { ] } +rtc_library("video_quality_metrics_reporter") { + visibility = [ "*" ] + + testonly = true + sources = [ + "analyzer/video/video_quality_metrics_reporter.cc", + "analyzer/video/video_quality_metrics_reporter.h", + ] + deps = [ + "../..:perf_test", + "../../../api:peer_connection_quality_test_fixture_api", + "../../../rtc_base:criticalsection", + "../../../rtc_base:rtc_numerics", + ] +} + rtc_library("default_video_quality_analyzer") { visibility = [ "*" ] diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc index 0b0c3b1add..a1c5d0aa25 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc @@ -26,7 +26,6 @@ namespace { constexpr int kMaxActiveComparisons = 10; constexpr int kFreezeThresholdMs = 150; constexpr int kMicrosPerSecond = 1000000; -constexpr int kBitsInByte = 8; void LogFrameCounters(const std::string& name, const FrameCounters& counters) { RTC_LOG(INFO) << "[" << name << "] Captured : " << counters.captured; @@ -422,59 +421,6 @@ AnalyzerStats DefaultVideoQualityAnalyzer::GetAnalyzerStats() const { return analyzer_stats_; } -// TODO(bugs.webrtc.org/10430): Migrate to the new GetStats as soon as -// bugs.webrtc.org/10428 is fixed. -void DefaultVideoQualityAnalyzer::OnStatsReports( - const std::string& pc_label, - const StatsReports& stats_reports) { - for (const StatsReport* stats_report : stats_reports) { - // The only stats collected by this analyzer are present in - // kStatsReportTypeBwe reports, so all other reports are just ignored. - if (stats_report->type() != StatsReport::StatsType::kStatsReportTypeBwe) { - continue; - } - const webrtc::StatsReport::Value* available_send_bandwidth = - stats_report->FindValue( - StatsReport::StatsValueName::kStatsValueNameAvailableSendBandwidth); - const webrtc::StatsReport::Value* retransmission_bitrate = - stats_report->FindValue( - StatsReport::StatsValueName::kStatsValueNameRetransmitBitrate); - const webrtc::StatsReport::Value* transmission_bitrate = - stats_report->FindValue( - StatsReport::StatsValueName::kStatsValueNameTransmitBitrate); - const webrtc::StatsReport::Value* actual_encode_bitrate = - stats_report->FindValue( - StatsReport::StatsValueName::kStatsValueNameActualEncBitrate); - const webrtc::StatsReport::Value* target_encode_bitrate = - stats_report->FindValue( - StatsReport::StatsValueName::kStatsValueNameTargetEncBitrate); - RTC_CHECK(available_send_bandwidth); - RTC_CHECK(retransmission_bitrate); - RTC_CHECK(transmission_bitrate); - RTC_CHECK(actual_encode_bitrate); - RTC_CHECK(target_encode_bitrate); - - rtc::CritScope crit(&video_bwe_stats_lock_); - VideoBweStats& video_bwe_stats = video_bwe_stats_[pc_label]; - video_bwe_stats.available_send_bandwidth.AddSample( - available_send_bandwidth->int_val()); - video_bwe_stats.transmission_bitrate.AddSample( - transmission_bitrate->int_val()); - video_bwe_stats.retransmission_bitrate.AddSample( - retransmission_bitrate->int_val()); - video_bwe_stats.actual_encode_bitrate.AddSample( - actual_encode_bitrate->int_val()); - video_bwe_stats.target_encode_bitrate.AddSample( - target_encode_bitrate->int_val()); - } -} - -std::map -DefaultVideoQualityAnalyzer::GetVideoBweStats() const { - rtc::CritScope crit(&video_bwe_stats_lock_); - return video_bwe_stats_; -} - void DefaultVideoQualityAnalyzer::AddComparison( absl::optional captured, absl::optional rendered, @@ -620,12 +566,6 @@ void DefaultVideoQualityAnalyzer::ReportResults() { ReportResults(GetTestCaseName(item.first), item.second, stream_frame_counters_.at(item.first)); } - { - rtc::CritScope video_bwe_crit(&video_bwe_stats_lock_); - for (const auto& item : video_bwe_stats_) { - ReportVideoBweResults(GetTestCaseName(item.first), item.second); - } - } LogFrameCounters("Global", frame_counters_); for (auto& item : stream_stats_) { LogFrameCounters(item.first, stream_frame_counters_.at(item.first)); @@ -645,26 +585,6 @@ void DefaultVideoQualityAnalyzer::ReportResults() { << analyzer_stats_.memory_overloaded_comparisons_done; } -void DefaultVideoQualityAnalyzer::ReportVideoBweResults( - const std::string& test_case_name, - const VideoBweStats& video_bwe_stats) { - ReportResult("available_send_bandwidth", test_case_name, - video_bwe_stats.available_send_bandwidth / kBitsInByte, - "bytesPerSecond"); - ReportResult("transmission_bitrate", test_case_name, - video_bwe_stats.transmission_bitrate / kBitsInByte, - "bytesPerSecond"); - ReportResult("retransmission_bitrate", test_case_name, - video_bwe_stats.retransmission_bitrate / kBitsInByte, - "bytesPerSecond"); - ReportResult("actual_encode_bitrate", test_case_name, - video_bwe_stats.actual_encode_bitrate / kBitsInByte, - "bytesPerSecond"); - ReportResult("target_encode_bitrate", test_case_name, - video_bwe_stats.target_encode_bitrate / kBitsInByte, - "bytesPerSecond"); -} - void DefaultVideoQualityAnalyzer::ReportResults( const std::string& test_case_name, const StreamStats& stats, diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h index 3ed7a65475..219a77baa2 100644 --- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h +++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h @@ -124,14 +124,6 @@ struct AnalyzerStats { int64_t memory_overloaded_comparisons_done = 0; }; -struct VideoBweStats { - SamplesStatsCounter available_send_bandwidth; - SamplesStatsCounter transmission_bitrate; - SamplesStatsCounter retransmission_bitrate; - SamplesStatsCounter actual_encode_bitrate; - SamplesStatsCounter target_encode_bitrate; -}; - class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { public: explicit DefaultVideoQualityAnalyzer( @@ -157,6 +149,8 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { void OnDecoderError(uint16_t frame_id, int32_t error_code) override; void Stop() override; std::string GetStreamLabel(uint16_t frame_id) override; + void OnStatsReports(const std::string& pc_label, + const StatsReports& stats_reports) override {} // Returns set of stream labels, that were met during test call. std::set GetKnownVideoStreams() const; @@ -169,13 +163,6 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { std::map GetStats() const; AnalyzerStats GetAnalyzerStats() const; - // Will be called everytime new stats reports are available for the - // Peer Connection identified by |pc_label|. - void OnStatsReports(const std::string& pc_label, - const StatsReports& stats_reports) override; - - std::map GetVideoBweStats() const; - private: struct FrameStats { FrameStats(std::string stream_label, Timestamp captured_time); @@ -285,8 +272,6 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { void ProcessComparison(const FrameComparison& comparison); // Report results for all metrics for all streams. void ReportResults(); - static void ReportVideoBweResults(const std::string& test_case_name, - const VideoBweStats& video_bwe_stats); void ReportResults(const std::string& test_case_name, const StreamStats& stats, const FrameCounters& frame_counters) @@ -344,12 +329,6 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { std::deque comparisons_ RTC_GUARDED_BY(comparison_lock_); AnalyzerStats analyzer_stats_ RTC_GUARDED_BY(comparison_lock_); - rtc::CriticalSection video_bwe_stats_lock_; - // Map between a peer connection label (provided by the framework) and - // its video BWE stats. - std::map video_bwe_stats_ - RTC_GUARDED_BY(video_bwe_stats_lock_); - std::vector> thread_pool_; rtc::Event comparison_available_event_; }; diff --git a/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc b/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc new file mode 100644 index 0000000000..2c7eb0e802 --- /dev/null +++ b/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +constexpr int kBitsInByte = 8; + +} // namespace + +void VideoQualityMetricsReporter::Start(absl::string_view test_case_name) { + test_case_name_ = std::string(test_case_name); +} + +// TODO(bugs.webrtc.org/10430): Migrate to the new GetStats as soon as +// bugs.webrtc.org/10428 is fixed. +void VideoQualityMetricsReporter::OnStatsReports( + const std::string& pc_label, + const StatsReports& stats_reports) { + for (const StatsReport* stats_report : stats_reports) { + // The only stats collected by this analyzer are present in + // kStatsReportTypeBwe reports, so all other reports are just ignored. + if (stats_report->type() != StatsReport::StatsType::kStatsReportTypeBwe) { + continue; + } + const webrtc::StatsReport::Value* available_send_bandwidth = + stats_report->FindValue( + StatsReport::StatsValueName::kStatsValueNameAvailableSendBandwidth); + const webrtc::StatsReport::Value* retransmission_bitrate = + stats_report->FindValue( + StatsReport::StatsValueName::kStatsValueNameRetransmitBitrate); + const webrtc::StatsReport::Value* transmission_bitrate = + stats_report->FindValue( + StatsReport::StatsValueName::kStatsValueNameTransmitBitrate); + const webrtc::StatsReport::Value* actual_encode_bitrate = + stats_report->FindValue( + StatsReport::StatsValueName::kStatsValueNameActualEncBitrate); + const webrtc::StatsReport::Value* target_encode_bitrate = + stats_report->FindValue( + StatsReport::StatsValueName::kStatsValueNameTargetEncBitrate); + RTC_CHECK(available_send_bandwidth); + RTC_CHECK(retransmission_bitrate); + RTC_CHECK(transmission_bitrate); + RTC_CHECK(actual_encode_bitrate); + RTC_CHECK(target_encode_bitrate); + + rtc::CritScope crit(&video_bwe_stats_lock_); + VideoBweStats& video_bwe_stats = video_bwe_stats_[pc_label]; + video_bwe_stats.available_send_bandwidth.AddSample( + available_send_bandwidth->int_val()); + video_bwe_stats.transmission_bitrate.AddSample( + transmission_bitrate->int_val()); + video_bwe_stats.retransmission_bitrate.AddSample( + retransmission_bitrate->int_val()); + video_bwe_stats.actual_encode_bitrate.AddSample( + actual_encode_bitrate->int_val()); + video_bwe_stats.target_encode_bitrate.AddSample( + target_encode_bitrate->int_val()); + } +} + +void VideoQualityMetricsReporter::StopAndReportResults() { + rtc::CritScope video_bwe_crit(&video_bwe_stats_lock_); + for (const auto& item : video_bwe_stats_) { + ReportVideoBweResults(GetTestCaseName(item.first), item.second); + } +} + +std::string VideoQualityMetricsReporter::GetTestCaseName( + const std::string& stream_label) const { + return test_case_name_ + "/" + stream_label; +} + +void VideoQualityMetricsReporter::ReportVideoBweResults( + const std::string& test_case_name, + const VideoBweStats& video_bwe_stats) { + ReportResult("available_send_bandwidth", test_case_name, + video_bwe_stats.available_send_bandwidth / kBitsInByte, + "bytesPerSecond"); + ReportResult("transmission_bitrate", test_case_name, + video_bwe_stats.transmission_bitrate / kBitsInByte, + "bytesPerSecond"); + ReportResult("retransmission_bitrate", test_case_name, + video_bwe_stats.retransmission_bitrate / kBitsInByte, + "bytesPerSecond"); + ReportResult("actual_encode_bitrate", test_case_name, + video_bwe_stats.actual_encode_bitrate / kBitsInByte, + "bytesPerSecond"); + ReportResult("target_encode_bitrate", test_case_name, + video_bwe_stats.target_encode_bitrate / kBitsInByte, + "bytesPerSecond"); +} + +void VideoQualityMetricsReporter::ReportResult( + const std::string& metric_name, + const std::string& test_case_name, + const SamplesStatsCounter& counter, + const std::string& unit, + webrtc::test::ImproveDirection improve_direction) { + test::PrintResult(metric_name, /*modifier=*/"", test_case_name, counter, unit, + /*important=*/false, improve_direction); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h b/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h new file mode 100644 index 0000000000..28cb0d80a5 --- /dev/null +++ b/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_METRICS_REPORTER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_METRICS_REPORTER_H_ + +#include +#include + +#include "api/test/peerconnection_quality_test_fixture.h" +#include "rtc_base/critical_section.h" +#include "rtc_base/numerics/samples_stats_counter.h" +#include "test/testsupport/perf_test.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +struct VideoBweStats { + SamplesStatsCounter available_send_bandwidth; + SamplesStatsCounter transmission_bitrate; + SamplesStatsCounter retransmission_bitrate; + SamplesStatsCounter actual_encode_bitrate; + SamplesStatsCounter target_encode_bitrate; +}; + +class VideoQualityMetricsReporter + : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter { + public: + VideoQualityMetricsReporter() = default; + ~VideoQualityMetricsReporter() override = default; + + void Start(absl::string_view test_case_name) override; + void OnStatsReports(const std::string& pc_label, + const StatsReports& reports) override; + void StopAndReportResults() override; + + private: + std::string GetTestCaseName(const std::string& stream_label) const; + static void ReportVideoBweResults(const std::string& test_case_name, + const VideoBweStats& video_bwe_stats); + // Report result for single metric for specified stream. + static void ReportResult(const std::string& metric_name, + const std::string& test_case_name, + const SamplesStatsCounter& counter, + const std::string& unit, + webrtc::test::ImproveDirection improve_direction = + webrtc::test::ImproveDirection::kNone); + + std::string test_case_name_; + + rtc::CriticalSection video_bwe_stats_lock_; + // Map between a peer connection label (provided by the framework) and + // its video BWE stats. + std::map video_bwe_stats_ + RTC_GUARDED_BY(video_bwe_stats_lock_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_METRICS_REPORTER_H_ diff --git a/test/pc/e2e/peer_connection_quality_test.cc b/test/pc/e2e/peer_connection_quality_test.cc index faf1aaabdf..f16389ff69 100644 --- a/test/pc/e2e/peer_connection_quality_test.cc +++ b/test/pc/e2e/peer_connection_quality_test.cc @@ -35,6 +35,7 @@ #include "test/frame_generator_capturer.h" #include "test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h" #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h" +#include "test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h" #include "test/pc/e2e/stats_poller.h" #include "test/platform_video_capturer.h" #include "test/testsupport/file_utils.h" @@ -321,6 +322,8 @@ void PeerConnectionE2EQualityTest::Run(RunParams run_params) { video_analyzer_threads = std::min(video_analyzer_threads, kMaxVideoAnalyzerThreads); RTC_LOG(INFO) << "video_analyzer_threads=" << video_analyzer_threads; + quality_metrics_reporters_.push_back( + std::make_unique()); video_quality_analyzer_injection_helper_->Start(test_case_name_, video_analyzer_threads);