zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[PCLF] Enable exporting of perf metric via new API

Browse Source 
Bug: b/246095034
Change-Id: I05f28e5dfc6df793c035110f89d9ac40783687f8
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/276267
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38161}
This commit is contained in:
Artem Titov
2022-09-21 16:08:03 +02:00
committed by WebRTC LUCI CQ
parent cbad8add12
commit 3680605caa
12 changed files with 827 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
api/BUILD.gn
View File

@ -622,6 +622,7 @@ if (rtc_include_tests) {
":time_controller",
":video_quality_analyzer_api",
"../test/pc/e2e:peerconnection_quality_test",
"test/metrics:global_metrics_logger_and_exporter",
]
}
}

4
api/test/create_peerconnection_quality_test_fixture.cc
View File

@ -13,6 +13,7 @@
#include <memory>
#include <utility>
#include "api/test/metrics/global_metrics_logger_and_exporter.h"
#include "api/test/time_controller.h"
#include "test/pc/e2e/peer_connection_quality_test.h"
@ -27,7 +28,8 @@ CreatePeerConnectionE2EQualityTestFixture(
std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer) {
return std::make_unique<PeerConnectionE2EQualityTest>(
std::move(test_case_name), time_controller,
std::move(audio_quality_analyzer), std::move(video_quality_analyzer));
std::move(audio_quality_analyzer), std::move(video_quality_analyzer),
test::GetGlobalMetricsLoggerAndExporter());
}
} // namespace webrtc_pc_e2e

59
api/test/metrics/BUILD.gn
View File

@ -13,6 +13,7 @@ if (rtc_enable_protobuf) {
group("metrics") {
deps = [
":global_metrics_logger_and_exporter",
":metric",
":metrics_exporter",
":metrics_logger_and_exporter",
@ -73,28 +74,6 @@ rtc_library("stdout_metrics_exporter") {
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
}
rtc_library("metrics_logger_and_exporter") {
visibility = [ "*" ]
sources = [
"metrics_logger_and_exporter.cc",
"metrics_logger_and_exporter.h",
]
deps = [
":metric",
":metrics_exporter",
"../../../rtc_base:checks",
"../../../rtc_base:logging",
"../../../rtc_base/synchronization:mutex",
"../../../system_wrappers",
"../../numerics",
]
absl_deps = [
"//third_party/abseil-cpp/absl/strings",
"//third_party/abseil-cpp/absl/types:optional",
]
}
rtc_library("chrome_perf_dashboard_metrics_exporter") {
visibility = [ "*" ]
testonly = true
@ -159,6 +138,42 @@ rtc_library("print_result_proxy_metrics_exporter") {
]
}
rtc_library("metrics_logger_and_exporter") {
visibility = [ "*" ]
sources = [
"metrics_logger_and_exporter.cc",
"metrics_logger_and_exporter.h",
]
deps = [
":metric",
":metrics_exporter",
"../../../rtc_base:checks",
"../../../rtc_base:logging",
"../../../rtc_base/synchronization:mutex",
"../../../system_wrappers",
"../../numerics",
]
absl_deps = [
"//third_party/abseil-cpp/absl/strings",
"//third_party/abseil-cpp/absl/types:optional",
]
}
rtc_library("global_metrics_logger_and_exporter") {
visibility = [ "*" ]
sources = [
"global_metrics_logger_and_exporter.cc",
"global_metrics_logger_and_exporter.h",
]
deps = [
":metrics_exporter",
":metrics_logger_and_exporter",
"../../../rtc_base:checks",
"../../../system_wrappers",
]
}
if (rtc_include_tests) {
rtc_library("stdout_metrics_exporter_test") {
testonly = true

46
api/test/metrics/global_metrics_logger_and_exporter.cc Normal file
View File

@ -0,0 +1,46 @@
/*
* Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "api/test/metrics/global_metrics_logger_and_exporter.h"
#include <memory>
#include <utility>
#include <vector>
#include "api/test/metrics/metrics_exporter.h"
#include "api/test/metrics/metrics_logger_and_exporter.h"
#include "rtc_base/checks.h"
#include "system_wrappers/include/clock.h"
namespace webrtc {
namespace test {
namespace {
MetricsLoggerAndExporter* global_metrics_logger_and_exporter = nullptr;
} // namespace
MetricsLoggerAndExporter* GetGlobalMetricsLoggerAndExporter() {
return global_metrics_logger_and_exporter;
}
void SetupGlobalMetricsLoggerAndExporter(
std::vector<std::unique_ptr<MetricsExporter>> exporters) {
RTC_CHECK(global_metrics_logger_and_exporter == nullptr);
global_metrics_logger_and_exporter = new MetricsLoggerAndExporter(
Clock::GetRealTimeClock(), std::move(exporters));
}
void ExportAndDestroyGlobalMetricsLoggerAndExporter() {
RTC_CHECK(global_metrics_logger_and_exporter != nullptr);
delete global_metrics_logger_and_exporter;
}
} // namespace test
} // namespace webrtc

40
api/test/metrics/global_metrics_logger_and_exporter.h Normal file
View File

@ -0,0 +1,40 @@
/*
* Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef API_TEST_METRICS_GLOBAL_METRICS_LOGGER_AND_EXPORTER_H_
#define API_TEST_METRICS_GLOBAL_METRICS_LOGGER_AND_EXPORTER_H_
#include <memory>
#include <vector>
#include "api/test/metrics/metrics_exporter.h"
#include "api/test/metrics/metrics_logger_and_exporter.h"
namespace webrtc {
namespace test {
// Returns current global `MetricsLoggerAndExporter`. Returns `nullptr` if there
// is global instance wasn't initialized.
MetricsLoggerAndExporter* GetGlobalMetricsLoggerAndExporter();
// Initialize global `MetricsLoggerAndExporter` with provided vector of
// exporters. Crashes if there is already initialized global instance.
void SetupGlobalMetricsLoggerAndExporter(
std::vector<std::unique_ptr<MetricsExporter>> exporters);
// Destroys global `MetricsLoggerAndExporter` forcing it to export all collected
// metrics to the configured exporters. Crashes if there is no initialized
// global instance.
void ExportAndDestroyGlobalMetricsLoggerAndExporter();
} // namespace test
} // namespace webrtc
#endif // API_TEST_METRICS_GLOBAL_METRICS_LOGGER_AND_EXPORTER_H_

3
test/BUILD.gn
View File

@ -509,6 +509,9 @@ if (rtc_include_tests && !build_with_chromium) {
":resources_dir_flag",
":test_flags",
":test_support",
"../api/test/metrics:global_metrics_logger_and_exporter",
"../api/test/metrics:metrics_exporter",
"../api/test/metrics:stdout_metrics_exporter",
"../rtc_base",
"../rtc_base:checks",
"../rtc_base:event_tracer",

21
test/pc/e2e/BUILD.gn
View File

@ -41,6 +41,7 @@ if (!build_with_chromium) {
":multi_reader_queue_test",
":names_collection_test",
":peer_connection_e2e_smoke_test",
":peer_connection_quality_test_metric_names_test",
":single_process_encoded_image_data_injector_unittest",
":stats_poller_test",
":video_frame_tracking_id_injector_unittest",
@ -407,6 +408,8 @@ if (!build_with_chromium) {
"../../../api:video_quality_analyzer_api",
"../../../api/rtc_event_log",
"../../../api/task_queue",
"../../../api/test/metrics:metric",
"../../../api/test/metrics:metrics_logger_and_exporter",
"../../../api/units:time_delta",
"../../../api/units:timestamp",
"../../../pc:pc_test_utils",
@ -484,6 +487,7 @@ if (!build_with_chromium) {
"../../../api:simulated_network_api",
"../../../api/audio_codecs:builtin_audio_decoder_factory",
"../../../api/audio_codecs:builtin_audio_encoder_factory",
"../../../api/test/metrics:global_metrics_logger_and_exporter",
"../../../api/video_codecs:builtin_video_decoder_factory",
"../../../api/video_codecs:builtin_video_encoder_factory",
"../../../call:simulated_network",
@ -510,6 +514,23 @@ if (!build_with_chromium) {
}
}
rtc_library("peer_connection_quality_test_metric_names_test") {
testonly = true
sources = [ "peer_connection_quality_test_metric_names_test.cc" ]
deps = [
":peerconnection_quality_test",
":stats_based_network_quality_metrics_reporter",
"../..:test_support",
"../../../api:create_network_emulation_manager",
"../../../api:create_peer_connection_quality_test_frame_generator",
"../../../api:network_emulation_manager_api",
"../../../api:peer_connection_quality_test_fixture_api",
"../../../api/test/metrics:metrics_logger_and_exporter",
"../../../api/test/metrics:stdout_metrics_exporter",
"../../../api/units:time_delta",
]
}
rtc_library("stats_provider") {
visibility = [ "*" ]
testonly = true

10
test/pc/e2e/peer_connection_e2e_smoke_test.cc
View File

@ -15,6 +15,7 @@
#include "api/test/create_network_emulation_manager.h"
#include "api/test/create_peer_connection_quality_test_frame_generator.h"
#include "api/test/create_peerconnection_quality_test_fixture.h"
#include "api/test/metrics/global_metrics_logger_and_exporter.h"
#include "api/test/network_emulation_manager.h"
#include "api/test/peerconnection_quality_test_fixture.h"
#include "call/simulated_network.h"
@ -54,7 +55,8 @@ class PeerConnectionE2EQualityTestSmokeTest : public ::testing::Test {
void SetUp() override {
network_emulation_ = CreateNetworkEmulationManager();
auto video_quality_analyzer = std::make_unique<DefaultVideoQualityAnalyzer>(
network_emulation_->time_controller()->GetClock());
network_emulation_->time_controller()->GetClock(),
test::GetGlobalMetricsLoggerAndExporter());
video_quality_analyzer_ = video_quality_analyzer.get();
fixture_ = CreatePeerConnectionE2EQualityTestFixture(
testing::UnitTest::GetInstance()->current_test_info()->name(),
@ -181,7 +183,7 @@ TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Smoke) {
std::map<std::string, std::vector<EmulatedEndpoint*>>(
{{"alice", network_links.first->endpoints()},
{"charlie", network_links.second->endpoints()}}),
network_emulation()));
network_emulation(), test::GetGlobalMetricsLoggerAndExporter()));
RunParams run_params(TimeDelta::Seconds(2));
run_params.enable_flex_fec_support = true;
RunAndCheckEachVideoStreamReceivedFrames(run_params);
@ -233,7 +235,7 @@ TEST_F(PeerConnectionE2EQualityTestSmokeTest, SmokeH264) {
std::map<std::string, std::vector<EmulatedEndpoint*>>(
{{"alice", network_links.first->endpoints()},
{"charlie", network_links.second->endpoints()}}),
network_emulation()));
network_emulation(), test::GetGlobalMetricsLoggerAndExporter()));
RunParams run_params(TimeDelta::Seconds(2));
run_params.enable_flex_fec_support = true;
RunAndCheckEachVideoStreamReceivedFrames(run_params);
@ -300,7 +302,7 @@ TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_ChangeNetworkConditions) {
std::map<std::string, std::vector<EmulatedEndpoint*>>(
{{"alice", alice_network->endpoints()},
{"bob", bob_network->endpoints()}}),
network_emulation()));
network_emulation(), test::GetGlobalMetricsLoggerAndExporter()));
fixture()->ExecuteAt(TimeDelta::Seconds(1), [alice_node](TimeDelta) {
BuiltInNetworkBehaviorConfig config;

55
test/pc/e2e/peer_connection_quality_test.cc
View File

@ -21,6 +21,7 @@
#include "api/rtc_event_log/rtc_event_log.h"
#include "api/rtc_event_log_output_file.h"
#include "api/scoped_refptr.h"
#include "api/test/metrics/metric.h"
#include "api/test/time_controller.h"
#include "api/test/video_quality_analyzer_interface.h"
#include "pc/sdp_utils.h"
@ -46,6 +47,8 @@ namespace webrtc {
namespace webrtc_pc_e2e {
namespace {
using ::webrtc::test::ImprovementDirection;
using ::webrtc::test::Unit;
using VideoConfig = PeerConnectionE2EQualityTestFixture::VideoConfig;
using VideoCodecConfig = PeerConnectionE2EQualityTestFixture::VideoCodecConfig;
@ -128,17 +131,30 @@ PeerConnectionE2EQualityTest::PeerConnectionE2EQualityTest(
TimeController& time_controller,
std::unique_ptr<AudioQualityAnalyzerInterface> audio_quality_analyzer,
std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer)
: PeerConnectionE2EQualityTest(std::move(test_case_name),
time_controller,
std::move(audio_quality_analyzer),
std::move(video_quality_analyzer),
/*metrics_logger_=*/nullptr) {}
PeerConnectionE2EQualityTest::PeerConnectionE2EQualityTest(
std::string test_case_name,
TimeController& time_controller,
std::unique_ptr<AudioQualityAnalyzerInterface> audio_quality_analyzer,
std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer,
test::MetricsLoggerAndExporter* metrics_logger)
: time_controller_(time_controller),
task_queue_factory_(time_controller_.CreateTaskQueueFactory()),
test_case_name_(std::move(test_case_name)),
executor_(std::make_unique<TestActivitiesExecutor>(
time_controller_.GetClock())) {
time_controller_.GetClock())),
metrics_logger_(metrics_logger) {
// Create default video quality analyzer. We will always create an analyzer,
// even if there are no video streams, because it will be installed into video
// encoder/decoder factories.
if (video_quality_analyzer == nullptr) {
video_quality_analyzer = std::make_unique<DefaultVideoQualityAnalyzer>(
time_controller_.GetClock());
time_controller_.GetClock(), metrics_logger_);
}
if (field_trial::IsEnabled("WebRTC-VideoFrameTrackingIdAdvertised")) {
encoded_image_data_propagator_ =
@ -154,7 +170,8 @@ PeerConnectionE2EQualityTest::PeerConnectionE2EQualityTest(
encoded_image_data_propagator_.get());
if (audio_quality_analyzer == nullptr) {
audio_quality_analyzer = std::make_unique<DefaultAudioQualityAnalyzer>();
audio_quality_analyzer =
std::make_unique<DefaultAudioQualityAnalyzer>(metrics_logger_);
}
audio_quality_analyzer_.swap(audio_quality_analyzer);
}
@ -289,10 +306,10 @@ void PeerConnectionE2EQualityTest::Run(RunParams run_params) {
std::min(video_analyzer_threads, kMaxVideoAnalyzerThreads);
RTC_LOG(LS_INFO) << "video_analyzer_threads=" << video_analyzer_threads;
quality_metrics_reporters_.push_back(
std::make_unique<VideoQualityMetricsReporter>(
time_controller_.GetClock()));
std::make_unique<VideoQualityMetricsReporter>(time_controller_.GetClock(),
metrics_logger_));
quality_metrics_reporters_.push_back(
std::make_unique<CrossMediaMetricsReporter>());
std::make_unique<CrossMediaMetricsReporter>(metrics_logger_));
video_quality_analyzer_injection_helper_->Start(
test_case_name_,
@ -719,14 +736,24 @@ void PeerConnectionE2EQualityTest::TearDownCall() {
}
void PeerConnectionE2EQualityTest::ReportGeneralTestResults() {
test::PrintResult(*alice_->params().name + "_connected", "", test_case_name_,
alice_connected_, "unitless",
/*important=*/false,
test::ImproveDirection::kBiggerIsBetter);
test::PrintResult(*bob_->params().name + "_connected", "", test_case_name_,
bob_connected_, "unitless",
/*important=*/false,
test::ImproveDirection::kBiggerIsBetter);
if (metrics_logger_ == nullptr) {
test::PrintResult(*alice_->params().name + "_connected", "",
test_case_name_, alice_connected_, "unitless",
/*important=*/false,
test::ImproveDirection::kBiggerIsBetter);
test::PrintResult(*bob_->params().name + "_connected", "", test_case_name_,
bob_connected_, "unitless",
/*important=*/false,
test::ImproveDirection::kBiggerIsBetter);
} else {
metrics_logger_->LogSingleValueMetric(
*alice_->params().name + "_connected", test_case_name_,
alice_connected_, Unit::kUnitless,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogSingleValueMetric(
*bob_->params().name + "_connected", test_case_name_, bob_connected_,
Unit::kUnitless, ImprovementDirection::kBiggerIsBetter);
}
}
Timestamp PeerConnectionE2EQualityTest::Now() const {

8
test/pc/e2e/peer_connection_quality_test.h
View File

@ -18,6 +18,7 @@
#include "absl/strings/string_view.h"
#include "api/task_queue/task_queue_factory.h"
#include "api/test/audio_quality_analyzer_interface.h"
#include "api/test/metrics/metrics_logger_and_exporter.h"
#include "api/test/peerconnection_quality_test_fixture.h"
#include "api/test/time_controller.h"
#include "api/units/time_delta.h"
@ -56,6 +57,12 @@ class PeerConnectionE2EQualityTest
TimeController& time_controller,
std::unique_ptr<AudioQualityAnalyzerInterface> audio_quality_analyzer,
std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer);
PeerConnectionE2EQualityTest(
std::string test_case_name,
TimeController& time_controller,
std::unique_ptr<AudioQualityAnalyzerInterface> audio_quality_analyzer,
std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer,
test::MetricsLoggerAndExporter* metrics_logger);
~PeerConnectionE2EQualityTest() override = default;
@ -117,6 +124,7 @@ class PeerConnectionE2EQualityTest
std::unique_ptr<EncodedImageDataPropagator> encoded_image_data_propagator_;
std::unique_ptr<AudioQualityAnalyzerInterface> audio_quality_analyzer_;
std::unique_ptr<TestActivitiesExecutor> executor_;
test::MetricsLoggerAndExporter* const metrics_logger_;
std::vector<std::unique_ptr<PeerConfigurerImpl>> peer_configurations_;
std::vector<PeerHandleImpl> peer_handles_;

602
test/pc/e2e/peer_connection_quality_test_metric_names_test.cc Normal file
View File

@ -0,0 +1,602 @@
/*
* Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <memory>
#include "api/test/create_network_emulation_manager.h"
#include "api/test/create_peer_connection_quality_test_frame_generator.h"
#include "api/test/metrics/metrics_logger_and_exporter.h"
#include "api/test/metrics/stdout_metrics_exporter.h"
#include "api/test/network_emulation_manager.h"
#include "api/test/peerconnection_quality_test_fixture.h"
#include "api/units/time_delta.h"
#include "test/gmock.h"
#include "test/gtest.h"
#include "test/pc/e2e/peer_connection_quality_test.h"
#include "test/pc/e2e/stats_based_network_quality_metrics_reporter.h"
namespace webrtc {
namespace webrtc_pc_e2e {
namespace {
using ::testing::UnorderedElementsAre;
using ::webrtc::test::ImprovementDirection;
using ::webrtc::test::Metric;
using ::webrtc::test::MetricsExporter;
using ::webrtc::test::MetricsLoggerAndExporter;
using ::webrtc::test::StdoutMetricsExporter;
using ::webrtc::test::Unit;
using RunParams =
::webrtc::webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture::RunParams;
using VideoConfig =
::webrtc::webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture::VideoConfig;
using AudioConfig =
::webrtc::webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture::AudioConfig;
using PeerConfigurer = ::webrtc::webrtc_pc_e2e::
PeerConnectionE2EQualityTestFixture::PeerConfigurer;
using VideoCodecConfig = ::webrtc::webrtc_pc_e2e::
PeerConnectionE2EQualityTestFixture::VideoCodecConfig;
// Adds a peer with some audio and video (the client should not care about
// details about audio and video configs).
void AddDefaultAudioVideoPeer(
absl::string_view peer_name,
absl::string_view audio_stream_label,
absl::string_view video_stream_label,
const PeerNetworkDependencies& network_dependencies,
PeerConnectionE2EQualityTestFixture& fixture) {
fixture.AddPeer(network_dependencies, [&](PeerConfigurer* peer) {
peer->SetName(peer_name);
AudioConfig audio{std::string(audio_stream_label)};
audio.sync_group = std::string(peer_name);
peer->SetAudioConfig(std::move(audio));
VideoConfig video(std::string(video_stream_label), 320, 180, 15);
video.sync_group = std::string(peer_name);
peer->AddVideoConfig(std::move(video));
peer->SetVideoCodecs({VideoCodecConfig(cricket::kVp8CodecName)});
});
}
// Metric fields to assert on
struct MetricValidationInfo {
std::string test_case;
std::string name;
Unit unit;
ImprovementDirection improvement_direction;
};
bool operator==(const MetricValidationInfo& a, const MetricValidationInfo& b) {
return a.name == b.name && a.test_case == b.test_case && a.unit == b.unit &&
a.improvement_direction == b.improvement_direction;
}
std::ostream& operator<<(std::ostream& os, const MetricValidationInfo& m) {
os << "{ test_case=" << m.test_case << "; name=" << m.name
<< "; unit=" << test::ToString(m.unit)
<< "; improvement_direction=" << test::ToString(m.improvement_direction)
<< " }";
return os;
}
std::vector<MetricValidationInfo> ToValidationInfo(
const std::vector<Metric>& metrics) {
std::vector<MetricValidationInfo> out;
for (const Metric& m : metrics) {
out.push_back(
MetricValidationInfo{.test_case = m.test_case,
.name = m.name,
.unit = m.unit,
.improvement_direction = m.improvement_direction});
}
return out;
}
TEST(PeerConnectionE2EQualityTestMetricNamesTest,
ExportedMetricsHasCorrectName) {
std::unique_ptr<NetworkEmulationManager> network_emulation =
CreateNetworkEmulationManager(TimeMode::kSimulated);
std::vector<std::unique_ptr<MetricsExporter>> exporters;
exporters.push_back(std::make_unique<StdoutMetricsExporter>());
MetricsLoggerAndExporter metrics_logger(
network_emulation->time_controller()->GetClock(), std::move(exporters));
PeerConnectionE2EQualityTest fixture(
"test_case", *network_emulation->time_controller(),
/*audio_quality_analyzer=*/nullptr, /*video_quality_analyzer=*/nullptr,
&metrics_logger);
EmulatedEndpoint* alice_endpoint =
network_emulation->CreateEndpoint(EmulatedEndpointConfig());
EmulatedEndpoint* bob_endpoint =
network_emulation->CreateEndpoint(EmulatedEndpointConfig());
network_emulation->CreateRoute(
alice_endpoint, {network_emulation->CreateUnconstrainedEmulatedNode()},
bob_endpoint);
network_emulation->CreateRoute(
bob_endpoint, {network_emulation->CreateUnconstrainedEmulatedNode()},
alice_endpoint);
EmulatedNetworkManagerInterface* alice_network =
network_emulation->CreateEmulatedNetworkManagerInterface(
{alice_endpoint});
EmulatedNetworkManagerInterface* bob_network =
network_emulation->CreateEmulatedNetworkManagerInterface({bob_endpoint});
AddDefaultAudioVideoPeer("alice", "alice_audio", "alice_video",
alice_network->network_dependencies(), fixture);
AddDefaultAudioVideoPeer("bob", "bob_audio", "bob_video",
bob_network->network_dependencies(), fixture);
fixture.AddQualityMetricsReporter(
std::make_unique<StatsBasedNetworkQualityMetricsReporter>(
std::map<std::string, std::vector<EmulatedEndpoint*>>(
{{"alice", alice_network->endpoints()},
{"bob", bob_network->endpoints()}}),
network_emulation.get(), &metrics_logger));
// Run for at least 7 seconds, so AV-sync metrics will be collected.
fixture.Run(RunParams(TimeDelta::Seconds(7)));
std::vector<MetricValidationInfo> metrics =
ToValidationInfo(metrics_logger.GetCollectedMetrics());
EXPECT_THAT(
metrics,
UnorderedElementsAre(
// Metrics from PeerConnectionE2EQualityTest
MetricValidationInfo{
.test_case = "test_case",
.name = "alice_connected",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case",
.name = "bob_connected",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
// Metrics from DefaultAudioQualityAnalyzer
MetricValidationInfo{
.test_case = "test_case/alice_audio",
.name = "expand_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_audio",
.name = "accelerate_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_audio",
.name = "preemptive_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_audio",
.name = "speech_expand_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_audio",
.name = "average_jitter_buffer_delay_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_audio",
.name = "preferred_buffer_size_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob_audio",
.name = "expand_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob_audio",
.name = "accelerate_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob_audio",
.name = "preemptive_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob_audio",
.name = "speech_expand_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob_audio",
.name = "average_jitter_buffer_delay_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob_audio",
.name = "preferred_buffer_size_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
// Metrics from DefaultVideoQualityAnalyzer
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "psnr_dB",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "ssim",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "transport_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "total_delay_incl_transport",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "time_between_rendered_frames",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "harmonic_framerate",
.unit = Unit::kHertz,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "encode_frame_rate",
.unit = Unit::kHertz,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "encode_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "time_between_freezes",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "freeze_time_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "pixels_per_frame",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "min_psnr_dB",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "decode_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "receive_to_render_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "dropped_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "frames_in_flight",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "rendered_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "max_skipped",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "target_encode_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "actual_encode_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "psnr_dB",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "ssim",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "transport_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "total_delay_incl_transport",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "time_between_rendered_frames",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "harmonic_framerate",
.unit = Unit::kHertz,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "encode_frame_rate",
.unit = Unit::kHertz,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "encode_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "time_between_freezes",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "freeze_time_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "pixels_per_frame",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "min_psnr_dB",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "decode_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "receive_to_render_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "dropped_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "frames_in_flight",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "rendered_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "max_skipped",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "target_encode_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob_video",
.name = "actual_encode_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case",
.name = "cpu_usage",
.unit = Unit::kPercent,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
// Metrics from StatsBasedNetworkQualityMetricsReporter
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "bytes_discarded_no_receiver",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "packets_discarded_no_receiver",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "payload_bytes_received",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "payload_bytes_sent",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "bytes_sent",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "packets_sent",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "average_send_rate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "bytes_received",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "packets_received",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "average_receive_rate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "sent_packets_loss",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "bytes_discarded_no_receiver",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "packets_discarded_no_receiver",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "payload_bytes_received",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "payload_bytes_sent",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "bytes_sent",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "packets_sent",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "average_send_rate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "bytes_received",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "packets_received",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "average_receive_rate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "sent_packets_loss",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
// Metrics from VideoQualityMetricsReporter
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "available_send_bandwidth",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "transmission_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "retransmission_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "available_send_bandwidth",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "transmission_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "retransmission_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
// Metrics from CrossMediaMetricsReporter
MetricValidationInfo{
.test_case = "test_case/alice_alice_audio",
.name = "audio_ahead_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_alice_video",
.name = "video_ahead_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/bob_bob_audio",
.name = "audio_ahead_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{.test_case = "test_case/bob_bob_video",
.name = "video_ahead_ms",
.unit = Unit::kMilliseconds,
.improvement_direction =
ImprovementDirection::kSmallerIsBetter}));
}
} // namespace
} // namespace webrtc_pc_e2e
} // namespace webrtc

19
test/test_main_lib.cc
View File

@ -13,11 +13,15 @@
#include <fstream>
#include <memory>
#include <string>
#include <vector>
#include "absl/flags/flag.h"
#include "absl/memory/memory.h"
#include "absl/strings/match.h"
#include "absl/types/optional.h"
#include "api/test/metrics/global_metrics_logger_and_exporter.h"
#include "api/test/metrics/metrics_exporter.h"
#include "api/test/metrics/stdout_metrics_exporter.h"
#include "rtc_base/checks.h"
#include "rtc_base/event_tracer.h"
#include "rtc_base/logging.h"
@ -65,6 +69,11 @@ ABSL_FLAG(std::string,
"",
"Path to output an empty JSON file which Chromium infra requires.");
ABSL_FLAG(bool,
export_perf_results_new_api,
false,
"Tells to initialize new API for exporting performance metrics");
ABSL_FLAG(bool, logs, true, "print logs to stderr");
ABSL_FLAG(bool, verbose, false, "verbose logs to stderr");
@ -101,6 +110,12 @@ class TestMainImpl : public TestMain {
rtc::LogMessage::SetLogToStderr(absl::GetFlag(FLAGS_logs) ||
absl::GetFlag(FLAGS_verbose));
if (absl::GetFlag(FLAGS_export_perf_results_new_api)) {
std::vector<std::unique_ptr<test::MetricsExporter>> exporters;
exporters.push_back(std::make_unique<test::StdoutMetricsExporter>());
test::SetupGlobalMetricsLoggerAndExporter(std::move(exporters));
}
// InitFieldTrialsFromString stores the char*, so the char array must
// outlive the application.
field_trials_ = absl::GetFlag(FLAGS_force_fieldtrials);
@ -158,6 +173,10 @@ class TestMainImpl : public TestMain {
webrtc::test::PrintPlottableResults(*metrics_to_plot);
}
if (absl::GetFlag(FLAGS_export_perf_results_new_api)) {
test::ExportAndDestroyGlobalMetricsLoggerAndExporter();
}
std::string result_filename =
absl::GetFlag(FLAGS_isolated_script_test_output);
if (!result_filename.empty()) {