zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[DVQA] Add ability to export metrics with MetricsLoggerAndExporter

Browse Source 
Bug: b/246095034
Change-Id: Ibbadd11ff27f65cc128efd891eace89df3c59316
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/276101
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38145}
This commit is contained in:
Artem Titov
2022-09-21 09:27:40 +02:00
committed by WebRTC LUCI CQ
parent 2d1907bfd2
commit e11d5e378c
6 changed files with 916 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
api/test/metrics/metrics_logger_and_exporter.cc
View File

@ -24,6 +24,19 @@
namespace webrtc {
namespace test {
namespace {
Metric::Stats ToStats(const SamplesStatsCounter& values) {
if (values.IsEmpty()) {
return Metric::Stats();
}
return Metric::Stats{.mean = values.GetAverage(),
.stddev = values.GetStandardDeviation(),
.min = values.GetMin(),
.max = values.GetMax()};
}
} // namespace
MetricsLoggerAndExporter::~MetricsLoggerAndExporter() {
bool export_result = Export();
@ -72,17 +85,13 @@ void MetricsLoggerAndExporter::LogMetric(
.sample_metadata = sample.metadata});
}
metrics_.push_back(
Metric{.name = std::string(name),
.unit = unit,
.improvement_direction = improvement_direction,
.test_case = std::string(test_case_name),
.metric_metadata = std::move(metadata),
.time_series = std::move(time_series),
.stats = Metric::Stats{.mean = values.GetAverage(),
.stddev = values.GetStandardDeviation(),
.min = values.GetMin(),
.max = values.GetMax()}});
metrics_.push_back(Metric{.name = std::string(name),
.unit = unit,
.improvement_direction = improvement_direction,
.test_case = std::string(test_case_name),
.metric_metadata = std::move(metadata),
.time_series = std::move(time_series),
.stats = ToStats(values)});
}
void MetricsLoggerAndExporter::LogMetric(

24
api/test/metrics/metrics_logger_and_exporter_test.cc
View File

@ -145,6 +145,30 @@ TEST(MetricsLoggerAndExporterTest,
ASSERT_THAT(metric.stats.max, absl::optional<double>(20.0));
}
TEST(MetricsLoggerAndExporterTest,
LogMetricWithEmptySamplesStatsCounterRecordsEmptyMetric) {
TestMetricsExporterFactory exporter_factory;
{
std::vector<std::unique_ptr<MetricsExporter>> exporters;
exporters.push_back(exporter_factory.CreateExporter());
MetricsLoggerAndExporter logger(Clock::GetRealTimeClock(),
std::move(exporters));
SamplesStatsCounter values;
logger.LogMetric("metric_name", "test_case_name", values, Unit::kUnitless,
ImprovementDirection::kBiggerIsBetter, DefaultMetadata());
}
std::vector<Metric> metrics = exporter_factory.exported_metrics;
ASSERT_THAT(metrics.size(), Eq(1lu));
EXPECT_THAT(metrics[0].name, Eq("metric_name"));
EXPECT_THAT(metrics[0].test_case, Eq("test_case_name"));
EXPECT_THAT(metrics[0].time_series.samples, IsEmpty());
ASSERT_THAT(metrics[0].stats.mean, Eq(absl::nullopt));
ASSERT_THAT(metrics[0].stats.stddev, Eq(absl::nullopt));
ASSERT_THAT(metrics[0].stats.min, Eq(absl::nullopt));
ASSERT_THAT(metrics[0].stats.max, Eq(absl::nullopt));
}
TEST(MetricsLoggerAndExporterTest, LogMetricWithStatsRecordsMetric) {
TestMetricsExporterFactory exporter_factory;
{

26
test/pc/e2e/BUILD.gn
View File

@ -35,6 +35,7 @@ if (!build_with_chromium) {
deps = [
":default_video_quality_analyzer_frames_comparator_test",
":default_video_quality_analyzer_metric_names_test",
":default_video_quality_analyzer_stream_state_test",
":default_video_quality_analyzer_test",
":multi_reader_queue_test",
@ -564,6 +565,29 @@ if (!build_with_chromium) {
]
}
rtc_library("default_video_quality_analyzer_metric_names_test") {
testonly = true
sources = [
"analyzer/video/default_video_quality_analyzer_metric_names_test.cc",
]
deps = [
":default_video_quality_analyzer",
":default_video_quality_analyzer_shared",
"../..:test_support",
"../../../api:create_frame_generator",
"../../../api:rtp_packet_info",
"../../../api/test/metrics:metric",
"../../../api/test/metrics:metrics_logger_and_exporter",
"../../../api/test/metrics:stdout_metrics_exporter",
"../../../api/video:encoded_image",
"../../../api/video:video_frame",
"../../../common_video",
"../../../rtc_base:stringutils",
"../../../rtc_tools:video_quality_analysis",
"../../../system_wrappers",
]
}
rtc_library("default_video_quality_analyzer_frames_comparator_test") {
testonly = true
sources = [ "analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc" ]
@ -719,6 +743,8 @@ if (!build_with_chromium) {
"../../../api:array_view",
"../../../api:video_quality_analyzer_api",
"../../../api/numerics",
"../../../api/test/metrics:metric",
"../../../api/test/metrics:metrics_logger_and_exporter",
"../../../api/units:data_size",
"../../../api/units:time_delta",
"../../../api/units:timestamp",

271
test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
View File

@ -18,6 +18,7 @@
#include "api/array_view.h"
#include "api/numerics/samples_stats_counter.h"
#include "api/test/metrics/metric.h"
#include "api/units/time_delta.h"
#include "api/video/i420_buffer.h"
#include "api/video/video_frame.h"
@ -36,6 +37,9 @@
namespace webrtc {
namespace {
using ::webrtc::test::ImprovementDirection;
using ::webrtc::test::Unit;
constexpr int kBitsInByte = 8;
constexpr absl::string_view kSkipRenderedFrameReasonProcessed = "processed";
constexpr absl::string_view kSkipRenderedFrameReasonRendered = "rendered";
@ -123,9 +127,19 @@ SamplesStatsCounter::StatsSample StatsSample(double value,
DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer(
webrtc::Clock* clock,
DefaultVideoQualityAnalyzerOptions options)
: DefaultVideoQualityAnalyzer(clock,
/*metrics_logger=*/nullptr,
std::move(options)) {}
DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer(
webrtc::Clock* clock,
test::MetricsLoggerAndExporter* metrics_logger,
DefaultVideoQualityAnalyzerOptions options)
: options_(options),
clock_(clock),
metrics_logger_(metrics_logger),
frames_comparator_(clock, cpu_measurer_, options) {}
DefaultVideoQualityAnalyzer::~DefaultVideoQualityAnalyzer() {
Stop();
}
@ -951,87 +965,186 @@ void DefaultVideoQualityAnalyzer::ReportResults(
video_duration.seconds<double>() / sum_squared_interframe_delays_secs;
}
ReportResult("psnr", test_case_name, stats.psnr, "dB",
ImproveDirection::kBiggerIsBetter);
ReportResult("ssim", test_case_name, stats.ssim, "unitless",
ImproveDirection::kBiggerIsBetter);
ReportResult("transport_time", test_case_name, stats.transport_time_ms, "ms",
ImproveDirection::kSmallerIsBetter);
ReportResult("total_delay_incl_transport", test_case_name,
stats.total_delay_incl_transport_ms, "ms",
ImproveDirection::kSmallerIsBetter);
ReportResult("time_between_rendered_frames", test_case_name,
stats.time_between_rendered_frames_ms, "ms",
ImproveDirection::kSmallerIsBetter);
test::PrintResult("harmonic_framerate", "", test_case_name,
harmonic_framerate_fps, "Hz", /*important=*/false,
ImproveDirection::kBiggerIsBetter);
test::PrintResult("encode_frame_rate", "", test_case_name,
stats.encode_frame_rate.IsEmpty()
? 0
: stats.encode_frame_rate.GetEventsPerSecond(),
"Hz", /*important=*/false,
ImproveDirection::kBiggerIsBetter);
ReportResult("encode_time", test_case_name, stats.encode_time_ms, "ms",
ImproveDirection::kSmallerIsBetter);
ReportResult("time_between_freezes", test_case_name,
stats.time_between_freezes_ms, "ms",
ImproveDirection::kBiggerIsBetter);
ReportResult("freeze_time_ms", test_case_name, stats.freeze_time_ms, "ms",
ImproveDirection::kSmallerIsBetter);
ReportResult("pixels_per_frame", test_case_name,
stats.resolution_of_rendered_frame, "count",
ImproveDirection::kBiggerIsBetter);
test::PrintResult("min_psnr", "", test_case_name,
stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), "dB",
/*important=*/false, ImproveDirection::kBiggerIsBetter);
ReportResult("decode_time", test_case_name, stats.decode_time_ms, "ms",
ImproveDirection::kSmallerIsBetter);
ReportResult("receive_to_render_time", test_case_name,
stats.receive_to_render_time_ms, "ms",
ImproveDirection::kSmallerIsBetter);
test::PrintResult("dropped_frames", "", test_case_name,
frame_counters.dropped, "count",
/*important=*/false, ImproveDirection::kSmallerIsBetter);
test::PrintResult("frames_in_flight", "", test_case_name,
frame_counters.captured - frame_counters.rendered -
frame_counters.dropped,
"count",
/*important=*/false, ImproveDirection::kSmallerIsBetter);
test::PrintResult("rendered_frames", "", test_case_name,
frame_counters.rendered, "count", /*important=*/false,
ImproveDirection::kBiggerIsBetter);
ReportResult("max_skipped", test_case_name, stats.skipped_between_rendered,
"count", ImproveDirection::kSmallerIsBetter);
ReportResult("target_encode_bitrate", test_case_name,
stats.target_encode_bitrate / kBitsInByte, "bytesPerSecond",
ImproveDirection::kNone);
test::PrintResult("actual_encode_bitrate", "", test_case_name,
static_cast<double>(stats.total_encoded_images_payload) /
test_duration.seconds<double>(),
"bytesPerSecond", /*important=*/false,
ImproveDirection::kNone);
if (options_.report_detailed_frame_stats) {
test::PrintResult("num_encoded_frames", "", test_case_name,
frame_counters.encoded, "count",
/*important=*/false, ImproveDirection::kBiggerIsBetter);
test::PrintResult("num_decoded_frames", "", test_case_name,
frame_counters.decoded, "count",
/*important=*/false, ImproveDirection::kBiggerIsBetter);
test::PrintResult("num_send_key_frames", "", test_case_name,
stats.num_send_key_frames, "count",
/*important=*/false, ImproveDirection::kBiggerIsBetter);
test::PrintResult("num_recv_key_frames", "", test_case_name,
stats.num_recv_key_frames, "count",
/*important=*/false, ImproveDirection::kBiggerIsBetter);
ReportResult("recv_key_frame_size_bytes", test_case_name,
stats.recv_key_frame_size_bytes, "count",
if (metrics_logger_ == nullptr) {
// Report results through old performance metrics API.
ReportResult("psnr", test_case_name, stats.psnr, "dB",
ImproveDirection::kBiggerIsBetter);
ReportResult("recv_delta_frame_size_bytes", test_case_name,
stats.recv_delta_frame_size_bytes, "count",
ReportResult("ssim", test_case_name, stats.ssim, "unitless",
ImproveDirection::kBiggerIsBetter);
ReportResult("transport_time", test_case_name, stats.transport_time_ms,
"ms", ImproveDirection::kSmallerIsBetter);
ReportResult("total_delay_incl_transport", test_case_name,
stats.total_delay_incl_transport_ms, "ms",
ImproveDirection::kSmallerIsBetter);
ReportResult("time_between_rendered_frames", test_case_name,
stats.time_between_rendered_frames_ms, "ms",
ImproveDirection::kSmallerIsBetter);
test::PrintResult("harmonic_framerate", "", test_case_name,
harmonic_framerate_fps, "Hz", /*important=*/false,
ImproveDirection::kBiggerIsBetter);
test::PrintResult("encode_frame_rate", "", test_case_name,
stats.encode_frame_rate.IsEmpty()
? 0
: stats.encode_frame_rate.GetEventsPerSecond(),
"Hz", /*important=*/false,
ImproveDirection::kBiggerIsBetter);
ReportResult("encode_time", test_case_name, stats.encode_time_ms, "ms",
ImproveDirection::kSmallerIsBetter);
ReportResult("time_between_freezes", test_case_name,
stats.time_between_freezes_ms, "ms",
ImproveDirection::kBiggerIsBetter);
ReportResult("freeze_time_ms", test_case_name, stats.freeze_time_ms, "ms",
ImproveDirection::kSmallerIsBetter);
ReportResult("pixels_per_frame", test_case_name,
stats.resolution_of_rendered_frame, "count",
ImproveDirection::kBiggerIsBetter);
test::PrintResult("min_psnr", "", test_case_name,
stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), "dB",
/*important=*/false, ImproveDirection::kBiggerIsBetter);
ReportResult("decode_time", test_case_name, stats.decode_time_ms, "ms",
ImproveDirection::kSmallerIsBetter);
ReportResult("receive_to_render_time", test_case_name,
stats.receive_to_render_time_ms, "ms",
ImproveDirection::kSmallerIsBetter);
test::PrintResult("dropped_frames", "", test_case_name,
frame_counters.dropped, "count",
/*important=*/false, ImproveDirection::kSmallerIsBetter);
test::PrintResult("frames_in_flight", "", test_case_name,
frame_counters.captured - frame_counters.rendered -
frame_counters.dropped,
"count",
/*important=*/false, ImproveDirection::kSmallerIsBetter);
test::PrintResult("rendered_frames", "", test_case_name,
frame_counters.rendered, "count", /*important=*/false,
ImproveDirection::kBiggerIsBetter);
ReportResult("max_skipped", test_case_name, stats.skipped_between_rendered,
"count", ImproveDirection::kSmallerIsBetter);
ReportResult("target_encode_bitrate", test_case_name,
stats.target_encode_bitrate / kBitsInByte, "bytesPerSecond",
ImproveDirection::kNone);
test::PrintResult("actual_encode_bitrate", "", test_case_name,
static_cast<double>(stats.total_encoded_images_payload) /
test_duration.seconds<double>(),
"bytesPerSecond", /*important=*/false,
ImproveDirection::kNone);
if (options_.report_detailed_frame_stats) {
test::PrintResult("num_encoded_frames", "", test_case_name,
frame_counters.encoded, "count",
/*important=*/false, ImproveDirection::kBiggerIsBetter);
test::PrintResult("num_decoded_frames", "", test_case_name,
frame_counters.decoded, "count",
/*important=*/false, ImproveDirection::kBiggerIsBetter);
test::PrintResult("num_send_key_frames", "", test_case_name,
stats.num_send_key_frames, "count",
/*important=*/false, ImproveDirection::kBiggerIsBetter);
test::PrintResult("num_recv_key_frames", "", test_case_name,
stats.num_recv_key_frames, "count",
/*important=*/false, ImproveDirection::kBiggerIsBetter);
ReportResult("recv_key_frame_size_bytes", test_case_name,
stats.recv_key_frame_size_bytes, "count",
ImproveDirection::kBiggerIsBetter);
ReportResult("recv_delta_frame_size_bytes", test_case_name,
stats.recv_delta_frame_size_bytes, "count",
ImproveDirection::kBiggerIsBetter);
}
} else {
metrics_logger_->LogMetric("psnr_dB", test_case_name, stats.psnr,
Unit::kUnitless,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("ssim", test_case_name, stats.ssim,
Unit::kUnitless,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("transport_time", test_case_name,
stats.transport_time_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("total_delay_incl_transport", test_case_name,
stats.total_delay_incl_transport_ms,
Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("time_between_rendered_frames", test_case_name,
stats.time_between_rendered_frames_ms,
Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogSingleValueMetric(
"harmonic_framerate", test_case_name, harmonic_framerate_fps,
Unit::kHertz, ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogSingleValueMetric(
"encode_frame_rate", test_case_name,
stats.encode_frame_rate.IsEmpty()
? 0
: stats.encode_frame_rate.GetEventsPerSecond(),
Unit::kHertz, ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("encode_time", test_case_name,
stats.encode_time_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric(
"time_between_freezes", test_case_name, stats.time_between_freezes_ms,
Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("freeze_time_ms", test_case_name,
stats.freeze_time_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("pixels_per_frame", test_case_name,
stats.resolution_of_rendered_frame, Unit::kCount,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogSingleValueMetric(
"min_psnr_dB", test_case_name,
stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), Unit::kUnitless,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("decode_time", test_case_name,
stats.decode_time_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("receive_to_render_time", test_case_name,
stats.receive_to_render_time_ms,
Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogSingleValueMetric(
"dropped_frames", test_case_name, frame_counters.dropped, Unit::kCount,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogSingleValueMetric(
"frames_in_flight", test_case_name,
frame_counters.captured - frame_counters.rendered -
frame_counters.dropped,
Unit::kCount, ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogSingleValueMetric(
"rendered_frames", test_case_name, frame_counters.rendered,
Unit::kCount, ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("max_skipped", test_case_name,
stats.skipped_between_rendered, Unit::kCount,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("target_encode_bitrate", test_case_name,
stats.target_encode_bitrate / 1000,
Unit::kKilobitsPerSecond,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"actual_encode_bitrate", test_case_name,
static_cast<double>(stats.total_encoded_images_payload) /
test_duration.seconds<double>() * kBitsInByte / 1000,
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
if (options_.report_detailed_frame_stats) {
metrics_logger_->LogSingleValueMetric(
"num_encoded_frames", test_case_name, frame_counters.encoded,
Unit::kCount, ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogSingleValueMetric(
"num_decoded_frames", test_case_name, frame_counters.decoded,
Unit::kCount, ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogSingleValueMetric(
"num_send_key_frames", test_case_name, stats.num_send_key_frames,
Unit::kCount, ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogSingleValueMetric(
"num_recv_key_frames", test_case_name, stats.num_recv_key_frames,
Unit::kCount, ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("recv_key_frame_size_bytes", test_case_name,
stats.recv_key_frame_size_bytes, Unit::kCount,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("recv_delta_frame_size_bytes", test_case_name,
stats.recv_delta_frame_size_bytes,
Unit::kCount,
ImprovementDirection::kBiggerIsBetter);
}
}
}

5
test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
View File

@ -22,6 +22,7 @@
#include "api/array_view.h"
#include "api/numerics/samples_stats_counter.h"
#include "api/test/metrics/metrics_logger_and_exporter.h"
#include "api/test/video_quality_analyzer_interface.h"
#include "api/units/data_size.h"
#include "api/units/timestamp.h"
@ -49,6 +50,9 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
explicit DefaultVideoQualityAnalyzer(
webrtc::Clock* clock,
DefaultVideoQualityAnalyzerOptions options = {});
DefaultVideoQualityAnalyzer(webrtc::Clock* clock,
test::MetricsLoggerAndExporter* metrics_logger,
DefaultVideoQualityAnalyzerOptions options = {});
~DefaultVideoQualityAnalyzer() override;
void Start(std::string test_case_name,
@ -148,6 +152,7 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
const DefaultVideoQualityAnalyzerOptions options_;
webrtc::Clock* const clock_;
test::MetricsLoggerAndExporter* const metrics_logger_;
std::string test_label_;

649
test/pc/e2e/analyzer/video/default_video_quality_analyzer_metric_names_test.cc Normal file
View File

@ -0,0 +1,649 @@
/*
* Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <memory>
#include <string>
#include <vector>
#include "api/rtp_packet_info.h"
#include "api/rtp_packet_infos.h"
#include "api/test/create_frame_generator.h"
#include "api/test/metrics/metric.h"
#include "api/test/metrics/metrics_logger_and_exporter.h"
#include "api/test/metrics/stdout_metrics_exporter.h"
#include "api/video/encoded_image.h"
#include "api/video/i420_buffer.h"
#include "api/video/video_frame.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "rtc_tools/frame_analyzer/video_geometry_aligner.h"
#include "system_wrappers/include/sleep.h"
#include "test/gmock.h"
#include "test/gtest.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h"
namespace webrtc {
namespace {
using ::testing::Contains;
using ::testing::SizeIs;
using ::testing::UnorderedElementsAre;
using ::webrtc::test::ImprovementDirection;
using ::webrtc::test::Metric;
using ::webrtc::test::MetricsExporter;
using ::webrtc::test::MetricsLoggerAndExporter;
using ::webrtc::test::StdoutMetricsExporter;
using ::webrtc::test::Unit;
constexpr int kAnalyzerMaxThreadsCount = 1;
constexpr int kMaxFramesInFlightPerStream = 10;
constexpr int kFrameWidth = 320;
constexpr int kFrameHeight = 240;
DefaultVideoQualityAnalyzerOptions AnalyzerOptionsForTest() {
DefaultVideoQualityAnalyzerOptions options;
options.compute_psnr = true;
options.compute_ssim = true;
options.adjust_cropping_before_comparing_frames = false;
options.max_frames_in_flight_per_stream_count = kMaxFramesInFlightPerStream;
options.report_detailed_frame_stats = true;
return options;
}
VideoFrame NextFrame(test::FrameGeneratorInterface* frame_generator,
int64_t timestamp_us) {
test::FrameGeneratorInterface::VideoFrameData frame_data =
frame_generator->NextFrame();
return VideoFrame::Builder()
.set_video_frame_buffer(frame_data.buffer)
.set_update_rect(frame_data.update_rect)
.set_timestamp_us(timestamp_us)
.build();
}
EncodedImage FakeEncode(const VideoFrame& frame) {
EncodedImage image;
std::vector<RtpPacketInfo> packet_infos;
packet_infos.push_back(RtpPacketInfo(
/*ssrc=*/1,
/*csrcs=*/{},
/*rtp_timestamp=*/frame.timestamp(),
/*receive_time=*/Timestamp::Micros(frame.timestamp_us() + 10000)));
image.SetPacketInfos(RtpPacketInfos(packet_infos));
return image;
}
VideoFrame DeepCopy(const VideoFrame& frame) {
VideoFrame copy = frame;
copy.set_video_frame_buffer(
I420Buffer::Copy(*frame.video_frame_buffer()->ToI420()));
return copy;
}
void PassFramesThroughAnalyzer(DefaultVideoQualityAnalyzer& analyzer,
absl::string_view sender,
absl::string_view stream_label,
std::vector<absl::string_view> receivers,
int frames_count,
test::FrameGeneratorInterface& frame_generator,
int interframe_delay_ms = 0) {
for (int i = 0; i < frames_count; ++i) {
VideoFrame frame = NextFrame(&frame_generator, /*timestamp_us=*/1);
uint16_t frame_id =
analyzer.OnFrameCaptured(sender, std::string(stream_label), frame);
frame.set_id(frame_id);
analyzer.OnFramePreEncode(sender, frame);
analyzer.OnFrameEncoded(sender, frame.id(), FakeEncode(frame),
VideoQualityAnalyzerInterface::EncoderStats());
for (absl::string_view receiver : receivers) {
VideoFrame received_frame = DeepCopy(frame);
analyzer.OnFramePreDecode(receiver, received_frame.id(),
FakeEncode(received_frame));
analyzer.OnFrameDecoded(receiver, received_frame,
VideoQualityAnalyzerInterface::DecoderStats());
analyzer.OnFrameRendered(receiver, received_frame);
}
if (i < frames_count - 1 && interframe_delay_ms > 0) {
SleepMs(interframe_delay_ms);
}
}
}
// Metric fields to assert on
struct MetricValidationInfo {
std::string test_case;
std::string name;
Unit unit;
ImprovementDirection improvement_direction;
};
bool operator==(const MetricValidationInfo& a, const MetricValidationInfo& b) {
return a.name == b.name && a.test_case == b.test_case && a.unit == b.unit &&
a.improvement_direction == b.improvement_direction;
}
std::ostream& operator<<(std::ostream& os, const MetricValidationInfo& m) {
os << "{ test_case=" << m.test_case << "; name=" << m.name
<< "; unit=" << test::ToString(m.unit)
<< "; improvement_direction=" << test::ToString(m.improvement_direction)
<< " }";
return os;
}
std::vector<MetricValidationInfo> ToValidationInfo(
const std::vector<Metric>& metrics) {
std::vector<MetricValidationInfo> out;
for (const Metric& m : metrics) {
out.push_back(
MetricValidationInfo{.test_case = m.test_case,
.name = m.name,
.unit = m.unit,
.improvement_direction = m.improvement_direction});
}
return out;
}
std::vector<std::string> ToTestCases(const std::vector<Metric>& metrics) {
std::vector<std::string> out;
for (const Metric& m : metrics) {
out.push_back(m.test_case);
}
return out;
}
TEST(DefaultVideoQualityAnalyzerMetricNamesTest, MetricNamesForP2PAreCorrect) {
std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
/*type=*/absl::nullopt,
/*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
std::vector<std::unique_ptr<MetricsExporter>> exporters;
exporters.push_back(std::make_unique<StdoutMetricsExporter>());
MetricsLoggerAndExporter metrics_logger(Clock::GetRealTimeClock(),
std::move(exporters));
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
&metrics_logger, options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount);
PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"bob"},
/*frames_count=*/5, *frame_generator,
/*interframe_delay_ms=*/50);
analyzer.Stop();
std::vector<MetricValidationInfo> metrics =
ToValidationInfo(metrics_logger.GetCollectedMetrics());
EXPECT_THAT(
metrics,
UnorderedElementsAre(
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "psnr_dB",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "ssim",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "transport_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "total_delay_incl_transport",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "time_between_rendered_frames",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "harmonic_framerate",
.unit = Unit::kHertz,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "encode_frame_rate",
.unit = Unit::kHertz,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "encode_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "time_between_freezes",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "freeze_time_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "pixels_per_frame",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "min_psnr_dB",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "decode_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "receive_to_render_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "dropped_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "frames_in_flight",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "rendered_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "max_skipped",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "target_encode_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "actual_encode_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "num_encoded_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "num_decoded_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "num_send_key_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "num_recv_key_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "recv_key_frame_size_bytes",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video",
.name = "recv_delta_frame_size_bytes",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter}));
}
TEST(DefaultVideoQualityAnalyzerMetricNamesTest,
MetricNamesFor3PeersAreCorrect) {
std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
/*type=*/absl::nullopt,
/*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
std::vector<std::unique_ptr<MetricsExporter>> exporters;
exporters.push_back(std::make_unique<StdoutMetricsExporter>());
MetricsLoggerAndExporter metrics_logger(Clock::GetRealTimeClock(),
std::move(exporters));
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
&metrics_logger, options);
analyzer.Start("test_case",
std::vector<std::string>{"alice", "bob", "charlie"},
kAnalyzerMaxThreadsCount);
PassFramesThroughAnalyzer(analyzer, "alice", "alice_video",
{"bob", "charlie"},
/*frames_count=*/5, *frame_generator,
/*interframe_delay_ms=*/50);
analyzer.Stop();
std::vector<MetricValidationInfo> metrics =
ToValidationInfo(metrics_logger.GetCollectedMetrics());
EXPECT_THAT(
metrics,
UnorderedElementsAre(
// Bob
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_bob",
.name = "psnr_dB",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_bob",
.name = "ssim",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_bob",
.name = "transport_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_bob",
.name = "total_delay_incl_transport",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_bob",
.name = "time_between_rendered_frames",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_bob",
.name = "harmonic_framerate",
.unit = Unit::kHertz,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_bob",
.name = "encode_frame_rate",
.unit = Unit::kHertz,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_bob",
.name = "encode_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_bob",
.name = "time_between_freezes",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_bob",
.name = "freeze_time_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_bob",
.name = "pixels_per_frame",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_bob",
.name = "min_psnr_dB",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_bob",
.name = "decode_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_bob",
.name = "receive_to_render_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_bob",
.name = "dropped_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_bob",
.name = "frames_in_flight",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_bob",
.name = "rendered_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_bob",
.name = "max_skipped",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_bob",
.name = "target_encode_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_bob",
.name = "actual_encode_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_bob",
.name = "num_encoded_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_bob",
.name = "num_decoded_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_bob",
.name = "num_send_key_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_bob",
.name = "num_recv_key_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_bob",
.name = "recv_key_frame_size_bytes",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_bob",
.name = "recv_delta_frame_size_bytes",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
// Charlie
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_charlie",
.name = "psnr_dB",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_charlie",
.name = "ssim",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_charlie",
.name = "transport_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_charlie",
.name = "total_delay_incl_transport",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_charlie",
.name = "time_between_rendered_frames",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_charlie",
.name = "harmonic_framerate",
.unit = Unit::kHertz,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_charlie",
.name = "encode_frame_rate",
.unit = Unit::kHertz,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_charlie",
.name = "encode_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_charlie",
.name = "time_between_freezes",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_charlie",
.name = "freeze_time_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_charlie",
.name = "pixels_per_frame",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_charlie",
.name = "min_psnr_dB",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_charlie",
.name = "decode_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_charlie",
.name = "receive_to_render_time",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_charlie",
.name = "dropped_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_charlie",
.name = "frames_in_flight",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_charlie",
.name = "rendered_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_charlie",
.name = "max_skipped",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_charlie",
.name = "target_encode_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_charlie",
.name = "actual_encode_bitrate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_charlie",
.name = "num_encoded_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_charlie",
.name = "num_decoded_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_charlie",
.name = "num_send_key_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_charlie",
.name = "num_recv_key_frames",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_charlie",
.name = "recv_key_frame_size_bytes",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter},
MetricValidationInfo{
.test_case = "test_case/alice_video_alice_charlie",
.name = "recv_delta_frame_size_bytes",
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter}));
}
TEST(DefaultVideoQualityAnalyzerMetricNamesTest,
TestCaseFor3PeerIsTheSameAfterAllPeersLeft) {
std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
/*type=*/absl::nullopt,
/*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
std::vector<std::unique_ptr<MetricsExporter>> exporters;
exporters.push_back(std::make_unique<StdoutMetricsExporter>());
MetricsLoggerAndExporter metrics_logger(Clock::GetRealTimeClock(),
std::move(exporters));
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
&metrics_logger, options);
analyzer.Start("test_case",
std::vector<std::string>{"alice", "bob", "charlie"},
kAnalyzerMaxThreadsCount);
PassFramesThroughAnalyzer(analyzer, "alice", "alice_video",
{"bob", "charlie"},
/*frames_count=*/5, *frame_generator,
/*interframe_delay_ms=*/50);
analyzer.UnregisterParticipantInCall("alice");
analyzer.UnregisterParticipantInCall("bob");
analyzer.UnregisterParticipantInCall("charlie");
analyzer.Stop();
std::vector<std::string> metrics =
ToTestCases(metrics_logger.GetCollectedMetrics());
EXPECT_THAT(metrics, SizeIs(52));
EXPECT_THAT(metrics, Contains("test_case/alice_video_alice_bob").Times(26));
EXPECT_THAT(metrics,
Contains("test_case/alice_video_alice_charlie").Times(26));
}
} // namespace
} // namespace webrtc