From 275d63a13e4be740b807ab6fbeeca72a53da2238 Mon Sep 17 00:00:00 2001 From: Artem Titov Date: Tue, 20 Sep 2022 10:27:36 +0200 Subject: [PATCH] Add MetricsSetProtoFileExporter Bug: b/246095034 Change-Id: I002d0d5b132e61887b4bc87542fbf70dd81e488b Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/275881 Commit-Queue: Artem Titov Reviewed-by: Mirko Bonadei Cr-Commit-Position: refs/heads/main@{#38125} --- api/test/metrics/BUILD.gn | 52 ++++++ .../metrics_set_proto_file_exporter.cc | 157 ++++++++++++++++++ .../metrics/metrics_set_proto_file_exporter.h | 54 ++++++ .../metrics_set_proto_file_exporter_test.cc | 151 +++++++++++++++++ api/test/metrics/proto/metric.proto | 87 ++++++++++ 5 files changed, 501 insertions(+) create mode 100644 api/test/metrics/metrics_set_proto_file_exporter.cc create mode 100644 api/test/metrics/metrics_set_proto_file_exporter.h create mode 100644 api/test/metrics/metrics_set_proto_file_exporter_test.cc create mode 100644 api/test/metrics/proto/metric.proto diff --git a/api/test/metrics/BUILD.gn b/api/test/metrics/BUILD.gn index 544808ffa7..8bfcf39c1f 100644 --- a/api/test/metrics/BUILD.gn +++ b/api/test/metrics/BUILD.gn @@ -7,6 +7,9 @@ # be found in the AUTHORS file in the root of the source tree. import("../../../webrtc.gni") +if (rtc_enable_protobuf) { + import("//third_party/protobuf/proto_library.gni") +} group("metrics") { deps = [ @@ -25,6 +28,10 @@ if (rtc_include_tests) { ":metrics_logger_and_exporter_test", ":stdout_metrics_exporter_test", ] + + if (rtc_enable_protobuf) { + deps += [ ":metrics_set_proto_file_exporter_test" ] + } } } @@ -84,6 +91,35 @@ rtc_library("metrics_logger_and_exporter") { ] } +if (rtc_enable_protobuf) { + proto_library("metric_proto") { + visibility = [ "*" ] + sources = [ "proto/metric.proto" ] + proto_out_dir = "api/test/metrics/proto" + cc_generator_options = "lite" + } +} + +rtc_library("metrics_set_proto_file_exporter") { + visibility = [ "*" ] + testonly = true + sources = [ + "metrics_set_proto_file_exporter.cc", + "metrics_set_proto_file_exporter.h", + ] + deps = [ + ":metric", + ":metrics_exporter", + "../..:array_view", + "../../../rtc_base:logging", + "../../../test:fileutils", + ] + + if (rtc_enable_protobuf) { + deps += [ ":metric_proto" ] + } +} + if (rtc_include_tests) { rtc_library("stdout_metrics_exporter_test") { testonly = true @@ -109,4 +145,20 @@ if (rtc_include_tests) { ] absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } + + if (rtc_enable_protobuf) { + rtc_library("metrics_set_proto_file_exporter_test") { + testonly = true + sources = [ "metrics_set_proto_file_exporter_test.cc" ] + deps = [ + ":metric", + ":metric_proto", + ":metrics_set_proto_file_exporter", + "../../../rtc_base:protobuf_utils", + "../../../test:fileutils", + "../../../test:test_support", + "../../units:timestamp", + ] + } + } } diff --git a/api/test/metrics/metrics_set_proto_file_exporter.cc b/api/test/metrics/metrics_set_proto_file_exporter.cc new file mode 100644 index 0000000000..6a41098375 --- /dev/null +++ b/api/test/metrics/metrics_set_proto_file_exporter.cc @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/test/metrics/metrics_set_proto_file_exporter.h" + +#include + +#include + +#include "api/test/metrics/metric.h" +#include "rtc_base/logging.h" +#include "test/testsupport/file_utils.h" + +#if WEBRTC_ENABLE_PROTOBUF +#include "api/test/metrics/proto/metric.pb.h" +#endif + +namespace webrtc { +namespace test { +namespace { + +#if WEBRTC_ENABLE_PROTOBUF +webrtc::test_metrics::Unit ToProtoUnit(Unit unit) { + switch (unit) { + case Unit::kTimeMs: + return webrtc::test_metrics::Unit::MILLISECONDS; + case Unit::kPercent: + return webrtc::test_metrics::Unit::PERCENT; + case Unit::kSizeInBytes: + return webrtc::test_metrics::Unit::BYTES; + case Unit::kKilobitsPerSecond: + return webrtc::test_metrics::Unit::KILOBITS_PER_SECOND; + case Unit::kHertz: + return webrtc::test_metrics::Unit::HERTZ; + case Unit::kUnitless: + return webrtc::test_metrics::Unit::UNITLESS; + case Unit::kCount: + return webrtc::test_metrics::Unit::COUNT; + } +} + +webrtc::test_metrics::ImprovementDirection ToProtoImprovementDirection( + ImprovementDirection direction) { + switch (direction) { + case ImprovementDirection::kBiggerIsBetter: + return webrtc::test_metrics::ImprovementDirection::BIGGER_IS_BETTER; + case ImprovementDirection::kNeitherIsBetter: + return webrtc::test_metrics::ImprovementDirection::NEITHER_IS_BETTER; + case ImprovementDirection::kSmallerIsBetter: + return webrtc::test_metrics::ImprovementDirection::SMALLER_IS_BETTER; + } +} + +void SetTimeSeries( + const Metric::TimeSeries& time_series, + webrtc::test_metrics::Metric::TimeSeries* proto_time_series) { + for (const Metric::TimeSeries::Sample& sample : time_series.samples) { + webrtc::test_metrics::Metric::TimeSeries::Sample* proto_sample = + proto_time_series->add_samples(); + proto_sample->set_value(sample.value); + proto_sample->set_timestamp_us(sample.timestamp.us()); + for (const auto& [key, value] : sample.sample_metadata) { + proto_sample->mutable_sample_metadata()->insert({key, value}); + } + } +} + +void SetStats(const Metric::Stats& stats, + webrtc::test_metrics::Metric::Stats* proto_stats) { + if (stats.mean.has_value()) { + proto_stats->set_mean(*stats.mean); + } + if (stats.stddev.has_value()) { + proto_stats->set_stddev(*stats.stddev); + } + if (stats.min.has_value()) { + proto_stats->set_min(*stats.min); + } + if (stats.max.has_value()) { + proto_stats->set_max(*stats.max); + } +} + +bool WriteMetricsToFile(const std::string& path, + const webrtc::test_metrics::MetricsSet& metrics_set) { + std::string data; + bool ok = metrics_set.SerializeToString(&data); + if (!ok) { + RTC_LOG(LS_ERROR) << "Failed to serialize histogram set to string"; + return false; + } + + CreateDir(DirName(path)); + FILE* output = fopen(path.c_str(), "wb"); + if (output == NULL) { + RTC_LOG(LS_ERROR) << "Failed to write to " << path; + return false; + } + size_t written = fwrite(data.c_str(), sizeof(char), data.size(), output); + fclose(output); + + if (written != data.size()) { + size_t expected = data.size(); + RTC_LOG(LS_ERROR) << "Wrote " << written << ", tried to write " << expected; + return false; + } + return true; +} +#endif // WEBRTC_ENABLE_PROTOBUF + +} // namespace + +MetricsSetProtoFileExporter::Options::Options( + absl::string_view export_file_path) + : export_file_path(export_file_path) {} +MetricsSetProtoFileExporter::Options::Options( + absl::string_view export_file_path, + bool export_whole_time_series) + : export_file_path(export_file_path), + export_whole_time_series(export_whole_time_series) {} + +bool MetricsSetProtoFileExporter::Export(rtc::ArrayView metrics) { +#if WEBRTC_ENABLE_PROTOBUF + webrtc::test_metrics::MetricsSet metrics_set; + for (const Metric& metric : metrics) { + webrtc::test_metrics::Metric* metric_proto = metrics_set.add_metrics(); + metric_proto->set_name(metric.name); + metric_proto->set_unit(ToProtoUnit(metric.unit)); + metric_proto->set_improvement_direction( + ToProtoImprovementDirection(metric.improvement_direction)); + metric_proto->set_test_case(metric.test_case); + for (const auto& [key, value] : metric.metric_metadata) { + metric_proto->mutable_metric_metadata()->insert({key, value}); + } + + if (options_.export_whole_time_series) { + SetTimeSeries(metric.time_series, metric_proto->mutable_time_series()); + } + SetStats(metric.stats, metric_proto->mutable_stats()); + } + + return WriteMetricsToFile(options_.export_file_path, metrics_set); +#else + RTC_LOG(LS_ERROR) + << "Compile with protobuf support to properly use this class"; + return false; +#endif +} + +} // namespace test +} // namespace webrtc diff --git a/api/test/metrics/metrics_set_proto_file_exporter.h b/api/test/metrics/metrics_set_proto_file_exporter.h new file mode 100644 index 0000000000..f996e9e7b0 --- /dev/null +++ b/api/test/metrics/metrics_set_proto_file_exporter.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_METRICS_METRICS_SET_PROTO_FILE_EXPORTER_H_ +#define API_TEST_METRICS_METRICS_SET_PROTO_FILE_EXPORTER_H_ + +#include + +#include "api/array_view.h" +#include "api/test/metrics/metric.h" +#include "api/test/metrics/metrics_exporter.h" + +namespace webrtc { +namespace test { + +// Exports all collected metrics to the proto file using +// `webrtc::test_metrics::MetricsSet` format. +class MetricsSetProtoFileExporter : public MetricsExporter { + public: + struct Options { + explicit Options(absl::string_view export_file_path); + Options(absl::string_view export_file_path, bool export_whole_time_series); + + // File to export proto. + std::string export_file_path; + // If true will write all time series values to the output proto file, + // otherwise will write stats only. + bool export_whole_time_series = true; + }; + + explicit MetricsSetProtoFileExporter(const Options& options) + : options_(options) {} + + MetricsSetProtoFileExporter(const MetricsSetProtoFileExporter&) = delete; + MetricsSetProtoFileExporter& operator=(const MetricsSetProtoFileExporter&) = + delete; + + bool Export(rtc::ArrayView metrics) override; + + private: + const Options options_; +}; + +} // namespace test +} // namespace webrtc + +#endif // API_TEST_METRICS_METRICS_SET_PROTO_FILE_EXPORTER_H_ diff --git a/api/test/metrics/metrics_set_proto_file_exporter_test.cc b/api/test/metrics/metrics_set_proto_file_exporter_test.cc new file mode 100644 index 0000000000..97987d228a --- /dev/null +++ b/api/test/metrics/metrics_set_proto_file_exporter_test.cc @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "api/test/metrics/metrics_set_proto_file_exporter.h" + +#include +#include +#include +#include + +#include "api/test/metrics/metric.h" +#include "api/test/metrics/proto/metric.pb.h" +#include "api/units/timestamp.h" +#include "rtc_base/protobuf_utils.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { +namespace test { +namespace { + +using ::testing::Eq; +using ::testing::Test; + +namespace proto = ::webrtc::test_metrics; + +std::string ReadFileAsString(const std::string& filename) { + std::ifstream infile(filename, std::ios_base::binary); + auto buffer = std::vector(std::istreambuf_iterator(infile), + std::istreambuf_iterator()); + return std::string(buffer.begin(), buffer.end()); +} + +std::map DefaultMetadata() { + return std::map{{"key", "value"}}; +} + +Metric::TimeSeries::Sample Sample(double value) { + return Metric::TimeSeries::Sample{.timestamp = Timestamp::Seconds(1), + .value = value, + .sample_metadata = DefaultMetadata()}; +} + +void AssertSamplesEqual(const proto::Metric::TimeSeries::Sample& actual_sample, + const Metric::TimeSeries::Sample& expected_sample) { + EXPECT_THAT(actual_sample.value(), Eq(expected_sample.value)); + EXPECT_THAT(actual_sample.timestamp_us(), Eq(expected_sample.timestamp.us())); + EXPECT_THAT(actual_sample.sample_metadata().size(), + Eq(expected_sample.sample_metadata.size())); + for (const auto& [key, value] : expected_sample.sample_metadata) { + EXPECT_THAT(actual_sample.sample_metadata().at(key), Eq(value)); + } +} + +class MetricsSetProtoFileExporterTest : public Test { + protected: + ~MetricsSetProtoFileExporterTest() override = default; + + void SetUp() override { + temp_filename_ = webrtc::test::TempFilename( + webrtc::test::OutputPath(), "metrics_set_proto_file_exporter_test"); + } + + void TearDown() override { + ASSERT_TRUE(webrtc::test::RemoveFile(temp_filename_)); + } + + std::string temp_filename_; +}; + +TEST_F(MetricsSetProtoFileExporterTest, MetricsAreExportedCorrectly) { + MetricsSetProtoFileExporter::Options options(temp_filename_); + MetricsSetProtoFileExporter exporter(options); + + Metric metric1{ + .name = "test_metric1", + .unit = Unit::kTimeMs, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .test_case = "test_case_name1", + .metric_metadata = DefaultMetadata(), + .time_series = + Metric::TimeSeries{.samples = std::vector{Sample(10), Sample(20)}}, + .stats = + Metric::Stats{.mean = 15.0, .stddev = 5.0, .min = 10.0, .max = 20.0}}; + Metric metric2{ + .name = "test_metric2", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .test_case = "test_case_name2", + .metric_metadata = DefaultMetadata(), + .time_series = + Metric::TimeSeries{.samples = std::vector{Sample(20), Sample(40)}}, + .stats = Metric::Stats{ + .mean = 30.0, .stddev = 10.0, .min = 20.0, .max = 40.0}}; + + ASSERT_TRUE(exporter.Export(std::vector{metric1, metric2})); + webrtc::test_metrics::MetricsSet actual_metrics_set; + actual_metrics_set.ParseFromString(ReadFileAsString(temp_filename_)); + EXPECT_THAT(actual_metrics_set.metrics().size(), Eq(2)); + + EXPECT_THAT(actual_metrics_set.metrics(0).name(), Eq("test_metric1")); + EXPECT_THAT(actual_metrics_set.metrics(0).test_case(), Eq("test_case_name1")); + EXPECT_THAT(actual_metrics_set.metrics(0).unit(), + Eq(proto::Unit::MILLISECONDS)); + EXPECT_THAT(actual_metrics_set.metrics(0).improvement_direction(), + Eq(proto::ImprovementDirection::BIGGER_IS_BETTER)); + EXPECT_THAT(actual_metrics_set.metrics(0).metric_metadata().size(), Eq(1lu)); + EXPECT_THAT(actual_metrics_set.metrics(0).metric_metadata().at("key"), + Eq("value")); + EXPECT_THAT(actual_metrics_set.metrics(0).time_series().samples().size(), + Eq(2)); + AssertSamplesEqual(actual_metrics_set.metrics(0).time_series().samples(0), + Sample(10.0)); + AssertSamplesEqual(actual_metrics_set.metrics(0).time_series().samples(1), + Sample(20.0)); + EXPECT_THAT(actual_metrics_set.metrics(0).stats().mean(), Eq(15.0)); + EXPECT_THAT(actual_metrics_set.metrics(0).stats().stddev(), Eq(5.0)); + EXPECT_THAT(actual_metrics_set.metrics(0).stats().min(), Eq(10.0)); + EXPECT_THAT(actual_metrics_set.metrics(0).stats().max(), Eq(20.0)); + + EXPECT_THAT(actual_metrics_set.metrics(1).name(), Eq("test_metric2")); + EXPECT_THAT(actual_metrics_set.metrics(1).test_case(), Eq("test_case_name2")); + EXPECT_THAT(actual_metrics_set.metrics(1).unit(), + Eq(proto::Unit::KILOBITS_PER_SECOND)); + EXPECT_THAT(actual_metrics_set.metrics(1).improvement_direction(), + Eq(proto::ImprovementDirection::SMALLER_IS_BETTER)); + EXPECT_THAT(actual_metrics_set.metrics(1).metric_metadata().size(), Eq(1lu)); + EXPECT_THAT(actual_metrics_set.metrics(1).metric_metadata().at("key"), + Eq("value")); + EXPECT_THAT(actual_metrics_set.metrics(1).time_series().samples().size(), + Eq(2)); + AssertSamplesEqual(actual_metrics_set.metrics(1).time_series().samples(0), + Sample(20.0)); + AssertSamplesEqual(actual_metrics_set.metrics(1).time_series().samples(1), + Sample(40.0)); + EXPECT_THAT(actual_metrics_set.metrics(1).stats().mean(), Eq(30.0)); + EXPECT_THAT(actual_metrics_set.metrics(1).stats().stddev(), Eq(10.0)); + EXPECT_THAT(actual_metrics_set.metrics(1).stats().min(), Eq(20.0)); + EXPECT_THAT(actual_metrics_set.metrics(1).stats().max(), Eq(40.0)); +} + +} // namespace +} // namespace test +} // namespace webrtc diff --git a/api/test/metrics/proto/metric.proto b/api/test/metrics/proto/metric.proto new file mode 100644 index 0000000000..cdd312589d --- /dev/null +++ b/api/test/metrics/proto/metric.proto @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +syntax = "proto3"; + +package webrtc.test_metrics; + +// Root message of the proto file. Contains collection of all the metrics. +message MetricsSet { + repeated Metric metrics = 1; +} + +enum Unit { + // Default value that has to be defined. + UNDEFINED_UNIT = 0; + // General unitless value. Can be used either for dimensionless quantities + // (ex ratio) or for units not presented in this enum and too specific to add + // to this enum. + UNITLESS = 1; + MILLISECONDS = 2; + PERCENT = 3; + BYTES = 4; + KILOBITS_PER_SECOND = 5; + HERTZ = 6; + COUNT = 7; +} + +enum ImprovementDirection { + // Default value that has to be defined. + UNDEFINED_IMPROVEMENT_DIRECTION = 0; + BIGGER_IS_BETTER = 1; + NEITHER_IS_BETTER = 2; + SMALLER_IS_BETTER = 3; +} + +// Single performance metric with all related metadata. +message Metric { + // Metric name, for example PSNR, SSIM, decode_time, etc. + string name = 1; + Unit unit = 2; + ImprovementDirection improvement_direction = 3; + // If the metric is generated by a test, this field can be used to specify + // this information. + string test_case = 4; + // Metadata associated with the whole metric. + map metric_metadata = 5; + + message TimeSeries { + message Sample { + // Timestamp in microseconds associated with a sample. For example, + // the timestamp when the sample was collected. + int64 timestamp_us = 1; + double value = 2; + // Metadata associated with this particular sample. + map sample_metadata = 3; + } + // All samples collected for this metric. It can be empty if the Metric + // object only contains `stats`. + repeated Sample samples = 1; + } + // Contains all samples of the metric collected during test execution. + // It can be empty if the user only stores precomputed statistics into + // `stats`. + TimeSeries time_series = 6; + + // Contains metric's precomputed statistics based on the `time_series` or if + // `time_series` is omitted (has 0 samples) contains precomputed statistics + // provided by the metric's calculator. + message Stats { + // Sample mean of the metric + // (https://en.wikipedia.org/wiki/Sample_mean_and_covariance). + optional double mean = 1; + // Standard deviation (https://en.wikipedia.org/wiki/Standard_deviation). + // Is undefined if `time_series` contains only a single sample. + optional double stddev = 2; + optional double min = 3; + optional double max = 4; + } + Stats stats = 7; +}