Add MetricsSetProtoFileExporter

Bug: b/246095034
Change-Id: I002d0d5b132e61887b4bc87542fbf70dd81e488b
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/275881
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38125}

api/test/metrics/BUILD.gn

@@ -7,6 +7,9 @@
 # be found in the AUTHORS file in the root of the source tree.
 
 import("../../../webrtc.gni")
+if (rtc_enable_protobuf) {
+  import("//third_party/protobuf/proto_library.gni")
+}
 
 group("metrics") {
   deps = [
@@ -25,6 +28,10 @@ if (rtc_include_tests) {
       ":metrics_logger_and_exporter_test",
       ":stdout_metrics_exporter_test",
     ]
+
+    if (rtc_enable_protobuf) {
+      deps += [ ":metrics_set_proto_file_exporter_test" ]
+    }
   }
 }
 
@@ -84,6 +91,35 @@ rtc_library("metrics_logger_and_exporter") {
   ]
 }
 
+if (rtc_enable_protobuf) {
+  proto_library("metric_proto") {
+    visibility = [ "*" ]
+    sources = [ "proto/metric.proto" ]
+    proto_out_dir = "api/test/metrics/proto"
+    cc_generator_options = "lite"
+  }
+}
+
+rtc_library("metrics_set_proto_file_exporter") {
+  visibility = [ "*" ]
+  testonly = true
+  sources = [
+    "metrics_set_proto_file_exporter.cc",
+    "metrics_set_proto_file_exporter.h",
+  ]
+  deps = [
+    ":metric",
+    ":metrics_exporter",
+    "../..:array_view",
+    "../../../rtc_base:logging",
+    "../../../test:fileutils",
+  ]
+
+  if (rtc_enable_protobuf) {
+    deps += [ ":metric_proto" ]
+  }
+}
+
 if (rtc_include_tests) {
   rtc_library("stdout_metrics_exporter_test") {
     testonly = true
@@ -109,4 +145,20 @@ if (rtc_include_tests) {
     ]
     absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
   }
+
+  if (rtc_enable_protobuf) {
+    rtc_library("metrics_set_proto_file_exporter_test") {
+      testonly = true
+      sources = [ "metrics_set_proto_file_exporter_test.cc" ]
+      deps = [
+        ":metric",
+        ":metric_proto",
+        ":metrics_set_proto_file_exporter",
+        "../../../rtc_base:protobuf_utils",
+        "../../../test:fileutils",
+        "../../../test:test_support",
+        "../../units:timestamp",
+      ]
+    }
+  }
 }

api/test/metrics/metrics_set_proto_file_exporter.cc

@@ -0,0 +1,157 @@
+/*
+ *  Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "api/test/metrics/metrics_set_proto_file_exporter.h"
+
+#include <stdio.h>
+
+#include <string>
+
+#include "api/test/metrics/metric.h"
+#include "rtc_base/logging.h"
+#include "test/testsupport/file_utils.h"
+
+#if WEBRTC_ENABLE_PROTOBUF
+#include "api/test/metrics/proto/metric.pb.h"
+#endif
+
+namespace webrtc {
+namespace test {
+namespace {
+
+#if WEBRTC_ENABLE_PROTOBUF
+webrtc::test_metrics::Unit ToProtoUnit(Unit unit) {
+  switch (unit) {
+    case Unit::kTimeMs:
+      return webrtc::test_metrics::Unit::MILLISECONDS;
+    case Unit::kPercent:
+      return webrtc::test_metrics::Unit::PERCENT;
+    case Unit::kSizeInBytes:
+      return webrtc::test_metrics::Unit::BYTES;
+    case Unit::kKilobitsPerSecond:
+      return webrtc::test_metrics::Unit::KILOBITS_PER_SECOND;
+    case Unit::kHertz:
+      return webrtc::test_metrics::Unit::HERTZ;
+    case Unit::kUnitless:
+      return webrtc::test_metrics::Unit::UNITLESS;
+    case Unit::kCount:
+      return webrtc::test_metrics::Unit::COUNT;
+  }
+}
+
+webrtc::test_metrics::ImprovementDirection ToProtoImprovementDirection(
+    ImprovementDirection direction) {
+  switch (direction) {
+    case ImprovementDirection::kBiggerIsBetter:
+      return webrtc::test_metrics::ImprovementDirection::BIGGER_IS_BETTER;
+    case ImprovementDirection::kNeitherIsBetter:
+      return webrtc::test_metrics::ImprovementDirection::NEITHER_IS_BETTER;
+    case ImprovementDirection::kSmallerIsBetter:
+      return webrtc::test_metrics::ImprovementDirection::SMALLER_IS_BETTER;
+  }
+}
+
+void SetTimeSeries(
+    const Metric::TimeSeries& time_series,
+    webrtc::test_metrics::Metric::TimeSeries* proto_time_series) {
+  for (const Metric::TimeSeries::Sample& sample : time_series.samples) {
+    webrtc::test_metrics::Metric::TimeSeries::Sample* proto_sample =
+        proto_time_series->add_samples();
+    proto_sample->set_value(sample.value);
+    proto_sample->set_timestamp_us(sample.timestamp.us());
+    for (const auto& [key, value] : sample.sample_metadata) {
+      proto_sample->mutable_sample_metadata()->insert({key, value});
+    }
+  }
+}
+
+void SetStats(const Metric::Stats& stats,
+              webrtc::test_metrics::Metric::Stats* proto_stats) {
+  if (stats.mean.has_value()) {
+    proto_stats->set_mean(*stats.mean);
+  }
+  if (stats.stddev.has_value()) {
+    proto_stats->set_stddev(*stats.stddev);
+  }
+  if (stats.min.has_value()) {
+    proto_stats->set_min(*stats.min);
+  }
+  if (stats.max.has_value()) {
+    proto_stats->set_max(*stats.max);
+  }
+}
+
+bool WriteMetricsToFile(const std::string& path,
+                        const webrtc::test_metrics::MetricsSet& metrics_set) {
+  std::string data;
+  bool ok = metrics_set.SerializeToString(&data);
+  if (!ok) {
+    RTC_LOG(LS_ERROR) << "Failed to serialize histogram set to string";
+    return false;
+  }
+
+  CreateDir(DirName(path));
+  FILE* output = fopen(path.c_str(), "wb");
+  if (output == NULL) {
+    RTC_LOG(LS_ERROR) << "Failed to write to " << path;
+    return false;
+  }
+  size_t written = fwrite(data.c_str(), sizeof(char), data.size(), output);
+  fclose(output);
+
+  if (written != data.size()) {
+    size_t expected = data.size();
+    RTC_LOG(LS_ERROR) << "Wrote " << written << ", tried to write " << expected;
+    return false;
+  }
+  return true;
+}
+#endif  // WEBRTC_ENABLE_PROTOBUF
+
+}  // namespace
+
+MetricsSetProtoFileExporter::Options::Options(
+    absl::string_view export_file_path)
+    : export_file_path(export_file_path) {}
+MetricsSetProtoFileExporter::Options::Options(
+    absl::string_view export_file_path,
+    bool export_whole_time_series)
+    : export_file_path(export_file_path),
+      export_whole_time_series(export_whole_time_series) {}
+
+bool MetricsSetProtoFileExporter::Export(rtc::ArrayView<const Metric> metrics) {
+#if WEBRTC_ENABLE_PROTOBUF
+  webrtc::test_metrics::MetricsSet metrics_set;
+  for (const Metric& metric : metrics) {
+    webrtc::test_metrics::Metric* metric_proto = metrics_set.add_metrics();
+    metric_proto->set_name(metric.name);
+    metric_proto->set_unit(ToProtoUnit(metric.unit));
+    metric_proto->set_improvement_direction(
+        ToProtoImprovementDirection(metric.improvement_direction));
+    metric_proto->set_test_case(metric.test_case);
+    for (const auto& [key, value] : metric.metric_metadata) {
+      metric_proto->mutable_metric_metadata()->insert({key, value});
+    }
+
+    if (options_.export_whole_time_series) {
+      SetTimeSeries(metric.time_series, metric_proto->mutable_time_series());
+    }
+    SetStats(metric.stats, metric_proto->mutable_stats());
+  }
+
+  return WriteMetricsToFile(options_.export_file_path, metrics_set);
+#else
+  RTC_LOG(LS_ERROR)
+      << "Compile with protobuf support to properly use this class";
+  return false;
+#endif
+}
+
+}  // namespace test
+}  // namespace webrtc

api/test/metrics/metrics_set_proto_file_exporter.h

@@ -0,0 +1,54 @@
+/*
+ *  Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_TEST_METRICS_METRICS_SET_PROTO_FILE_EXPORTER_H_
+#define API_TEST_METRICS_METRICS_SET_PROTO_FILE_EXPORTER_H_
+
+#include <string>
+
+#include "api/array_view.h"
+#include "api/test/metrics/metric.h"
+#include "api/test/metrics/metrics_exporter.h"
+
+namespace webrtc {
+namespace test {
+
+// Exports all collected metrics to the proto file using
+// `webrtc::test_metrics::MetricsSet` format.
+class MetricsSetProtoFileExporter : public MetricsExporter {
+ public:
+  struct Options {
+    explicit Options(absl::string_view export_file_path);
+    Options(absl::string_view export_file_path, bool export_whole_time_series);
+
+    // File to export proto.
+    std::string export_file_path;
+    // If true will write all time series values to the output proto file,
+    // otherwise will write stats only.
+    bool export_whole_time_series = true;
+  };
+
+  explicit MetricsSetProtoFileExporter(const Options& options)
+      : options_(options) {}
+
+  MetricsSetProtoFileExporter(const MetricsSetProtoFileExporter&) = delete;
+  MetricsSetProtoFileExporter& operator=(const MetricsSetProtoFileExporter&) =
+      delete;
+
+  bool Export(rtc::ArrayView<const Metric> metrics) override;
+
+ private:
+  const Options options_;
+};
+
+}  // namespace test
+}  // namespace webrtc
+
+#endif  // API_TEST_METRICS_METRICS_SET_PROTO_FILE_EXPORTER_H_

api/test/metrics/metrics_set_proto_file_exporter_test.cc

@@ -0,0 +1,151 @@
+/*
+ *  Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "api/test/metrics/metrics_set_proto_file_exporter.h"
+
+#include <fstream>
+#include <map>
+#include <string>
+#include <vector>
+
+#include "api/test/metrics/metric.h"
+#include "api/test/metrics/proto/metric.pb.h"
+#include "api/units/timestamp.h"
+#include "rtc_base/protobuf_utils.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+#include "test/testsupport/file_utils.h"
+
+namespace webrtc {
+namespace test {
+namespace {
+
+using ::testing::Eq;
+using ::testing::Test;
+
+namespace proto = ::webrtc::test_metrics;
+
+std::string ReadFileAsString(const std::string& filename) {
+  std::ifstream infile(filename, std::ios_base::binary);
+  auto buffer = std::vector<char>(std::istreambuf_iterator<char>(infile),
+                                  std::istreambuf_iterator<char>());
+  return std::string(buffer.begin(), buffer.end());
+}
+
+std::map<std::string, std::string> DefaultMetadata() {
+  return std::map<std::string, std::string>{{"key", "value"}};
+}
+
+Metric::TimeSeries::Sample Sample(double value) {
+  return Metric::TimeSeries::Sample{.timestamp = Timestamp::Seconds(1),
+                                    .value = value,
+                                    .sample_metadata = DefaultMetadata()};
+}
+
+void AssertSamplesEqual(const proto::Metric::TimeSeries::Sample& actual_sample,
+                        const Metric::TimeSeries::Sample& expected_sample) {
+  EXPECT_THAT(actual_sample.value(), Eq(expected_sample.value));
+  EXPECT_THAT(actual_sample.timestamp_us(), Eq(expected_sample.timestamp.us()));
+  EXPECT_THAT(actual_sample.sample_metadata().size(),
+              Eq(expected_sample.sample_metadata.size()));
+  for (const auto& [key, value] : expected_sample.sample_metadata) {
+    EXPECT_THAT(actual_sample.sample_metadata().at(key), Eq(value));
+  }
+}
+
+class MetricsSetProtoFileExporterTest : public Test {
+ protected:
+  ~MetricsSetProtoFileExporterTest() override = default;
+
+  void SetUp() override {
+    temp_filename_ = webrtc::test::TempFilename(
+        webrtc::test::OutputPath(), "metrics_set_proto_file_exporter_test");
+  }
+
+  void TearDown() override {
+    ASSERT_TRUE(webrtc::test::RemoveFile(temp_filename_));
+  }
+
+  std::string temp_filename_;
+};
+
+TEST_F(MetricsSetProtoFileExporterTest, MetricsAreExportedCorrectly) {
+  MetricsSetProtoFileExporter::Options options(temp_filename_);
+  MetricsSetProtoFileExporter exporter(options);
+
+  Metric metric1{
+      .name = "test_metric1",
+      .unit = Unit::kTimeMs,
+      .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+      .test_case = "test_case_name1",
+      .metric_metadata = DefaultMetadata(),
+      .time_series =
+          Metric::TimeSeries{.samples = std::vector{Sample(10), Sample(20)}},
+      .stats =
+          Metric::Stats{.mean = 15.0, .stddev = 5.0, .min = 10.0, .max = 20.0}};
+  Metric metric2{
+      .name = "test_metric2",
+      .unit = Unit::kKilobitsPerSecond,
+      .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+      .test_case = "test_case_name2",
+      .metric_metadata = DefaultMetadata(),
+      .time_series =
+          Metric::TimeSeries{.samples = std::vector{Sample(20), Sample(40)}},
+      .stats = Metric::Stats{
+          .mean = 30.0, .stddev = 10.0, .min = 20.0, .max = 40.0}};
+
+  ASSERT_TRUE(exporter.Export(std::vector<Metric>{metric1, metric2}));
+  webrtc::test_metrics::MetricsSet actual_metrics_set;
+  actual_metrics_set.ParseFromString(ReadFileAsString(temp_filename_));
+  EXPECT_THAT(actual_metrics_set.metrics().size(), Eq(2));
+
+  EXPECT_THAT(actual_metrics_set.metrics(0).name(), Eq("test_metric1"));
+  EXPECT_THAT(actual_metrics_set.metrics(0).test_case(), Eq("test_case_name1"));
+  EXPECT_THAT(actual_metrics_set.metrics(0).unit(),
+              Eq(proto::Unit::MILLISECONDS));
+  EXPECT_THAT(actual_metrics_set.metrics(0).improvement_direction(),
+              Eq(proto::ImprovementDirection::BIGGER_IS_BETTER));
+  EXPECT_THAT(actual_metrics_set.metrics(0).metric_metadata().size(), Eq(1lu));
+  EXPECT_THAT(actual_metrics_set.metrics(0).metric_metadata().at("key"),
+              Eq("value"));
+  EXPECT_THAT(actual_metrics_set.metrics(0).time_series().samples().size(),
+              Eq(2));
+  AssertSamplesEqual(actual_metrics_set.metrics(0).time_series().samples(0),
+                     Sample(10.0));
+  AssertSamplesEqual(actual_metrics_set.metrics(0).time_series().samples(1),
+                     Sample(20.0));
+  EXPECT_THAT(actual_metrics_set.metrics(0).stats().mean(), Eq(15.0));
+  EXPECT_THAT(actual_metrics_set.metrics(0).stats().stddev(), Eq(5.0));
+  EXPECT_THAT(actual_metrics_set.metrics(0).stats().min(), Eq(10.0));
+  EXPECT_THAT(actual_metrics_set.metrics(0).stats().max(), Eq(20.0));
+
+  EXPECT_THAT(actual_metrics_set.metrics(1).name(), Eq("test_metric2"));
+  EXPECT_THAT(actual_metrics_set.metrics(1).test_case(), Eq("test_case_name2"));
+  EXPECT_THAT(actual_metrics_set.metrics(1).unit(),
+              Eq(proto::Unit::KILOBITS_PER_SECOND));
+  EXPECT_THAT(actual_metrics_set.metrics(1).improvement_direction(),
+              Eq(proto::ImprovementDirection::SMALLER_IS_BETTER));
+  EXPECT_THAT(actual_metrics_set.metrics(1).metric_metadata().size(), Eq(1lu));
+  EXPECT_THAT(actual_metrics_set.metrics(1).metric_metadata().at("key"),
+              Eq("value"));
+  EXPECT_THAT(actual_metrics_set.metrics(1).time_series().samples().size(),
+              Eq(2));
+  AssertSamplesEqual(actual_metrics_set.metrics(1).time_series().samples(0),
+                     Sample(20.0));
+  AssertSamplesEqual(actual_metrics_set.metrics(1).time_series().samples(1),
+                     Sample(40.0));
+  EXPECT_THAT(actual_metrics_set.metrics(1).stats().mean(), Eq(30.0));
+  EXPECT_THAT(actual_metrics_set.metrics(1).stats().stddev(), Eq(10.0));
+  EXPECT_THAT(actual_metrics_set.metrics(1).stats().min(), Eq(20.0));
+  EXPECT_THAT(actual_metrics_set.metrics(1).stats().max(), Eq(40.0));
+}
+
+}  // namespace
+}  // namespace test
+}  // namespace webrtc

api/test/metrics/proto/metric.proto

@@ -0,0 +1,87 @@
+/*
+ *  Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+syntax = "proto3";
+
+package webrtc.test_metrics;
+
+// Root message of the proto file. Contains collection of all the metrics.
+message MetricsSet {
+  repeated Metric metrics = 1;
+}
+
+enum Unit {
+  // Default value that has to be defined.
+  UNDEFINED_UNIT = 0;
+  // General unitless value. Can be used either for dimensionless quantities
+  // (ex ratio) or for units not presented in this enum and too specific to add
+  // to this enum.
+  UNITLESS = 1;
+  MILLISECONDS = 2;
+  PERCENT = 3;
+  BYTES = 4;
+  KILOBITS_PER_SECOND = 5;
+  HERTZ = 6;
+  COUNT = 7;
+}
+
+enum ImprovementDirection {
+  // Default value that has to be defined.
+  UNDEFINED_IMPROVEMENT_DIRECTION = 0;
+  BIGGER_IS_BETTER = 1;
+  NEITHER_IS_BETTER = 2;
+  SMALLER_IS_BETTER = 3;
+}
+
+// Single performance metric with all related metadata.
+message Metric {
+  // Metric name, for example PSNR, SSIM, decode_time, etc.
+  string name = 1;
+  Unit unit = 2;
+  ImprovementDirection improvement_direction = 3;
+  // If the metric is generated by a test, this field can be used to specify
+  // this information.
+  string test_case = 4;
+  // Metadata associated with the whole metric.
+  map<string, string> metric_metadata = 5;
+
+  message TimeSeries {
+    message Sample {
+      // Timestamp in microseconds associated with a sample. For example,
+      // the timestamp when the sample was collected.
+      int64 timestamp_us = 1;
+      double value = 2;
+      // Metadata associated with this particular sample.
+      map<string, string> sample_metadata = 3;
+    }
+    // All samples collected for this metric. It can be empty if the Metric
+    // object only contains `stats`.
+    repeated Sample samples = 1;
+  }
+  // Contains all samples of the metric collected during test execution.
+  // It can be empty if the user only stores precomputed statistics into
+  // `stats`.
+  TimeSeries time_series = 6;
+
+  // Contains metric's precomputed statistics based on the `time_series` or if
+  // `time_series` is omitted (has 0 samples) contains precomputed statistics
+  // provided by the metric's calculator.
+  message Stats {
+    // Sample mean of the metric
+    // (https://en.wikipedia.org/wiki/Sample_mean_and_covariance).
+    optional double mean = 1;
+    // Standard deviation (https://en.wikipedia.org/wiki/Standard_deviation).
+    // Is undefined if `time_series` contains only a single sample.
+    optional double stddev = 2;
+    optional double min = 3;
+    optional double max = 4;
+  }
+  Stats stats = 7;
+}