Introduce QualityMetricsReporter and implement network stats gathering

QualityMetricsReporter helps to keep network emulation framework and
peer connection level test framework separated. Also it provides
ability to gather statistics from any component around with
correlation with call start and end.

Bug: webrtc:10138
Change-Id: Ib3330a8d35481fde77fcf77d2271d6cfcf188fec
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/132718
Reviewed-by: Karl Wiberg <kwiberg@webrtc.org>
Reviewed-by: Peter Slatala <psla@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#27759}

api/test/peerconnection_quality_test_fixture.h

@@ -193,6 +193,20 @@ class PeerConnectionE2EQualityTestFixture {
     double video_encoder_bitrate_multiplier = 1.0;
   };
 
+  // Represent an entity that will report quality metrics after test.
+  class QualityMetricsReporter {
+   public:
+    virtual ~QualityMetricsReporter() = default;
+
+    // Invoked by framework after peer connection factory and peer connection
+    // itself will be created but before offer/answer exchange will be started.
+    virtual void Start(absl::string_view test_case_name) = 0;
+
+    // Invoked by framework after call is ended and peer connection factory and
+    // peer connection are destroyed.
+    virtual void StopAndReportResults() = 0;
+  };
+
   virtual ~PeerConnectionE2EQualityTestFixture() = default;
 
   // Add activity that will be executed on the best effort at least after
@@ -209,6 +223,10 @@ class PeerConnectionE2EQualityTestFixture {
                             TimeDelta interval,
                             std::function<void(TimeDelta)> func) = 0;
 
+  // Add stats reporter entity to observe the test.
+  virtual void AddQualityMetricsReporter(
+      std::unique_ptr<QualityMetricsReporter> quality_metrics_reporter) = 0;
+
   // Add a new peer to the call and return an object through which caller
   // can configure peer's behavior.
   // |network_thread| will be used as network thread for peer's peer connection

test/pc/e2e/BUILD.gn

@@ -312,6 +312,7 @@ if (rtc_include_tests) {
     deps = [
       ":default_audio_quality_analyzer",
       ":default_video_quality_analyzer",
+      ":network_quality_metrics_reporter",
       "../../../api:callfactory_api",
       "../../../api:create_network_emulation_manager",
       "../../../api:create_peerconnection_quality_test_fixture",
@@ -436,3 +437,18 @@ rtc_source_set("default_video_quality_analyzer") {
     "//third_party/abseil-cpp/absl/memory",
   ]
 }
+
+rtc_source_set("network_quality_metrics_reporter") {
+  visibility = [ "*" ]
+  testonly = true
+  sources = [
+    "network_quality_metrics_reporter.cc",
+    "network_quality_metrics_reporter.h",
+  ]
+  deps = [
+    "../..:perf_test",
+    "../../../api:network_emulation_manager_api",
+    "../../../api:peer_connection_quality_test_fixture_api",
+    "../../../rtc_base:rtc_event",
+  ]
+}

test/pc/e2e/network_quality_metrics_reporter.cc

@@ -0,0 +1,96 @@
+/*
+ *  Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "test/pc/e2e/network_quality_metrics_reporter.h"
+
+#include <utility>
+
+#include "rtc_base/event.h"
+#include "test/testsupport/perf_test.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+namespace {
+
+constexpr int kStatsWaitTimeoutMs = 1000;
+
+}
+
+void NetworkQualityMetricsReporter::Start(absl::string_view test_case_name) {
+  test_case_name_ = std::string(test_case_name);
+  // Check that network stats are clean before test execution.
+  EmulatedNetworkStats alice_stats = PopulateStats(alice_network_);
+  RTC_CHECK_EQ(alice_stats.packets_sent, 0);
+  RTC_CHECK_EQ(alice_stats.packets_received, 0);
+  EmulatedNetworkStats bob_stats = PopulateStats(bob_network_);
+  RTC_CHECK_EQ(bob_stats.packets_sent, 0);
+  RTC_CHECK_EQ(bob_stats.packets_received, 0);
+}
+
+void NetworkQualityMetricsReporter::StopAndReportResults() {
+  EmulatedNetworkStats alice_stats = PopulateStats(alice_network_);
+  EmulatedNetworkStats bob_stats = PopulateStats(bob_network_);
+  ReportStats("alice", alice_stats,
+              alice_stats.packets_sent - bob_stats.packets_received);
+  ReportStats("bob", bob_stats,
+              bob_stats.packets_sent - alice_stats.packets_received);
+}
+
+EmulatedNetworkStats NetworkQualityMetricsReporter::PopulateStats(
+    EmulatedNetworkManagerInterface* network) {
+  rtc::Event wait;
+  EmulatedNetworkStats stats;
+  network->GetStats([&](const EmulatedNetworkStats& s) {
+    stats = s;
+    wait.Set();
+  });
+  bool stats_received = wait.Wait(kStatsWaitTimeoutMs);
+  RTC_CHECK(stats_received);
+  return stats;
+}
+
+void NetworkQualityMetricsReporter::ReportStats(
+    const std::string& network_label,
+    const EmulatedNetworkStats& stats,
+    int64_t packet_loss) {
+  ReportResult("bytes_sent", network_label, stats.bytes_sent.bytes(),
+               "sizeInBytes");
+  ReportResult("packets_sent", network_label, stats.packets_sent, "unitless");
+  ReportResult("average_send_rate", network_label,
+               stats.AverageSendRate().bytes_per_sec(), "bytesPerSecond");
+  ReportResult("bytes_dropped", network_label, stats.bytes_dropped.bytes(),
+               "sizeInBytes");
+  ReportResult("packets_dropped", network_label, stats.packets_dropped,
+               "unitless");
+  ReportResult("bytes_received", network_label, stats.bytes_received.bytes(),
+               "sizeInBytes");
+  ReportResult("packets_received", network_label, stats.packets_received,
+               "unitless");
+  ReportResult("average_receive_rate", network_label,
+               stats.AverageReceiveRate().bytes_per_sec(), "bytesPerSecond");
+  ReportResult("sent_packets_loss", network_label, packet_loss, "unitless");
+}
+
+void NetworkQualityMetricsReporter::ReportResult(
+    const std::string& metric_name,
+    const std::string& network_label,
+    const double value,
+    const std::string& unit) const {
+  test::PrintResult(metric_name, /*modifier=*/"",
+                    GetTestCaseName(network_label), value, unit,
+                    /*important=*/false);
+}
+
+std::string NetworkQualityMetricsReporter::GetTestCaseName(
+    const std::string& network_label) const {
+  return test_case_name_ + "/" + network_label;
+}
+
+}  // namespace webrtc_pc_e2e
+}  // namespace webrtc

test/pc/e2e/network_quality_metrics_reporter.h

@@ -0,0 +1,55 @@
+/*
+ *  Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_NETWORK_QUALITY_METRICS_REPORTER_H_
+#define TEST_PC_E2E_NETWORK_QUALITY_METRICS_REPORTER_H_
+
+#include <string>
+
+#include "api/test/network_emulation_manager.h"
+#include "api/test/peerconnection_quality_test_fixture.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+class NetworkQualityMetricsReporter
+    : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter {
+ public:
+  NetworkQualityMetricsReporter(EmulatedNetworkManagerInterface* alice_network,
+                                EmulatedNetworkManagerInterface* bob_network)
+      : alice_network_(alice_network), bob_network_(bob_network) {}
+  ~NetworkQualityMetricsReporter() override = default;
+
+  // Network stats must be empty when this method will be invoked.
+  void Start(absl::string_view test_case_name) override;
+  void StopAndReportResults() override;
+
+ private:
+  static EmulatedNetworkStats PopulateStats(
+      EmulatedNetworkManagerInterface* network);
+  void ReportStats(const std::string& network_label,
+                   const EmulatedNetworkStats& stats,
+                   int64_t packet_loss);
+  void ReportResult(const std::string& metric_name,
+                    const std::string& network_label,
+                    const double value,
+                    const std::string& unit) const;
+  std::string GetTestCaseName(const std::string& network_label) const;
+
+  std::string test_case_name_;
+
+  EmulatedNetworkManagerInterface* alice_network_;
+  EmulatedNetworkManagerInterface* bob_network_;
+};
+
+}  // namespace webrtc_pc_e2e
+}  // namespace webrtc
+
+#endif  // TEST_PC_E2E_NETWORK_QUALITY_METRICS_REPORTER_H_

test/pc/e2e/peer_connection_e2e_smoke_test.cc

@@ -20,6 +20,7 @@
 #include "test/gtest.h"
 #include "test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h"
 #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h"
+#include "test/pc/e2e/network_quality_metrics_reporter.h"
 #include "test/testsupport/file_utils.h"
 
 namespace webrtc {
@@ -113,6 +114,10 @@ TEST(PeerConnectionE2EQualityTestSmokeTest, MAYBE_RunWithEmulatedNetwork) {
                      bob->SetAudioConfig(std::move(audio_config));
                    });
 
+  fixture->AddQualityMetricsReporter(
+      absl::make_unique<NetworkQualityMetricsReporter>(alice_network,
+                                                       bob_network));
+
   RunParams run_params(TimeDelta::seconds(7));
   run_params.video_encoder_bitrate_multiplier = 1.1;
   fixture->Run(run_params);

test/pc/e2e/peer_connection_quality_test.cc

@@ -197,6 +197,11 @@ void PeerConnectionE2EQualityTest::PostTask(ScheduledActivity activity) {
       remaining_delay.ms());
 }
 
+void PeerConnectionE2EQualityTest::AddQualityMetricsReporter(
+    std::unique_ptr<QualityMetricsReporter> quality_metrics_reporter) {
+  quality_metrics_reporters_.push_back(std::move(quality_metrics_reporter));
+}
+
 void PeerConnectionE2EQualityTest::AddPeer(
     rtc::Thread* network_thread,
     rtc::NetworkManager* network_manager,
@@ -292,6 +297,9 @@ void PeerConnectionE2EQualityTest::Run(
   video_quality_analyzer_injection_helper_->Start(test_case_name_,
                                                   video_analyzer_threads);
   audio_quality_analyzer_->Start(test_case_name_, &analyzer_helper_);
+  for (auto& reporter : quality_metrics_reporters_) {
+    reporter->Start(test_case_name_);
+  }
 
   // Start RTCEventLog recording if requested.
   if (alice_->params()->rtc_event_log_path) {
@@ -364,6 +372,9 @@ void PeerConnectionE2EQualityTest::Run(
 
   audio_quality_analyzer_->Stop();
   video_quality_analyzer_injection_helper_->Stop();
+  for (auto& reporter : quality_metrics_reporters_) {
+    reporter->StopAndReportResults();
+  }
 
   // Ensuring that TestPeers have been destroyed in order to correctly close
   // Audio dumps.

test/pc/e2e/peer_connection_quality_test.h

@@ -155,6 +155,8 @@ class PeerConnectionE2EQualityTest
   using RunParams = PeerConnectionE2EQualityTestFixture::RunParams;
   using VideoConfig = PeerConnectionE2EQualityTestFixture::VideoConfig;
   using PeerConfigurer = PeerConnectionE2EQualityTestFixture::PeerConfigurer;
+  using QualityMetricsReporter =
+      PeerConnectionE2EQualityTestFixture::QualityMetricsReporter;
 
   PeerConnectionE2EQualityTest(
       std::string test_case_name,
@@ -169,6 +171,9 @@ class PeerConnectionE2EQualityTest
                     TimeDelta interval,
                     std::function<void(TimeDelta)> func) override;
 
+  void AddQualityMetricsReporter(std::unique_ptr<QualityMetricsReporter>
+                                     quality_metrics_reporter) override;
+
   void AddPeer(rtc::Thread* network_thread,
                rtc::NetworkManager* network_manager,
                rtc::FunctionView<void(PeerConfigurer*)> configurer) override;
@@ -232,6 +237,8 @@ class PeerConnectionE2EQualityTest
 
   std::unique_ptr<TestPeer> alice_;
   std::unique_ptr<TestPeer> bob_;
+  std::vector<std::unique_ptr<QualityMetricsReporter>>
+      quality_metrics_reporters_;
 
   std::vector<rtc::scoped_refptr<FrameGeneratorCapturerVideoTrackSource>>
       alice_video_sources_;