Add av sync metrics to pc level tests

Bug: webrtc:11381
Change-Id: I0a44583114401f09425d49dbb36957160b3f149f
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/178201
Commit-Queue: Andrey Logvin <landrey@webrtc.org>
Reviewed-by: Artem Titov <titovartem@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#31603}

test/pc/e2e/BUILD.gn

@@ -361,6 +361,7 @@ if (!build_with_chromium) {
       ]
       deps = [
         ":analyzer_helper",
+        ":cross_media_metrics_reporter",
         ":default_audio_quality_analyzer",
         ":default_video_quality_analyzer",
         ":media_helper",
@@ -659,6 +660,31 @@ if (!build_with_chromium) {
     absl_deps = [ "//third_party/abseil-cpp/absl/strings" ]
   }
 
+  rtc_library("cross_media_metrics_reporter") {
+    visibility = [ "*" ]
+    testonly = true
+    sources = [
+      "cross_media_metrics_reporter.cc",
+      "cross_media_metrics_reporter.h",
+    ]
+    deps = [
+      "../..:perf_test",
+      "../../../api:network_emulation_manager_api",
+      "../../../api:peer_connection_quality_test_fixture_api",
+      "../../../api:rtc_stats_api",
+      "../../../api:track_id_stream_info_map",
+      "../../../api/units:timestamp",
+      "../../../rtc_base:criticalsection",
+      "../../../rtc_base:rtc_event",
+      "../../../rtc_base:rtc_numerics",
+      "../../../system_wrappers:field_trial",
+    ]
+    absl_deps = [
+      "//third_party/abseil-cpp/absl/strings",
+      "//third_party/abseil-cpp/absl/types:optional",
+    ]
+  }
+
   rtc_library("sdp_changer") {
     visibility = [ "*" ]
     testonly = true

test/pc/e2e/cross_media_metrics_reporter.cc

@@ -0,0 +1,129 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "test/pc/e2e/cross_media_metrics_reporter.h"
+
+#include <utility>
+#include <vector>
+
+#include "api/stats/rtc_stats.h"
+#include "api/stats/rtcstats_objects.h"
+#include "api/units/timestamp.h"
+#include "rtc_base/event.h"
+#include "system_wrappers/include/field_trial.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+void CrossMediaMetricsReporter::Start(
+    absl::string_view test_case_name,
+    const TrackIdStreamInfoMap* reporter_helper) {
+  test_case_name_ = std::string(test_case_name);
+  reporter_helper_ = reporter_helper;
+}
+
+void CrossMediaMetricsReporter::OnStatsReports(
+    absl::string_view pc_label,
+    const rtc::scoped_refptr<const RTCStatsReport>& report) {
+  auto inbound_stats = report->GetStatsOfType<RTCInboundRTPStreamStats>();
+  std::map<absl::string_view, std::vector<const RTCInboundRTPStreamStats*>>
+      sync_group_stats;
+  for (const auto& stat : inbound_stats) {
+    auto media_source_stat =
+        report->GetAs<RTCMediaStreamTrackStats>(*stat->track_id);
+    if (stat->estimated_playout_timestamp.ValueOrDefault(0.) > 0 &&
+        media_source_stat->track_identifier.is_defined()) {
+      sync_group_stats[reporter_helper_->GetSyncGroupLabelFromTrackId(
+                           *media_source_stat->track_identifier)]
+          .push_back(stat);
+    }
+  }
+
+  rtc::CritScope cs(&lock_);
+  for (const auto& pair : sync_group_stats) {
+    // If there is less than two streams, it is not a sync group.
+    if (pair.second.size() < 2) {
+      continue;
+    }
+    auto sync_group = std::string(pair.first);
+    const RTCInboundRTPStreamStats* audio_stat = pair.second[0];
+    const RTCInboundRTPStreamStats* video_stat = pair.second[1];
+
+    RTC_CHECK(pair.second.size() == 2 && audio_stat->kind.is_defined() &&
+              video_stat->kind.is_defined() &&
+              *audio_stat->kind != *video_stat->kind)
+        << "Sync group should consist of one audio and one video stream.";
+
+    if (*audio_stat->kind == RTCMediaStreamTrackKind::kVideo) {
+      std::swap(audio_stat, video_stat);
+    }
+    // Stream labels of a sync group are same for all polls, so we need it add
+    // it only once.
+    if (stats_info_.find(sync_group) == stats_info_.end()) {
+      auto audio_source_stat =
+          report->GetAs<RTCMediaStreamTrackStats>(*audio_stat->track_id);
+      auto video_source_stat =
+          report->GetAs<RTCMediaStreamTrackStats>(*video_stat->track_id);
+      // *_source_stat->track_identifier is always defined here because we
+      // checked it while grouping stats.
+      stats_info_[sync_group].audio_stream_label =
+          std::string(reporter_helper_->GetStreamLabelFromTrackId(
+              *audio_source_stat->track_identifier));
+      stats_info_[sync_group].video_stream_label =
+          std::string(reporter_helper_->GetStreamLabelFromTrackId(
+              *video_source_stat->track_identifier));
+    }
+
+    double audio_video_playout_diff = *audio_stat->estimated_playout_timestamp -
+                                      *video_stat->estimated_playout_timestamp;
+    if (audio_video_playout_diff > 0) {
+      stats_info_[sync_group].audio_ahead_ms.AddSample(
+          audio_video_playout_diff);
+      stats_info_[sync_group].video_ahead_ms.AddSample(0);
+    } else {
+      stats_info_[sync_group].audio_ahead_ms.AddSample(0);
+      stats_info_[sync_group].video_ahead_ms.AddSample(
+          std::abs(audio_video_playout_diff));
+    }
+  }
+}
+
+void CrossMediaMetricsReporter::StopAndReportResults() {
+  rtc::CritScope cs(&lock_);
+  for (const auto& pair : stats_info_) {
+    const std::string& sync_group = pair.first;
+    ReportResult("audio_ahead_ms",
+                 GetTestCaseName(pair.second.audio_stream_label, sync_group),
+                 pair.second.audio_ahead_ms, "ms",
+                 webrtc::test::ImproveDirection::kSmallerIsBetter);
+    ReportResult("video_ahead_ms",
+                 GetTestCaseName(pair.second.video_stream_label, sync_group),
+                 pair.second.video_ahead_ms, "ms",
+                 webrtc::test::ImproveDirection::kSmallerIsBetter);
+  }
+}
+
+void CrossMediaMetricsReporter::ReportResult(
+    const std::string& metric_name,
+    const std::string& test_case_name,
+    const SamplesStatsCounter& counter,
+    const std::string& unit,
+    webrtc::test::ImproveDirection improve_direction) {
+  test::PrintResult(metric_name, /*modifier=*/"", test_case_name, counter, unit,
+                    /*important=*/false, improve_direction);
+}
+
+std::string CrossMediaMetricsReporter::GetTestCaseName(
+    const std::string& stream_label,
+    const std::string& sync_group) const {
+  return test_case_name_ + "/" + sync_group + "_" + stream_label;
+}
+
+}  // namespace webrtc_pc_e2e
+}  // namespace webrtc

test/pc/e2e/cross_media_metrics_reporter.h

@@ -0,0 +1,70 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_CROSS_MEDIA_METRICS_REPORTER_H_
+#define TEST_PC_E2E_CROSS_MEDIA_METRICS_REPORTER_H_
+
+#include <map>
+#include <string>
+
+#include "absl/strings/string_view.h"
+#include "absl/types/optional.h"
+#include "api/test/peerconnection_quality_test_fixture.h"
+#include "api/test/track_id_stream_info_map.h"
+#include "api/units/timestamp.h"
+#include "rtc_base/critical_section.h"
+#include "rtc_base/numerics/samples_stats_counter.h"
+#include "test/testsupport/perf_test.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+class CrossMediaMetricsReporter
+    : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter {
+ public:
+  CrossMediaMetricsReporter() = default;
+  ~CrossMediaMetricsReporter() override = default;
+
+  void Start(absl::string_view test_case_name,
+             const TrackIdStreamInfoMap* reporter_helper) override;
+  void OnStatsReports(
+      absl::string_view pc_label,
+      const rtc::scoped_refptr<const RTCStatsReport>& report) override;
+  void StopAndReportResults() override;
+
+ private:
+  struct StatsInfo {
+    SamplesStatsCounter audio_ahead_ms;
+    SamplesStatsCounter video_ahead_ms;
+
+    std::string audio_stream_label;
+    std::string video_stream_label;
+  };
+
+  static void ReportResult(const std::string& metric_name,
+                           const std::string& test_case_name,
+                           const SamplesStatsCounter& counter,
+                           const std::string& unit,
+                           webrtc::test::ImproveDirection improve_direction =
+                               webrtc::test::ImproveDirection::kNone);
+  std::string GetTestCaseName(const std::string& stream_label,
+                              const std::string& sync_group) const;
+
+  std::string test_case_name_;
+  const TrackIdStreamInfoMap* reporter_helper_;
+
+  rtc::CriticalSection lock_;
+  std::map<std::string, StatsInfo> stats_info_ RTC_GUARDED_BY(lock_);
+};
+
+}  // namespace webrtc_pc_e2e
+}  // namespace webrtc
+
+#endif  // TEST_PC_E2E_CROSS_MEDIA_METRICS_REPORTER_H_

test/pc/e2e/peer_connection_quality_test.cc

@@ -33,6 +33,7 @@
 #include "test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h"
 #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h"
 #include "test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h"
+#include "test/pc/e2e/cross_media_metrics_reporter.h"
 #include "test/pc/e2e/stats_poller.h"
 #include "test/pc/e2e/test_peer_factory.h"
 #include "test/testsupport/file_utils.h"
@@ -251,6 +252,8 @@ void PeerConnectionE2EQualityTest::Run(RunParams run_params) {
   RTC_LOG(INFO) << "video_analyzer_threads=" << video_analyzer_threads;
   quality_metrics_reporters_.push_back(
       std::make_unique<VideoQualityMetricsReporter>(clock_));
+  quality_metrics_reporters_.push_back(
+      std::make_unique<CrossMediaMetricsReporter>());
 
   video_quality_analyzer_injection_helper_->Start(
       test_case_name_,