Introduce ability to test echo in PC level test framework

Bug: webrtc:10138
Change-Id: Ie638eaec5a46e37dc0eb52e9432fdebd0e4a1c4d
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/147866
Reviewed-by: Karl Wiberg <kwiberg@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Reviewed-by: Sam Zackrisson <saza@webrtc.org>
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#28892}

api/test/peerconnection_quality_test_fixture.h

@@ -279,6 +279,13 @@ class PeerConnectionE2EQualityTestFixture {
         PeerConnectionInterface::BitrateParameters bitrate_params) = 0;
   };
 
+  // Contains configuration for echo emulator.
+  struct EchoEmulationConfig {
+    // Delay which represents the echo path delay, i.e. how soon rendered signal
+    // should reach capturer.
+    TimeDelta echo_delay = TimeDelta::ms(50);
+  };
+
   // Contains parameters, that describe how long framework should run quality
   // test.
   struct RunParams {
@@ -314,6 +321,10 @@ class PeerConnectionE2EQualityTestFixture {
     // If true will set conference mode in SDP media section for all video
     // tracks for all peers.
     bool use_conference_mode = false;
+    // If specified echo emulation will be done, by mixing the render audio into
+    // the capture signal. In such case input signal will be reduced by half to
+    // avoid saturation or compression in the echo path simulation.
+    absl::optional<EchoEmulationConfig> echo_emulation_config;
   };
 
   // Represent an entity that will report quality metrics after test.

rtc_base/swap_queue.h

@@ -200,6 +200,16 @@ class SwapQueue {
     return true;
   }
 
+  // Returns the current number of elements in the queue. Since elements may be
+  // concurrently added to the queue, the caller must treat this as a lower
+  // bound, not an exact count.
+  // May only be called by the consumer.
+  size_t SizeAtLeast() const {
+    // Acquire memory ordering ensures that we wait for the producer to finish
+    // inserting any element in progress.
+    return std::atomic_load_explicit(&num_elements_, std::memory_order_acquire);
+  }
+
  private:
   // Verify that the queue slots complies with the ItemVerifier test. This
   // function is not thread-safe and can only be used in the constructors.

test/pc/e2e/BUILD.gn

@@ -203,6 +203,20 @@ if (rtc_include_tests) {
     ]
   }
 
+  rtc_source_set("echo_emulation") {
+    visibility = [ "*" ]
+    testonly = true
+    sources = [
+      "echo/echo_emulation.cc",
+      "echo/echo_emulation.h",
+    ]
+    deps = [
+      "../../../api:peer_connection_quality_test_fixture_api",
+      "../../../modules/audio_device:audio_device_impl",
+      "../../../rtc_base:rtc_base_approved",
+    ]
+  }
+
   rtc_source_set("test_peer") {
     visibility = [ "*" ]
     testonly = true
@@ -211,6 +225,7 @@ if (rtc_include_tests) {
       "test_peer.h",
     ]
     deps = [
+      ":echo_emulation",
       ":peer_connection_quality_test_params",
       ":video_quality_analyzer_injection_helper",
       "../../../api:peer_connection_quality_test_fixture_api",

test/pc/e2e/echo/echo_emulation.cc

@@ -0,0 +1,123 @@
+/*
+ *  Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "test/pc/e2e/echo/echo_emulation.h"
+
+#include <limits>
+#include <utility>
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+namespace {
+
+constexpr int kSingleBufferDurationMs = 10;
+
+}  // namespace
+
+EchoEmulatingCapturer::EchoEmulatingCapturer(
+    std::unique_ptr<TestAudioDeviceModule::Capturer> capturer,
+    PeerConnectionE2EQualityTestFixture::EchoEmulationConfig config)
+    : delegate_(std::move(capturer)),
+      config_(config),
+      renderer_queue_(2 * config_.echo_delay.ms() / kSingleBufferDurationMs),
+      queue_input_(TestAudioDeviceModule::SamplesPerFrame(
+                       delegate_->SamplingFrequency()) *
+                   delegate_->NumChannels()),
+      queue_output_(TestAudioDeviceModule::SamplesPerFrame(
+                        delegate_->SamplingFrequency()) *
+                    delegate_->NumChannels()) {
+  renderer_thread_.Detach();
+  capturer_thread_.Detach();
+}
+
+void EchoEmulatingCapturer::OnAudioRendered(
+    rtc::ArrayView<const int16_t> data) {
+  RTC_DCHECK_RUN_ON(&renderer_thread_);
+  if (!recording_started_) {
+    // Because rendering can start before capturing in the beginning we can have
+    // a set of empty audio data frames. So we will skip them and will start
+    // fill the queue only after 1st non-empty audio data frame will arrive.
+    bool is_empty = true;
+    for (auto d : data) {
+      if (d != 0) {
+        is_empty = false;
+        break;
+      }
+    }
+    if (is_empty) {
+      return;
+    }
+    recording_started_ = true;
+  }
+  queue_input_.assign(data.begin(), data.end());
+  if (!renderer_queue_.Insert(&queue_input_)) {
+    // Test audio device works too slow with sanitizers and on some platforms
+    // and can't properly process audio, so when capturer will be stopped
+    // renderer will quickly overfill the queue.
+    // TODO(crbug.com/webrtc/10850) remove it when test ADM will be fast enough.
+#if !defined(THREAD_SANITIZER) && !defined(MEMORY_SANITIZER) && \
+    !defined(ADDRESS_SANITIZER) && !defined(WEBRTC_ANDROID) &&  \
+    !(defined(_MSC_VER) && !defined(__clang__) && !defined(NDEBUG))
+    RTC_CHECK(false) << "Echo queue is full";
+#endif
+  }
+}
+
+bool EchoEmulatingCapturer::Capture(rtc::BufferT<int16_t>* buffer) {
+  RTC_DCHECK_RUN_ON(&capturer_thread_);
+  bool result = delegate_->Capture(buffer);
+  // Now we have to reduce input signal to avoid saturation when mixing in the
+  // fake echo.
+  for (size_t i = 0; i < buffer->size(); ++i) {
+    (*buffer)[i] /= 2;
+  }
+
+  // When we accumulated enough delay in the echo buffer we will pop from
+  // that buffer on each ::Capture(...) call. If the buffer become empty it
+  // will mean some bug, so we will crash during removing item from the queue.
+  if (!delay_accumulated_) {
+    delay_accumulated_ =
+        renderer_queue_.SizeAtLeast() >=
+        static_cast<size_t>(config_.echo_delay.ms() / kSingleBufferDurationMs);
+  }
+
+  if (delay_accumulated_) {
+    RTC_CHECK(renderer_queue_.Remove(&queue_output_));
+    for (size_t i = 0; i < buffer->size() && i < queue_output_.size(); ++i) {
+      int32_t res = (*buffer)[i] + queue_output_[i];
+      if (res < std::numeric_limits<int16_t>::min()) {
+        res = std::numeric_limits<int16_t>::min();
+      }
+      if (res > std::numeric_limits<int16_t>::max()) {
+        res = std::numeric_limits<int16_t>::max();
+      }
+      (*buffer)[i] = static_cast<int16_t>(res);
+    }
+  }
+
+  return result;
+}
+
+EchoEmulatingRenderer::EchoEmulatingRenderer(
+    std::unique_ptr<TestAudioDeviceModule::Renderer> renderer,
+    EchoEmulatingCapturer* echo_emulating_capturer)
+    : delegate_(std::move(renderer)),
+      echo_emulating_capturer_(echo_emulating_capturer) {
+  RTC_DCHECK(echo_emulating_capturer_);
+}
+
+bool EchoEmulatingRenderer::Render(rtc::ArrayView<const int16_t> data) {
+  if (data.size() > 0) {
+    echo_emulating_capturer_->OnAudioRendered(data);
+  }
+  return delegate_->Render(data);
+}
+
+}  // namespace webrtc_pc_e2e
+}  // namespace webrtc

test/pc/e2e/echo/echo_emulation.h

@@ -0,0 +1,79 @@
+/*
+ *  Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_ECHO_ECHO_EMULATION_H_
+#define TEST_PC_E2E_ECHO_ECHO_EMULATION_H_
+
+#include <atomic>
+#include <deque>
+#include <memory>
+#include <vector>
+
+#include "api/test/peerconnection_quality_test_fixture.h"
+#include "modules/audio_device/include/test_audio_device.h"
+#include "rtc_base/swap_queue.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+// Reduces audio input strength from provided capturer twice and adds input
+// provided into EchoEmulatingCapturer::OnAudioRendered(...).
+class EchoEmulatingCapturer : public TestAudioDeviceModule::Capturer {
+ public:
+  EchoEmulatingCapturer(
+      std::unique_ptr<TestAudioDeviceModule::Capturer> capturer,
+      PeerConnectionE2EQualityTestFixture::EchoEmulationConfig config);
+
+  void OnAudioRendered(rtc::ArrayView<const int16_t> data);
+
+  int SamplingFrequency() const override {
+    return delegate_->SamplingFrequency();
+  }
+  int NumChannels() const override { return delegate_->NumChannels(); }
+  bool Capture(rtc::BufferT<int16_t>* buffer) override;
+
+ private:
+  std::unique_ptr<TestAudioDeviceModule::Capturer> delegate_;
+  const PeerConnectionE2EQualityTestFixture::EchoEmulationConfig config_;
+
+  SwapQueue<std::vector<int16_t>> renderer_queue_;
+
+  SequenceChecker renderer_thread_;
+  std::vector<int16_t> queue_input_ RTC_GUARDED_BY(renderer_thread_);
+  bool recording_started_ RTC_GUARDED_BY(renderer_thread_) = false;
+
+  SequenceChecker capturer_thread_;
+  std::vector<int16_t> queue_output_ RTC_GUARDED_BY(capturer_thread_);
+  bool delay_accumulated_ RTC_GUARDED_BY(capturer_thread_) = false;
+};
+
+// Renders output into provided renderer and also copy output into provided
+// EchoEmulationCapturer.
+class EchoEmulatingRenderer : public TestAudioDeviceModule::Renderer {
+ public:
+  EchoEmulatingRenderer(
+      std::unique_ptr<TestAudioDeviceModule::Renderer> renderer,
+      EchoEmulatingCapturer* echo_emulating_capturer);
+
+  int SamplingFrequency() const override {
+    return delegate_->SamplingFrequency();
+  }
+  int NumChannels() const override { return delegate_->NumChannels(); }
+  bool Render(rtc::ArrayView<const int16_t> data) override;
+
+ private:
+  std::unique_ptr<TestAudioDeviceModule::Renderer> delegate_;
+  EchoEmulatingCapturer* echo_emulating_capturer_;
+};
+
+}  // namespace webrtc_pc_e2e
+}  // namespace webrtc
+
+#endif  // TEST_PC_E2E_ECHO_ECHO_EMULATION_H_

test/pc/e2e/peer_connection_e2e_smoke_test.cc

@@ -38,6 +38,8 @@ class PeerConnectionE2EQualityTestSmokeTest : public ::testing::Test {
   using ScrollingParams = PeerConnectionE2EQualityTestFixture::ScrollingParams;
   using VideoSimulcastConfig =
       PeerConnectionE2EQualityTestFixture::VideoSimulcastConfig;
+  using EchoEmulationConfig =
+      PeerConnectionE2EQualityTestFixture::EchoEmulationConfig;
 
   void RunTest(const std::string& test_case_name,
                const RunParams& run_params,
@@ -136,6 +138,7 @@ TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Smoke) {
   run_params.use_flex_fec = true;
   run_params.use_ulp_fec = true;
   run_params.video_encoder_bitrate_multiplier = 1.1;
+  run_params.echo_emulation_config = EchoEmulationConfig();
   RunTest(
       "smoke", run_params,
       [](PeerConfigurer* alice) {

test/pc/e2e/peer_connection_quality_test.cc

@@ -276,7 +276,7 @@ void PeerConnectionE2EQualityTest::Run(RunParams run_params) {
           [this]() { StartVideo(alice_video_sources_); }),
       video_quality_analyzer_injection_helper_.get(), signaling_thread.get(),
       alice_remote_audio_config, run_params.video_encoder_bitrate_multiplier,
-      task_queue_.get());
+      run_params.echo_emulation_config, task_queue_.get());
   bob_ = TestPeer::CreateTestPeer(
       std::move(bob_components), std::move(bob_params),
       absl::make_unique<FixturePeerConnectionObserver>(
@@ -287,7 +287,7 @@ void PeerConnectionE2EQualityTest::Run(RunParams run_params) {
           [this]() { StartVideo(bob_video_sources_); }),
       video_quality_analyzer_injection_helper_.get(), signaling_thread.get(),
       bob_remote_audio_config, run_params.video_encoder_bitrate_multiplier,
-      task_queue_.get());
+      run_params.echo_emulation_config, task_queue_.get());
 
   int num_cores = CpuInfo::DetectNumberOfCores();
   RTC_DCHECK_GE(num_cores, 1);

test/pc/e2e/test_peer.cc

@@ -26,6 +26,7 @@
 #include "modules/audio_processing/include/audio_processing.h"
 #include "p2p/client/basic_port_allocator.h"
 #include "rtc_base/location.h"
+#include "test/pc/e2e/echo/echo_emulation.h"
 #include "test/testsupport/copy_to_file_audio_capturer.h"
 
 namespace webrtc {
@@ -36,6 +37,8 @@ using RemotePeerAudioConfig =
     ::webrtc::webrtc_pc_e2e::TestPeer::RemotePeerAudioConfig;
 using AudioConfig =
     ::webrtc::webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture::AudioConfig;
+using EchoEmulationConfig = ::webrtc::webrtc_pc_e2e::
+    PeerConnectionE2EQualityTestFixture::EchoEmulationConfig;
 
 constexpr int16_t kGeneratedAudioMaxAmplitude = 32000;
 constexpr int kDefaultSamplingFrequencyInHz = 48000;
@@ -72,13 +75,15 @@ class TestPeerComponents {
                      rtc::Thread* signaling_thread,
                      absl::optional<RemotePeerAudioConfig> remote_audio_config,
                      double bitrate_multiplier,
+                     absl::optional<EchoEmulationConfig> echo_emulation_config,
                      rtc::TaskQueue* task_queue)
       : audio_config_opt_(params.audio_config),
         observer_(observer),
         video_analyzer_helper_(video_analyzer_helper),
         signaling_thread_(signaling_thread),
         remote_audio_config_(std::move(remote_audio_config)),
-        bitrate_multiplier_(bitrate_multiplier) {
+        bitrate_multiplier_(bitrate_multiplier),
+        echo_emulation_config_(std::move(echo_emulation_config)) {
     for (auto& video_config : params.video_configs) {
       // Stream label should be set by fixture implementation here.
       RTC_DCHECK(video_config.stream_label);
@@ -177,31 +182,26 @@ class TestPeerComponents {
 
   rtc::scoped_refptr<AudioDeviceModule> CreateAudioDeviceModule(
       TaskQueueFactory* task_queue_factory) {
-    std::unique_ptr<TestAudioDeviceModule::Capturer> capturer;
+    std::unique_ptr<TestAudioDeviceModule::Renderer> renderer =
-    if (audio_config_opt_) {
+        CreateAudioRenderer(remote_audio_config_);
-      capturer = CreateAudioCapturer(*audio_config_opt_);
+    std::unique_ptr<TestAudioDeviceModule::Capturer> capturer =
-      if (audio_config_opt_->input_dump_file_name) {
+        CreateAudioCapturer(audio_config_opt_);
-        capturer = absl::make_unique<test::CopyToFileAudioCapturer>(
+    RTC_DCHECK(renderer);
-            std::move(capturer),
-            audio_config_opt_->input_dump_file_name.value());
-      }
-    } else {
-      // If we have no audio config we still need to provide some audio device.
-      // In such case use generated capturer. Despite of we provided audio here,
-      // in test media setup audio stream won't be added into peer connection.
-      capturer = TestAudioDeviceModule::CreatePulsedNoiseCapturer(
-          kGeneratedAudioMaxAmplitude, kDefaultSamplingFrequencyInHz);
-    }
     RTC_DCHECK(capturer);
 
-    std::unique_ptr<TestAudioDeviceModule::Renderer> renderer;
+    // Setup echo emulation if required.
-    if (remote_audio_config_ && remote_audio_config_->output_file_name) {
+    if (echo_emulation_config_) {
-      renderer = TestAudioDeviceModule::CreateBoundedWavFileWriter(
+      capturer = absl::make_unique<EchoEmulatingCapturer>(
-          remote_audio_config_->output_file_name.value(),
+          std::move(capturer), *echo_emulation_config_);
-          remote_audio_config_->sampling_frequency_in_hz);
+      renderer = absl::make_unique<EchoEmulatingRenderer>(
-    } else {
+          std::move(renderer),
-      renderer = TestAudioDeviceModule::CreateDiscardRenderer(
+          static_cast<EchoEmulatingCapturer*>(capturer.get()));
-          kDefaultSamplingFrequencyInHz);
+    }
+
+    // Setup input stream dumping if required.
+    if (audio_config_opt_ && audio_config_opt_->input_dump_file_name) {
+      capturer = absl::make_unique<test::CopyToFileAudioCapturer>(
+          std::move(capturer), audio_config_opt_->input_dump_file_name.value());
     }
 
     return TestAudioDeviceModule::Create(task_queue_factory,
@@ -209,19 +209,41 @@ class TestPeerComponents {
                                          std::move(renderer), /*speed=*/1.f);
   }
 
+  std::unique_ptr<TestAudioDeviceModule::Renderer> CreateAudioRenderer(
+      const absl::optional<RemotePeerAudioConfig>& config) {
+    if (!config) {
+      // Return default renderer because we always require some renderer.
+      return TestAudioDeviceModule::CreateDiscardRenderer(
+          kDefaultSamplingFrequencyInHz);
+    }
+    if (config->output_file_name) {
+      return TestAudioDeviceModule::CreateBoundedWavFileWriter(
+          config->output_file_name.value(), config->sampling_frequency_in_hz);
+    }
+    return TestAudioDeviceModule::CreateDiscardRenderer(
+        config->sampling_frequency_in_hz);
+  }
+
   std::unique_ptr<TestAudioDeviceModule::Capturer> CreateAudioCapturer(
-      const AudioConfig& audio_config) {
+      const absl::optional<AudioConfig>& audio_config) {
-    if (audio_config.mode == AudioConfig::Mode::kGenerated) {
+    if (!audio_config) {
+      // If we have no audio config we still need to provide some audio device.
+      // In such case use generated capturer. Despite of we provided audio here,
+      // in test media setup audio stream won't be added into peer connection.
       return TestAudioDeviceModule::CreatePulsedNoiseCapturer(
-          kGeneratedAudioMaxAmplitude, audio_config.sampling_frequency_in_hz);
+          kGeneratedAudioMaxAmplitude, kDefaultSamplingFrequencyInHz);
     }
-    if (audio_config.mode == AudioConfig::Mode::kFile) {
+
-      RTC_DCHECK(audio_config.input_file_name);
+    switch (audio_config->mode) {
-      return TestAudioDeviceModule::CreateWavFileReader(
+      case AudioConfig::Mode::kGenerated:
-          audio_config.input_file_name.value(), /*repeat=*/true);
+        return TestAudioDeviceModule::CreatePulsedNoiseCapturer(
+            kGeneratedAudioMaxAmplitude,
+            audio_config->sampling_frequency_in_hz);
+      case AudioConfig::Mode::kFile:
+        RTC_DCHECK(audio_config->input_file_name);
+        return TestAudioDeviceModule::CreateWavFileReader(
+            audio_config->input_file_name.value(), /*repeat=*/true);
     }
-    RTC_NOTREACHED() << "Unknown audio_config->mode";
-    return nullptr;
   }
 
   std::unique_ptr<VideoEncoderFactory> CreateVideoEncoderFactory(
@@ -290,6 +312,7 @@ class TestPeerComponents {
   rtc::Thread* signaling_thread_;
   absl::optional<RemotePeerAudioConfig> remote_audio_config_;
   double bitrate_multiplier_;
+  absl::optional<EchoEmulationConfig> echo_emulation_config_;
 };
 
 }  // namespace
@@ -310,6 +333,7 @@ std::unique_ptr<TestPeer> TestPeer::CreateTestPeer(
     rtc::Thread* signaling_thread,
     absl::optional<RemotePeerAudioConfig> remote_audio_config,
     double bitrate_multiplier,
+    absl::optional<EchoEmulationConfig> echo_emulation_config,
     rtc::TaskQueue* task_queue) {
   RTC_DCHECK(components);
   RTC_DCHECK(params);
@@ -319,7 +343,7 @@ std::unique_ptr<TestPeer> TestPeer::CreateTestPeer(
   TestPeerComponents tpc(std::move(components), *params, observer.get(),
                          video_analyzer_helper, signaling_thread,
                          std::move(remote_audio_config), bitrate_multiplier,
-                         task_queue);
+                         echo_emulation_config, task_queue);
 
   return absl::WrapUnique(new TestPeer(
       tpc.peer_connection_factory(), tpc.peer_connection(), std::move(observer),

test/pc/e2e/test_peer.h

@@ -36,6 +36,8 @@ class TestPeer final : public PeerConnectionWrapper {
   using PeerConnectionWrapper::PeerConnectionWrapper;
   using VideoConfig = PeerConnectionE2EQualityTestFixture::VideoConfig;
   using AudioConfig = PeerConnectionE2EQualityTestFixture::AudioConfig;
+  using EchoEmulationConfig =
+      PeerConnectionE2EQualityTestFixture::EchoEmulationConfig;
 
   struct RemotePeerAudioConfig {
     RemotePeerAudioConfig(AudioConfig config)
@@ -55,11 +57,8 @@ class TestPeer final : public PeerConnectionWrapper {
   // injection.
   //
   // |signaling_thread| will be provided by test fixture implementation.
-  // |params| - describes current peer paramters, like current peer video
+  // |params| - describes current peer parameters, like current peer video
   // streams and audio streams
-  // |audio_outpu_file_name| - the name of output file, where incoming audio
-  // stream should be written. It should be provided from remote peer
-  // |params.audio_config.output_file_name|
   static std::unique_ptr<TestPeer> CreateTestPeer(
       std::unique_ptr<InjectableComponents> components,
       std::unique_ptr<Params> params,
@@ -68,6 +67,7 @@ class TestPeer final : public PeerConnectionWrapper {
       rtc::Thread* signaling_thread,
       absl::optional<RemotePeerAudioConfig> remote_audio_config,
       double bitrate_multiplier,
+      absl::optional<EchoEmulationConfig> echo_emulation_config,
       rtc::TaskQueue* task_queue);
 
   Params* params() const { return params_.get(); }