Several subcomponents inside APM copy render audio from

the render side to the capture side using the same
pattern. Currently this is done independently for the
submodules.

This CL moves the the AEC functionality for this into
APM.

BUG=webrtc:5298, webrtc:6540

Review-Url: https://codereview.webrtc.org/2427553003
Cr-Commit-Position: refs/heads/master@{#14726}

webrtc/modules/audio_processing/audio_processing_impl.cc

@@ -119,6 +119,13 @@ int FindNativeProcessRateToUse(int minimum_rate, bool band_splitting_required) {
   return uppermost_native_rate;
 }
 
+// Maximum length that a frame of samples can have.
+static const size_t kMaxAllowedValuesOfSamplesPerFrame = 160;
+// Maximum number of frames to buffer in the render queue.
+// TODO(peah): Decrease this once we properly handle hugely unbalanced
+// reverse and forward call numbers.
+static const size_t kMaxNumFramesToBuffer = 100;
+
 }  // namespace
 
 // Throughout webrtc, it's assumed that success is represented by zero.
@@ -430,9 +437,12 @@ int AudioProcessingImpl::InitializeLocked() {
 
   public_submodules_->gain_control->Initialize(num_proc_channels(),
                                                proc_sample_rate_hz());
+
   public_submodules_->echo_cancellation->Initialize(
       proc_sample_rate_hz(), num_reverse_channels(), num_output_channels(),
       num_proc_channels());
+  AllocateRenderQueue();
+
   public_submodules_->echo_control_mobile->Initialize(
       proc_split_sample_rate_hz(), num_reverse_channels(),
       num_output_channels());
@@ -697,7 +707,7 @@ int AudioProcessingImpl::ProcessStream(const float* const* src,
     // that retrieves the render side data. This function accesses apm
     // getters that need the capture lock held when being called.
     rtc::CritScope cs_capture(&crit_capture_);
-    public_submodules_->echo_cancellation->ReadQueuedRenderData();
+    EmptyQueuedRenderAudio();
     public_submodules_->echo_control_mobile->ReadQueuedRenderData();
     public_submodules_->gain_control->ReadQueuedRenderData();
 
@@ -757,6 +767,58 @@ int AudioProcessingImpl::ProcessStream(const float* const* src,
   return kNoError;
 }
 
+void AudioProcessingImpl::QueueRenderAudio(const AudioBuffer* audio) {
+  EchoCancellationImpl::PackRenderAudioBuffer(audio, num_output_channels(),
+                                              num_reverse_channels(),
+                                              &render_queue_buffer_);
+
+  RTC_DCHECK_GE(160u, audio->num_frames_per_band());
+
+  // Insert the samples into the queue.
+  if (!render_signal_queue_->Insert(&render_queue_buffer_)) {
+    // The data queue is full and needs to be emptied.
+    EmptyQueuedRenderAudio();
+
+    // Retry the insert (should always work).
+    bool result = render_signal_queue_->Insert(&render_queue_buffer_);
+    RTC_DCHECK(result);
+  }
+}
+
+void AudioProcessingImpl::AllocateRenderQueue() {
+  const size_t new_render_queue_element_max_size =
+      std::max(static_cast<size_t>(1),
+               kMaxAllowedValuesOfSamplesPerFrame *
+                   EchoCancellationImpl::NumCancellersRequired(
+                       num_output_channels(), num_reverse_channels()));
+
+  // Reallocate the queue if the queue item size is too small to fit the
+  // data to put in the queue.
+  if (render_queue_element_max_size_ < new_render_queue_element_max_size) {
+    render_queue_element_max_size_ = new_render_queue_element_max_size;
+
+    std::vector<float> template_queue_element(render_queue_element_max_size_);
+
+    render_signal_queue_.reset(
+        new SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>(
+            kMaxNumFramesToBuffer, template_queue_element,
+            RenderQueueItemVerifier<float>(render_queue_element_max_size_)));
+
+    render_queue_buffer_.resize(render_queue_element_max_size_);
+    capture_queue_buffer_.resize(render_queue_element_max_size_);
+  } else {
+    render_signal_queue_->Clear();
+  }
+}
+
+void AudioProcessingImpl::EmptyQueuedRenderAudio() {
+  rtc::CritScope cs_capture(&crit_capture_);
+  while (render_signal_queue_->Remove(&capture_queue_buffer_)) {
+    public_submodules_->echo_cancellation->ProcessRenderAudio(
+        capture_queue_buffer_);
+  }
+}
+
 int AudioProcessingImpl::ProcessStream(AudioFrame* frame) {
   TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_AudioFrame");
   {
@@ -767,7 +829,7 @@ int AudioProcessingImpl::ProcessStream(AudioFrame* frame) {
     // public_submodules_->echo_control_mobile->is_enabled() aquires this lock
     // as well.
     rtc::CritScope cs_capture(&crit_capture_);
-    public_submodules_->echo_cancellation->ReadQueuedRenderData();
+    EmptyQueuedRenderAudio();
     public_submodules_->echo_control_mobile->ReadQueuedRenderData();
     public_submodules_->gain_control->ReadQueuedRenderData();
   }
@@ -1130,8 +1192,7 @@ int AudioProcessingImpl::ProcessRenderStreamLocked() {
   }
 #endif
 
-  RETURN_ON_ERR(
+  QueueRenderAudio(render_buffer);
-      public_submodules_->echo_cancellation->ProcessRenderAudio(render_buffer));
   RETURN_ON_ERR(public_submodules_->echo_control_mobile->ProcessRenderAudio(
       render_buffer));
   if (!constants_.use_experimental_agc) {

webrtc/modules/audio_processing/audio_processing_impl.h

@@ -19,9 +19,11 @@
 #include "webrtc/base/criticalsection.h"
 #include "webrtc/base/gtest_prod_util.h"
 #include "webrtc/base/ignore_wundef.h"
+#include "webrtc/base/swap_queue.h"
 #include "webrtc/base/thread_annotations.h"
 #include "webrtc/modules/audio_processing/audio_buffer.h"
 #include "webrtc/modules/audio_processing/include/audio_processing.h"
+#include "webrtc/modules/audio_processing/render_queue_item_verifier.h"
 #include "webrtc/system_wrappers/include/file_wrapper.h"
 
 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
@@ -233,6 +235,12 @@ class AudioProcessingImpl : public AudioProcessing {
       EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
   void InitializeLevelController() EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
 
+  void EmptyQueuedRenderAudio();
+  void AllocateRenderQueue()
+      EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
+  void QueueRenderAudio(const AudioBuffer* audio)
+      EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
+
   // Capture-side exclusive methods possibly running APM in a multi-threaded
   // manner that are called with the render lock already acquired.
   int ProcessCaptureStreamLocked() EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
@@ -362,6 +370,15 @@ class AudioProcessingImpl : public AudioProcessing {
     std::unique_ptr<AudioConverter> render_converter;
     std::unique_ptr<AudioBuffer> render_audio;
   } render_ GUARDED_BY(crit_render_);
+
+  size_t render_queue_element_max_size_ GUARDED_BY(crit_render_)
+      GUARDED_BY(crit_capture_) = 0;
+  std::vector<float> render_queue_buffer_ GUARDED_BY(crit_render_);
+  std::vector<float> capture_queue_buffer_ GUARDED_BY(crit_capture_);
+
+  // Lock protection not needed.
+  std::unique_ptr<SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>>
+      render_signal_queue_;
 };
 
 }  // namespace webrtc

webrtc/modules/audio_processing/echo_cancellation_bit_exact_unittest.cc

@@ -49,8 +49,11 @@ void ProcessOneFrame(int sample_rate_hz,
     capture_audio_buffer->SplitIntoFrequencyBands();
   }
 
-  echo_canceller->ProcessRenderAudio(render_audio_buffer);
+  std::vector<float> render_audio;
-  echo_canceller->ReadQueuedRenderData();
+  EchoCancellationImpl::PackRenderAudioBuffer(
+      render_audio_buffer, 1, render_audio_buffer->num_channels(),
+      &render_audio);
+  echo_canceller->ProcessRenderAudio(render_audio);
 
   if (drift_compensation_enabled) {
     static_cast<EchoCancellation*>(echo_canceller)

webrtc/modules/audio_processing/echo_cancellation_impl.cc

@@ -49,12 +49,6 @@ AudioProcessing::Error MapError(int err) {
   }
 }
 
-// Maximum length that a frame of samples can have.
-static const size_t kMaxAllowedValuesOfSamplesPerFrame = 160;
-// Maximum number of frames to buffer in the render queue.
-// TODO(peah): Decrease this once we properly handle hugely unbalanced
-// reverse and forward call numbers.
-static const size_t kMaxNumFramesToBuffer = 100;
 }  // namespace
 
 struct EchoCancellationImpl::StreamProperties {
@@ -113,94 +107,38 @@ EchoCancellationImpl::EchoCancellationImpl(rtc::CriticalSection* crit_render,
       delay_logging_enabled_(false),
       extended_filter_enabled_(false),
       delay_agnostic_enabled_(false),
-      aec3_enabled_(false),
+      aec3_enabled_(false) {
-      render_queue_element_max_size_(0) {
   RTC_DCHECK(crit_render);
   RTC_DCHECK(crit_capture);
 }
 
-EchoCancellationImpl::~EchoCancellationImpl() {}
+EchoCancellationImpl::~EchoCancellationImpl() = default;
 
-int EchoCancellationImpl::ProcessRenderAudio(const AudioBuffer* audio) {
+void EchoCancellationImpl::ProcessRenderAudio(
-  rtc::CritScope cs_render(crit_render_);
+    rtc::ArrayView<const float> packed_render_audio) {
-  if (!enabled_) {
-    return AudioProcessing::kNoError;
-  }
-
-  RTC_DCHECK(stream_properties_);
-  RTC_DCHECK_GE(160u, audio->num_frames_per_band());
-  RTC_DCHECK_EQ(audio->num_channels(),
-                stream_properties_->num_reverse_channels);
-  RTC_DCHECK_GE(cancellers_.size(), stream_properties_->num_output_channels *
-                                        audio->num_channels());
-
-  int err = AudioProcessing::kNoError;
-
-  // The ordering convention must be followed to pass to the correct AEC.
-  size_t handle_index = 0;
-  render_queue_buffer_.clear();
-  for (size_t i = 0; i < stream_properties_->num_output_channels; i++) {
-    for (size_t j = 0; j < audio->num_channels(); j++) {
-      // Retrieve any error code produced by the buffering of the farend
-      // signal.
-      err = WebRtcAec_GetBufferFarendError(
-          cancellers_[handle_index++]->state(),
-          audio->split_bands_const_f(j)[kBand0To8kHz],
-          audio->num_frames_per_band());
-
-      if (err != AudioProcessing::kNoError) {
-        return MapError(err);  // TODO(ajm): warning possible?
-      }
-
-      // Buffer the samples in the render queue.
-      render_queue_buffer_.insert(render_queue_buffer_.end(),
-                                  audio->split_bands_const_f(j)[kBand0To8kHz],
-                                  (audio->split_bands_const_f(j)[kBand0To8kHz] +
-                                   audio->num_frames_per_band()));
-    }
-  }
-
-  // Insert the samples into the queue.
-  if (!render_signal_queue_->Insert(&render_queue_buffer_)) {
-    // The data queue is full and needs to be emptied.
-    ReadQueuedRenderData();
-
-    // Retry the insert (should always work).
-    bool result = render_signal_queue_->Insert(&render_queue_buffer_);
-    RTC_DCHECK(result);
-  }
-
-  return AudioProcessing::kNoError;
-}
-
-// Read chunks of data that were received and queued on the render side from
-// a queue. All the data chunks are buffered into the farend signal of the AEC.
-void EchoCancellationImpl::ReadQueuedRenderData() {
   rtc::CritScope cs_capture(crit_capture_);
   if (!enabled_) {
     return;
   }
 
   RTC_DCHECK(stream_properties_);
-  while (render_signal_queue_->Remove(&capture_queue_buffer_)) {
   size_t handle_index = 0;
   size_t buffer_index = 0;
   const size_t num_frames_per_band =
-        capture_queue_buffer_.size() /
+      packed_render_audio.size() / (stream_properties_->num_output_channels *
-        (stream_properties_->num_output_channels *
                                     stream_properties_->num_reverse_channels);
   for (size_t i = 0; i < stream_properties_->num_output_channels; i++) {
     for (size_t j = 0; j < stream_properties_->num_reverse_channels; j++) {
       WebRtcAec_BufferFarend(cancellers_[handle_index++]->state(),
-                               &capture_queue_buffer_[buffer_index],
+                             &packed_render_audio[buffer_index],
                              num_frames_per_band);
 
       buffer_index += num_frames_per_band;
     }
   }
-  }
 }
 
+
 int EchoCancellationImpl::ProcessCaptureAudio(AudioBuffer* audio,
                                               int stream_delay_ms) {
   rtc::CritScope cs_capture(crit_capture_);
@@ -480,9 +418,12 @@ void EchoCancellationImpl::Initialize(int sample_rate_hz,
     return;
   }
 
-  if (NumCancellersRequired() > cancellers_.size()) {
+  const size_t num_cancellers_required =
+      NumCancellersRequired(stream_properties_->num_output_channels,
+                            stream_properties_->num_reverse_channels);
+  if (num_cancellers_required > cancellers_.size()) {
     const size_t cancellers_old_size = cancellers_.size();
-    cancellers_.resize(NumCancellersRequired());
+    cancellers_.resize(num_cancellers_required);
 
     for (size_t i = cancellers_old_size; i < cancellers_.size(); ++i) {
       cancellers_[i].reset(new Canceller());
@@ -494,8 +435,6 @@ void EchoCancellationImpl::Initialize(int sample_rate_hz,
   }
 
   Configure();
-
-  AllocateRenderQueue();
 }
 
 int EchoCancellationImpl::GetSystemDelayInSamples() const {
@@ -506,30 +445,24 @@ int EchoCancellationImpl::GetSystemDelayInSamples() const {
       WebRtcAec_aec_core(cancellers_[0]->state()));
 }
 
-void EchoCancellationImpl::AllocateRenderQueue() {
+void EchoCancellationImpl::PackRenderAudioBuffer(
-  const size_t new_render_queue_element_max_size = std::max<size_t>(
+    const AudioBuffer* audio,
-      static_cast<size_t>(1),
+    size_t num_output_channels,
-      kMaxAllowedValuesOfSamplesPerFrame * NumCancellersRequired());
+    size_t num_channels,
+    std::vector<float>* packed_buffer) {
+  RTC_DCHECK_GE(160u, audio->num_frames_per_band());
+  RTC_DCHECK_EQ(num_channels, audio->num_channels());
 
-  rtc::CritScope cs_render(crit_render_);
+  packed_buffer->clear();
-  rtc::CritScope cs_capture(crit_capture_);
+  // The ordering convention must be followed to pass the correct data.
-
+  for (size_t i = 0; i < num_output_channels; i++) {
-  // Reallocate the queue if the queue item size is too small to fit the
+    for (size_t j = 0; j < audio->num_channels(); j++) {
-  // data to put in the queue.
+      // Buffer the samples in the render queue.
-  if (render_queue_element_max_size_ < new_render_queue_element_max_size) {
+      packed_buffer->insert(packed_buffer->end(),
-    render_queue_element_max_size_ = new_render_queue_element_max_size;
+                            audio->split_bands_const_f(j)[kBand0To8kHz],
-
+                            (audio->split_bands_const_f(j)[kBand0To8kHz] +
-    std::vector<float> template_queue_element(render_queue_element_max_size_);
+                             audio->num_frames_per_band()));
-
+    }
-    render_signal_queue_.reset(
-        new SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>(
-            kMaxNumFramesToBuffer, template_queue_element,
-            RenderQueueItemVerifier<float>(render_queue_element_max_size_)));
-
-    render_queue_buffer_.resize(render_queue_element_max_size_);
-    capture_queue_buffer_.resize(render_queue_element_max_size_);
-  } else {
-    render_signal_queue_->Clear();
   }
 }
 
@@ -573,10 +506,10 @@ int EchoCancellationImpl::Configure() {
   return error;
 }
 
-size_t EchoCancellationImpl::NumCancellersRequired() const {
+size_t EchoCancellationImpl::NumCancellersRequired(
-  RTC_DCHECK(stream_properties_);
+    size_t num_output_channels,
-  return stream_properties_->num_output_channels *
+    size_t num_reverse_channels) {
-         stream_properties_->num_reverse_channels;
+  return num_output_channels * num_reverse_channels;
 }
 
 }  // namespace webrtc

webrtc/modules/audio_processing/echo_cancellation_impl.h

@@ -16,9 +16,7 @@
 
 #include "webrtc/base/constructormagic.h"
 #include "webrtc/base/criticalsection.h"
-#include "webrtc/base/swap_queue.h"
 #include "webrtc/modules/audio_processing/include/audio_processing.h"
-#include "webrtc/modules/audio_processing/render_queue_item_verifier.h"
 
 namespace webrtc {
 
@@ -30,7 +28,7 @@ class EchoCancellationImpl : public EchoCancellation {
                        rtc::CriticalSection* crit_capture);
   ~EchoCancellationImpl() override;
 
-  int ProcessRenderAudio(const AudioBuffer* audio);
+  void ProcessRenderAudio(rtc::ArrayView<const float> packed_render_audio);
   int ProcessCaptureAudio(AudioBuffer* audio, int stream_delay_ms);
 
   // EchoCancellation implementation.
@@ -50,13 +48,16 @@ class EchoCancellationImpl : public EchoCancellation {
   std::string GetExperimentsDescription();
   bool is_refined_adaptive_filter_enabled() const;
 
-  // Reads render side data that has been queued on the render call.
-  // Called holding the capture lock.
-  void ReadQueuedRenderData();
-
   // Returns the system delay of the first AEC component.
   int GetSystemDelayInSamples() const;
 
+  static void PackRenderAudioBuffer(const AudioBuffer* audio,
+                                    size_t num_output_channels,
+                                    size_t num_channels,
+                                    std::vector<float>* packed_buffer);
+  static size_t NumCancellersRequired(size_t num_output_channels,
+                                      size_t num_reverse_channels);
+
  private:
   class Canceller;
   struct StreamProperties;
@@ -79,8 +80,6 @@ class EchoCancellationImpl : public EchoCancellation {
 
   struct AecCore* aec_core() const override;
 
-  size_t NumCancellersRequired() const;
-
   void AllocateRenderQueue();
   int Configure();
 
@@ -100,15 +99,6 @@ class EchoCancellationImpl : public EchoCancellation {
   bool aec3_enabled_ GUARDED_BY(crit_capture_);
   bool refined_adaptive_filter_enabled_ GUARDED_BY(crit_capture_) = false;
 
-  size_t render_queue_element_max_size_ GUARDED_BY(crit_render_)
-      GUARDED_BY(crit_capture_);
-  std::vector<float> render_queue_buffer_ GUARDED_BY(crit_render_);
-  std::vector<float> capture_queue_buffer_ GUARDED_BY(crit_capture_);
-
-  // Lock protection not needed.
-  std::unique_ptr<SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>>
-      render_signal_queue_;
-
   std::vector<std::unique_ptr<Canceller>> cancellers_;
   std::unique_ptr<StreamProperties> stream_properties_;