Make VectorBuffer in AEC3 multi-channel

All dependent modules are hardcoded to do their regular mono processing on the first channel.

This _almost_ makes RenderBuffer multi-channel: FftData is still only mono.

Bug: webrtc:10913
Change-Id: Id5cc34dbabfe59e1cc72a9675dc7979794e870ed
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/151139
Commit-Queue: Sam Zackrisson <saza@webrtc.org>
Reviewed-by: Per Åhgren <peah@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#29074}

modules/audio_processing/aec3/aec_state.cc

@@ -153,7 +153,8 @@ void AecState::Update(
     erle_estimator_.Reset(false);
   }
 
-  const auto& X2 = render_buffer.Spectrum(delay_state_.DirectPathFilterDelay());
+  const auto& X2 = render_buffer.Spectrum(delay_state_.DirectPathFilterDelay(),
+                                          /*channel=*/0);
   const auto& X2_input_erle = X2_reverb;
 
   erle_estimator_.Update(render_buffer, adaptive_filter_frequency_response,

modules/audio_processing/aec3/echo_audibility.cc

@@ -83,7 +83,8 @@ void EchoAudibility::UpdateRenderNoiseEstimator(
     for (int idx = render_spectrum_write_prev_.value();
          idx != render_spectrum_write_current;
          idx = spectrum_buffer.DecIndex(idx)) {
-      render_stationarity_.UpdateNoiseEstimator(spectrum_buffer.buffer[idx]);
+      render_stationarity_.UpdateNoiseEstimator(
+          spectrum_buffer.buffer[idx][/*channel=*/0]);
     }
   }
   render_spectrum_write_prev_ = render_spectrum_write_current;

modules/audio_processing/aec3/echo_remover.cc

@@ -346,7 +346,8 @@ void EchoRemoverImpl::ProcessCapture(
   data_dumper_->DumpRaw("aec3_S2_linear", S2_linear);
   data_dumper_->DumpRaw("aec3_Y2", Y2);
   data_dumper_->DumpRaw(
-      "aec3_X2", render_buffer->Spectrum(aec_state_.FilterDelayBlocks()));
+      "aec3_X2",
+      render_buffer->Spectrum(aec_state_.FilterDelayBlocks(), /*channel=*/0));
   data_dumper_->DumpRaw("aec3_R2", R2);
   data_dumper_->DumpRaw("aec3_R2_reverb",
                         residual_echo_estimator_.GetReverbPowerSpectrum());

modules/audio_processing/aec3/erle_estimator_unittest.cc

@@ -70,7 +70,8 @@ void FormFarendFrame(const RenderBuffer& render_buffer,
                      std::array<float, kFftLengthBy2Plus1>* Y2,
                      float erle) {
   const auto& spectrum_buffer = render_buffer.GetSpectrumBuffer();
-  const auto& X2_from_buffer = spectrum_buffer.buffer[spectrum_buffer.write];
+  const auto& X2_from_buffer =
+      spectrum_buffer.buffer[spectrum_buffer.write][/*channel=*/0];
   std::copy(X2_from_buffer.begin(), X2_from_buffer.end(), X2->begin());
   std::transform(X2->begin(), X2->end(), Y2->begin(),
                  [](float a) { return a * kEchoPathGain * kEchoPathGain; });

modules/audio_processing/aec3/mock/mock_render_delay_buffer.cc

@@ -19,7 +19,9 @@ MockRenderDelayBuffer::MockRenderDelayBuffer(int sample_rate_hz,
                     NumBandsForRate(sample_rate_hz),
                     num_channels,
                     kBlockSize),
-      spectrum_buffer_(block_buffer_.buffer.size(), kFftLengthBy2Plus1),
+      spectrum_buffer_(block_buffer_.buffer.size(),
+                       num_channels,
+                       kFftLengthBy2Plus1),
       fft_buffer_(block_buffer_.buffer.size()),
       render_buffer_(&block_buffer_, &spectrum_buffer_, &fft_buffer_),
       downsampled_render_buffer_(GetDownSampledBufferSize(4, 4)) {

modules/audio_processing/aec3/render_buffer.cc

@@ -41,9 +41,10 @@ void RenderBuffer::SpectralSum(
   X2->fill(0.f);
   int position = spectrum_buffer_->read;
   for (size_t j = 0; j < num_spectra; ++j) {
-    std::transform(X2->begin(), X2->end(),
-                   spectrum_buffer_->buffer[position].begin(), X2->begin(),
-                   std::plus<float>());
+    for (const auto& channel_spectrum : spectrum_buffer_->buffer[position]) {
+      std::transform(X2->begin(), X2->end(), channel_spectrum.begin(),
+                     X2->begin(), std::plus<float>());
+    }
     position = spectrum_buffer_->IncIndex(position);
   }
 }
@@ -58,16 +59,20 @@ void RenderBuffer::SpectralSums(
   int position = spectrum_buffer_->read;
   size_t j = 0;
   for (; j < num_spectra_shorter; ++j) {
-    std::transform(X2_shorter->begin(), X2_shorter->end(),
-                   spectrum_buffer_->buffer[position].begin(),
-                   X2_shorter->begin(), std::plus<float>());
+    for (const auto& channel_spectrum : spectrum_buffer_->buffer[position]) {
+      std::transform(X2_shorter->begin(), X2_shorter->end(),
+                     channel_spectrum.begin(), X2_shorter->begin(),
+                     std::plus<float>());
+    }
     position = spectrum_buffer_->IncIndex(position);
   }
   std::copy(X2_shorter->begin(), X2_shorter->end(), X2_longer->begin());
   for (; j < num_spectra_longer; ++j) {
-    std::transform(X2_longer->begin(), X2_longer->end(),
-                   spectrum_buffer_->buffer[position].begin(),
-                   X2_longer->begin(), std::plus<float>());
+    for (const auto& channel_spectrum : spectrum_buffer_->buffer[position]) {
+      std::transform(X2_longer->begin(), X2_longer->end(),
+                     channel_spectrum.begin(), X2_longer->begin(),
+                     std::plus<float>());
+    }
     position = spectrum_buffer_->IncIndex(position);
   }
 }

modules/audio_processing/aec3/render_buffer.h

@@ -44,10 +44,11 @@ class RenderBuffer {
   }
 
   // Get the spectrum from one of the FFTs in the buffer.
-  rtc::ArrayView<const float> Spectrum(int buffer_offset_ffts) const {
+  rtc::ArrayView<const float> Spectrum(int buffer_offset_ffts,
+                                       size_t channel) const {
     int position = spectrum_buffer_->OffsetIndex(spectrum_buffer_->read,
                                                  buffer_offset_ffts);
-    return spectrum_buffer_->buffer[position];
+    return spectrum_buffer_->buffer[position][channel];
   }
 
   // Returns the circular fft buffer.

modules/audio_processing/aec3/render_buffer_unittest.cc

@@ -23,7 +23,7 @@ namespace webrtc {
 // Verifies the check for non-null fft buffer.
 TEST(RenderBuffer, NullExternalFftBuffer) {
   MatrixBuffer block_buffer(10, 3, 1, kBlockSize);
-  VectorBuffer spectrum_buffer(10, kFftLengthBy2Plus1);
+  VectorBuffer spectrum_buffer(10, 1, kFftLengthBy2Plus1);
   EXPECT_DEATH(RenderBuffer(&block_buffer, &spectrum_buffer, nullptr), "");
 }
 
@@ -37,7 +37,7 @@ TEST(RenderBuffer, NullExternalSpectrumBuffer) {
 // Verifies the check for non-null block buffer.
 TEST(RenderBuffer, NullExternalBlockBuffer) {
   FftBuffer fft_buffer(10);
-  VectorBuffer spectrum_buffer(10, kFftLengthBy2Plus1);
+  VectorBuffer spectrum_buffer(10, 1, kFftLengthBy2Plus1);
   EXPECT_DEATH(RenderBuffer(nullptr, &spectrum_buffer, &fft_buffer), "");
 }
 

modules/audio_processing/aec3/render_delay_buffer.cc

@@ -127,7 +127,7 @@ RenderDelayBufferImpl::RenderDelayBufferImpl(const EchoCanceller3Config& config,
               NumBandsForRate(sample_rate_hz),
               num_render_channels,
               kBlockSize),
-      spectra_(blocks_.buffer.size(), kFftLengthBy2Plus1),
+      spectra_(blocks_.buffer.size(), num_render_channels, kFftLengthBy2Plus1),
       ffts_(blocks_.buffer.size()),
       delay_(config_.delay.default_delay),
       echo_remover_buffer_(&blocks_, &spectra_, &ffts_),
@@ -381,7 +381,9 @@ void RenderDelayBufferImpl::InsertBlock(
   std::copy(ds.rbegin(), ds.rend(), lr.buffer.begin() + lr.write);
   fft_.PaddedFft(block[0][0], b.buffer[previous_write][0][0],
                  &f.buffer[f.write]);
-  f.buffer[f.write].Spectrum(optimization_, s.buffer[s.write]);
+  // TODO(http://bugs.webrtc.org/10913): Loop over all channels when FftBuffer
+  // supports multi-channel.
+  f.buffer[f.write].Spectrum(optimization_, s.buffer[s.write][/*channel=*/0]);
 }
 
 bool RenderDelayBufferImpl::DetectActiveRender(

modules/audio_processing/aec3/render_reverb_model.cc

@@ -34,11 +34,12 @@ void RenderReverbModel::Apply(const VectorBuffer& spectrum_buffer,
   int idx_at_delay =
       spectrum_buffer.OffsetIndex(spectrum_buffer.read, delay_blocks);
   int idx_past = spectrum_buffer.IncIndex(idx_at_delay);
-  const auto& X2 = spectrum_buffer.buffer[idx_at_delay];
+  const auto& X2 = spectrum_buffer.buffer[idx_at_delay][/*channel=*/0];
   RTC_DCHECK_EQ(X2.size(), reverb_power_spectrum.size());
   std::copy(X2.begin(), X2.end(), reverb_power_spectrum.begin());
-  render_reverb_.AddReverbNoFreqShaping(spectrum_buffer.buffer[idx_past], 1.0f,
-                                        reverb_decay, reverb_power_spectrum);
+  render_reverb_.AddReverbNoFreqShaping(
+      spectrum_buffer.buffer[idx_past][/*channel=*/0], 1.0f, reverb_decay,
+      reverb_power_spectrum);
 }
 
 }  // namespace webrtc

modules/audio_processing/aec3/render_signal_analyzer.cc

@@ -34,7 +34,8 @@ void IdentifySmallNarrowBandRegions(
     return;
   }
 
-  rtc::ArrayView<const float> X2 = render_buffer.Spectrum(*delay_partitions);
+  rtc::ArrayView<const float> X2 =
+      render_buffer.Spectrum(*delay_partitions, /*channel=*/0);
   RTC_DCHECK_EQ(kFftLengthBy2Plus1, X2.size());
 
   for (size_t k = 1; k < (X2.size() - 1); ++k) {
@@ -49,7 +50,7 @@ void IdentifyStrongNarrowBandComponent(const RenderBuffer& render_buffer,
                                        int strong_peak_freeze_duration,
                                        absl::optional<int>* narrow_peak_band,
                                        size_t* narrow_peak_counter) {
-  const auto X2_latest = render_buffer.Spectrum(0);
+  const auto X2_latest = render_buffer.Spectrum(0, /*channel=*/0);
 
   // Identify the spectral peak.
   const int peak_bin = static_cast<int>(

modules/audio_processing/aec3/residual_echo_estimator.cc

@@ -77,7 +77,8 @@ void ResidualEchoEstimator::Estimate(
     // Adds the estimated unmodelled echo power to the residual echo power
     // estimate.
     echo_reverb_.AddReverb(
-        render_buffer.Spectrum(aec_state.FilterLengthBlocks() + 1),
+        render_buffer.Spectrum(aec_state.FilterLengthBlocks() + 1,
+                               /*channel=*/0),
         aec_state.GetReverbFrequencyResponse(), aec_state.ReverbDecay(), *R2);
   } else {
     // Estimate the echo generating signal power.
@@ -108,7 +109,8 @@ void ResidualEchoEstimator::Estimate(
 
     if (!(aec_state.TransparentMode())) {
       echo_reverb_.AddReverbNoFreqShaping(
-          render_buffer.Spectrum(aec_state.FilterDelayBlocks() + 1),
+          render_buffer.Spectrum(aec_state.FilterDelayBlocks() + 1,
+                                 /*channel=*/0),
           echo_path_gain * echo_path_gain, aec_state.ReverbDecay(), *R2);
     }
   }
@@ -171,7 +173,8 @@ void ResidualEchoEstimator::EchoGeneratingPower(
 
   X2->fill(0.f);
   for (int k = idx_start; k != idx_stop; k = spectrum_buffer.IncIndex(k)) {
-    std::transform(X2->begin(), X2->end(), spectrum_buffer.buffer[k].begin(),
+    std::transform(X2->begin(), X2->end(),
+                   spectrum_buffer.buffer[k][/*channel=*/0].begin(),
                    X2->begin(),
                    [](float a, float b) { return std::max(a, b); });
   }
@@ -194,7 +197,7 @@ void ResidualEchoEstimator::RenderNoisePower(
   RTC_DCHECK(X2_noise_floor);
   RTC_DCHECK(X2_noise_floor_counter);
 
-  const auto render_power = render_buffer.Spectrum(0);
+  const auto render_power = render_buffer.Spectrum(0, /*channel=*/0);
   RTC_DCHECK_EQ(X2_noise_floor->size(), render_power.size());
   RTC_DCHECK_EQ(X2_noise_floor_counter->size(), render_power.size());
 

modules/audio_processing/aec3/signal_dependent_erle_estimator.cc

@@ -332,9 +332,10 @@ void SignalDependentErleEstimator::ComputeEchoEstimatePerFilterSection(
                                         filter_frequency_response.size());
     for (size_t block = section_boundaries_blocks_[section];
          block < block_limit; ++block) {
-      std::transform(X2_section.begin(), X2_section.end(),
-                     spectrum_render_buffer.buffer[idx_render].begin(),
-                     X2_section.begin(), std::plus<float>());
+      std::transform(
+          X2_section.begin(), X2_section.end(),
+          spectrum_render_buffer.buffer[idx_render][/*channel=*/0].begin(),
+          X2_section.begin(), std::plus<float>());
       std::transform(H2_section.begin(), H2_section.end(),
                      filter_frequency_response[block].begin(),
                      H2_section.begin(), std::plus<float>());

modules/audio_processing/aec3/signal_dependent_erle_estimator_unittest.cc

@@ -99,8 +99,8 @@ void TestInputs::UpdateCurrentPowerSpectra() {
       render_buffer_->GetSpectrumBuffer();
   size_t idx = render_buffer_->Position();
   size_t prev_idx = spectrum_render_buffer.OffsetIndex(idx, 1);
-  auto& X2 = spectrum_render_buffer.buffer[idx];
-  auto& X2_prev = spectrum_render_buffer.buffer[prev_idx];
+  auto& X2 = spectrum_render_buffer.buffer[idx][/*channel=*/0];
+  auto& X2_prev = spectrum_render_buffer.buffer[prev_idx][/*channel=*/0];
   std::copy(X2.begin(), X2.end(), X2_.begin());
   RTC_DCHECK_EQ(X2.size(), Y2_.size());
   for (size_t k = 0; k < X2.size(); ++k) {

modules/audio_processing/aec3/stationarity_estimator.cc

@@ -105,7 +105,7 @@ bool StationarityEstimator::EstimateBandStationarity(
   constexpr float kThrStationarity = 10.f;
   float acum_power = 0.f;
   for (auto idx : indexes) {
-    acum_power += spectrum_buffer.buffer[idx][band];
+    acum_power += spectrum_buffer.buffer[idx][/*channel=*/0][band];
   }
   acum_power += reverb[band];
   float noise = kWindowLength * GetStationarityPowerBand(band);

modules/audio_processing/aec3/vector_buffer.cc

@@ -14,11 +14,18 @@
 
 namespace webrtc {
 
-VectorBuffer::VectorBuffer(size_t size, size_t height)
+VectorBuffer::VectorBuffer(size_t size,
+                           size_t num_channels,
+                           size_t spectrum_length)
     : size(static_cast<int>(size)),
-      buffer(size, std::vector<float>(height, 0.f)) {
-  for (auto& c : buffer) {
-    std::fill(c.begin(), c.end(), 0.f);
+      buffer(size,
+             std::vector<std::vector<float>>(
+                 num_channels,
+                 std::vector<float>(spectrum_length, 0.f))) {
+  for (auto& channel : buffer) {
+    for (auto& c : channel) {
+      std::fill(c.begin(), c.end(), 0.f);
+    }
   }
 }
 

modules/audio_processing/aec3/vector_buffer.h

@@ -21,8 +21,9 @@ namespace webrtc {
 
 // Struct for bundling a circular buffer of one dimensional vector objects
 // together with the read and write indices.
+// TODO(peah): Change name of this class to be more specific to what it does.
 struct VectorBuffer {
-  VectorBuffer(size_t size, size_t height);
+  VectorBuffer(size_t size, size_t num_channels, size_t spectrum_length);
   ~VectorBuffer();
 
   int IncIndex(int index) const {
@@ -50,7 +51,7 @@ struct VectorBuffer {
   void DecReadIndex() { read = DecIndex(read); }
 
   const int size;
-  std::vector<std::vector<float>> buffer;
+  std::vector<std::vector<std::vector<float>>> buffer;
   int write = 0;
   int read = 0;
 };