Remove additional channel constraints when Beamforming is enabled in AudioProcessing

The general constraints on number of channels for AudioProcessing is:
num_in_channels == num_out_channels || num_out_channels == 1

When Beamforming is enabled and additional constraint was added forcing:
num_out_channels == 1

This artificial constraint was removed by adding upmixing support in CopyTo, since it was already supported for the AudioFrame interface using InterleaveTo.

Review URL: https://codereview.webrtc.org/1571013002

Cr-Commit-Position: refs/heads/master@{#11215}

talk/media/webrtc/fakewebrtcvoiceengine.h

@@ -78,6 +78,7 @@ class FakeAudioProcessing : public webrtc::AudioProcessing {
   WEBRTC_STUB_CONST(proc_sample_rate_hz, ());
   WEBRTC_STUB_CONST(proc_split_sample_rate_hz, ());
   WEBRTC_STUB_CONST(num_input_channels, ());
+  WEBRTC_STUB_CONST(num_proc_channels, ());
   WEBRTC_STUB_CONST(num_output_channels, ());
   WEBRTC_STUB_CONST(num_reverse_channels, ());
   WEBRTC_VOID_STUB(set_output_will_be_muted, (bool muted));

webrtc/modules/audio_processing/audio_buffer.cc

@@ -150,7 +150,7 @@ void AudioBuffer::CopyFrom(const float* const* data,
 void AudioBuffer::CopyTo(const StreamConfig& stream_config,
                          float* const* data) {
   assert(stream_config.num_frames() == output_num_frames_);
-  assert(stream_config.num_channels() == num_channels_);
+  assert(stream_config.num_channels() == num_channels_ || num_channels_ == 1);
 
   // Convert to the float range.
   float* const* data_ptr = data;
@@ -173,6 +173,11 @@ void AudioBuffer::CopyTo(const StreamConfig& stream_config,
                                       output_num_frames_);
     }
   }
+
+  // Upmix.
+  for (int i = num_channels_; i < stream_config.num_channels(); ++i) {
+    memcpy(data[i], data[0], output_num_frames_ * sizeof(**data));
+  }
 }
 
 void AudioBuffer::InitForNewData() {

webrtc/modules/audio_processing/audio_processing_impl.cc

@@ -226,9 +226,9 @@ AudioProcessingImpl::AudioProcessingImpl(const Config& config,
 #else
       capture_(config.Get<ExperimentalNs>().enabled,
 #endif
-               config.Get<Beamforming>().enabled,
                config.Get<Beamforming>().array_geometry,
-               config.Get<Beamforming>().target_direction)
+               config.Get<Beamforming>().target_direction),
+      capture_nonlocked_(config.Get<Beamforming>().enabled)
 {
   {
     rtc::CritScope cs_render(&crit_render_);
@@ -345,7 +345,7 @@ int AudioProcessingImpl::MaybeInitialize(
 
 int AudioProcessingImpl::InitializeLocked() {
   const int fwd_audio_buffer_channels =
-      capture_.beamformer_enabled
+      capture_nonlocked_.beamformer_enabled
           ? formats_.api_format.input_stream().num_channels()
           : formats_.api_format.output_stream().num_channels();
   const int rev_audio_buffer_out_num_frames =
@@ -428,9 +428,8 @@ int AudioProcessingImpl::InitializeLocked(const ProcessingConfig& config) {
     return kBadNumberChannelsError;
   }
 
-  if (capture_.beamformer_enabled &&
+  if (capture_nonlocked_.beamformer_enabled &&
-      (static_cast<size_t>(num_in_channels) != capture_.array_geometry.size() ||
+      static_cast<size_t>(num_in_channels) != capture_.array_geometry.size()) {
-       num_out_channels > 1)) {
     return kBadNumberChannelsError;
   }
 
@@ -500,8 +499,9 @@ void AudioProcessingImpl::SetExtraOptions(const Config& config) {
   }
 
 #ifdef WEBRTC_ANDROID_PLATFORM_BUILD
-  if (capture_.beamformer_enabled != config.Get<Beamforming>().enabled) {
+  if (capture_nonlocked_.beamformer_enabled !=
-    capture_.beamformer_enabled = config.Get<Beamforming>().enabled;
+          config.Get<Beamforming>().enabled) {
+    capture_nonlocked_.beamformer_enabled = config.Get<Beamforming>().enabled;
     if (config.Get<Beamforming>().array_geometry.size() > 1) {
       capture_.array_geometry = config.Get<Beamforming>().array_geometry;
     }
@@ -537,6 +537,11 @@ int AudioProcessingImpl::num_input_channels() const {
   return formats_.api_format.input_stream().num_channels();
 }
 
+int AudioProcessingImpl::num_proc_channels() const {
+  // Used as callback from submodules, hence locking is not allowed.
+  return capture_nonlocked_.beamformer_enabled ? 1 : num_output_channels();
+}
+
 int AudioProcessingImpl::num_output_channels() const {
   // Used as callback from submodules, hence locking is not allowed.
   return formats_.api_format.output_stream().num_channels();
@@ -771,7 +776,7 @@ int AudioProcessingImpl::ProcessStreamLocked() {
         ca->num_channels());
   }
 
-  if (capture_.beamformer_enabled) {
+  if (capture_nonlocked_.beamformer_enabled) {
     private_submodules_->beamformer->ProcessChunk(*ca->split_data_f(),
                                                   ca->split_data_f());
     ca->set_num_channels(1);
@@ -793,7 +798,7 @@ int AudioProcessingImpl::ProcessStreamLocked() {
 
   if (constants_.use_new_agc &&
       public_submodules_->gain_control->is_enabled() &&
-      (!capture_.beamformer_enabled ||
+      (!capture_nonlocked_.beamformer_enabled ||
        private_submodules_->beamformer->is_target_present())) {
     private_submodules_->agc_manager->Process(
         ca->split_bands_const(0)[kBand0To8kHz], ca->num_frames_per_band(),
@@ -1183,7 +1188,7 @@ VoiceDetection* AudioProcessingImpl::voice_detection() const {
 }
 
 bool AudioProcessingImpl::is_data_processed() const {
-  if (capture_.beamformer_enabled) {
+  if (capture_nonlocked_.beamformer_enabled) {
     return true;
   }
 
@@ -1293,12 +1298,12 @@ void AudioProcessingImpl::InitializeTransient() {
     public_submodules_->transient_suppressor->Initialize(
         capture_nonlocked_.fwd_proc_format.sample_rate_hz(),
         capture_nonlocked_.split_rate,
-        formats_.api_format.output_stream().num_channels());
+        num_proc_channels());
   }
 }
 
 void AudioProcessingImpl::InitializeBeamformer() {
-  if (capture_.beamformer_enabled) {
+  if (capture_nonlocked_.beamformer_enabled) {
     if (!private_submodules_->beamformer) {
       private_submodules_->beamformer.reset(new NonlinearBeamformer(
           capture_.array_geometry, capture_.target_direction));
@@ -1320,12 +1325,12 @@ void AudioProcessingImpl::InitializeIntelligibility() {
 }
 
 void AudioProcessingImpl::InitializeHighPassFilter() {
-  public_submodules_->high_pass_filter->Initialize(num_output_channels(),
+  public_submodules_->high_pass_filter->Initialize(num_proc_channels(),
                                                    proc_sample_rate_hz());
 }
 
 void AudioProcessingImpl::InitializeNoiseSuppression() {
-  public_submodules_->noise_suppression->Initialize(num_output_channels(),
+  public_submodules_->noise_suppression->Initialize(num_proc_channels(),
                                                     proc_sample_rate_hz());
 }
 

webrtc/modules/audio_processing/audio_processing_impl.h

@@ -102,6 +102,7 @@ class AudioProcessingImpl : public AudioProcessing {
   int proc_sample_rate_hz() const override;
   int proc_split_sample_rate_hz() const override;
   int num_input_channels() const override;
+  int num_proc_channels() const override;
   int num_output_channels() const override;
   int num_reverse_channels() const override;
   int stream_delay_ms() const override;
@@ -280,7 +281,6 @@ class AudioProcessingImpl : public AudioProcessing {
 
   struct ApmCaptureState {
     ApmCaptureState(bool transient_suppressor_enabled,
-                    bool beamformer_enabled,
                     const std::vector<Point>& array_geometry,
                     SphericalPointf target_direction)
         : aec_system_delay_jumps(-1),
@@ -292,7 +292,6 @@ class AudioProcessingImpl : public AudioProcessing {
           output_will_be_muted(false),
           key_pressed(false),
           transient_suppressor_enabled(transient_suppressor_enabled),
-          beamformer_enabled(beamformer_enabled),
           array_geometry(array_geometry),
           target_direction(target_direction),
           fwd_proc_format(kSampleRate16kHz),
@@ -306,7 +305,6 @@ class AudioProcessingImpl : public AudioProcessing {
     bool output_will_be_muted;
     bool key_pressed;
     bool transient_suppressor_enabled;
-    bool beamformer_enabled;
     std::vector<Point> array_geometry;
     SphericalPointf target_direction;
     rtc::scoped_ptr<AudioBuffer> capture_audio;
@@ -318,16 +316,18 @@ class AudioProcessingImpl : public AudioProcessing {
   } capture_ GUARDED_BY(crit_capture_);
 
   struct ApmCaptureNonLockedState {
-    ApmCaptureNonLockedState()
+    ApmCaptureNonLockedState(bool beamformer_enabled)
         : fwd_proc_format(kSampleRate16kHz),
           split_rate(kSampleRate16kHz),
-          stream_delay_ms(0) {}
+          stream_delay_ms(0),
+          beamformer_enabled(beamformer_enabled) {}
     // Only the rate and samples fields of fwd_proc_format_ are used because the
     // forward processing number of channels is mutable and is tracked by the
     // capture_audio_.
     StreamConfig fwd_proc_format;
     int split_rate;
     int stream_delay_ms;
+    bool beamformer_enabled;
   } capture_nonlocked_;
 
   struct ApmRenderState {

webrtc/modules/audio_processing/echo_cancellation_impl.cc

@@ -174,7 +174,7 @@ int EchoCancellationImpl::ProcessCaptureAudio(AudioBuffer* audio) {
   }
 
   assert(audio->num_frames_per_band() <= 160);
-  assert(audio->num_channels() == apm_->num_output_channels());
+  assert(audio->num_channels() == apm_->num_proc_channels());
 
   int err = AudioProcessing::kNoError;
 

webrtc/modules/audio_processing/gain_control_impl.cc

@@ -435,7 +435,7 @@ int GainControlImpl::ConfigureHandle(void* handle) const {
 
 int GainControlImpl::num_handles_required() const {
   // Not locked as it only relies on APM public API which is threadsafe.
-  return apm_->num_output_channels();
+  return apm_->num_proc_channels();
 }
 
 int GainControlImpl::GetHandleError(void* handle) const {

webrtc/modules/audio_processing/include/audio_processing.h

@@ -288,6 +288,7 @@ class AudioProcessing {
   virtual int proc_sample_rate_hz() const = 0;
   virtual int proc_split_sample_rate_hz() const = 0;
   virtual int num_input_channels() const = 0;
+  virtual int num_proc_channels() const = 0;
   virtual int num_output_channels() const = 0;
   virtual int num_reverse_channels() const = 0;