zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Only create AECm when needed

Browse Source 
This CL ensures that the AECm is only created when needed.
The changes in the CL are bitexact when running AECm via
audioproc_f

The CL also corrects an issue where there is a risk for
AEC2 to not be correctly setup when the sample rate
changes inbetween activations.

Bug: webrtc:8671
Change-Id: Id3b33e20969b1543e28c885d47495246cfbe549d
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/134216
Commit-Queue: Per Åhgren <peah@webrtc.org>
Reviewed-by: Sam Zackrisson <saza@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#27800}
This commit is contained in:
Per Åhgren
2019-04-29 12:14:50 +02:00
committed by Commit Bot
parent 8a9778efa4
commit b6e24d7f35
2 changed files with 112 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

230
modules/audio_processing/audio_processing_impl.cc
View File

@ -428,7 +428,6 @@ AudioProcessingImpl::AudioProcessingImpl(
new rtc::RefCountedObject<ResidualEchoDetector>();
}
private_submodules_->echo_control_mobile.reset(new EchoControlMobileImpl());
// TODO(alessiob): Move the injected gain controller once injection is
// implemented.
private_submodules_->gain_controller2.reset(new GainController2());
@ -548,10 +547,6 @@ int AudioProcessingImpl::InitializeLocked() {
AllocateRenderQueue();
private_submodules_->echo_control_mobile->Initialize(
proc_split_sample_rate_hz(), num_reverse_channels(),
num_output_channels());
public_submodules_->gain_control->Initialize(num_proc_channels(),
proc_sample_rate_hz());
if (constants_.use_experimental_agc) {
@ -1036,15 +1031,15 @@ void AudioProcessingImpl::HandleRenderRuntimeSettings() {
}
void AudioProcessingImpl::QueueBandedRenderAudio(AudioBuffer* audio) {
EchoCancellationImpl::PackRenderAudioBuffer(audio, num_output_channels(),
num_reverse_channels(),
&aec_render_queue_buffer_);
RTC_DCHECK_GE(160, audio->num_frames_per_band());
// Insert the samples into the queue.
if (private_submodules_->echo_cancellation) {
RTC_DCHECK(aec_render_signal_queue_);
EchoCancellationImpl::PackRenderAudioBuffer(audio, num_output_channels(),
num_reverse_channels(),
&aec_render_queue_buffer_);
if (!aec_render_signal_queue_->Insert(&aec_render_queue_buffer_)) {
// The data queue is full and needs to be emptied.
EmptyQueuedRenderAudio();
@ -1055,18 +1050,21 @@ void AudioProcessingImpl::QueueBandedRenderAudio(AudioBuffer* audio) {
}
}
EchoControlMobileImpl::PackRenderAudioBuffer(audio, num_output_channels(),
num_reverse_channels(),
&aecm_render_queue_buffer_);
if (private_submodules_->echo_control_mobile) {
EchoControlMobileImpl::PackRenderAudioBuffer(audio, num_output_channels(),
num_reverse_channels(),
&aecm_render_queue_buffer_);
RTC_DCHECK(aecm_render_signal_queue_);
// Insert the samples into the queue.
if (!aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_)) {
// The data queue is full and needs to be emptied.
EmptyQueuedRenderAudio();
// Insert the samples into the queue.
if (!aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_)) {
// The data queue is full and needs to be emptied.
EmptyQueuedRenderAudio();
// Retry the insert (should always work).
bool result = aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_);
RTC_DCHECK(result);
// Retry the insert (should always work).
bool result =
aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_);
RTC_DCHECK(result);
}
}
if (!constants_.use_experimental_agc) {
@ -1098,12 +1096,6 @@ void AudioProcessingImpl::QueueNonbandedRenderAudio(AudioBuffer* audio) {
}
void AudioProcessingImpl::AllocateRenderQueue() {
const size_t new_aecm_render_queue_element_max_size =
std::max(static_cast<size_t>(1),
kMaxAllowedValuesOfSamplesPerBand *
EchoControlMobileImpl::NumCancellersRequired(
num_output_channels(), num_reverse_channels()));
const size_t new_agc_render_queue_element_max_size =
std::max(static_cast<size_t>(1), kMaxAllowedValuesOfSamplesPerBand);
@ -1112,25 +1104,6 @@ void AudioProcessingImpl::AllocateRenderQueue() {
// Reallocate the queues if the queue item sizes are too small to fit the
// data to put in the queues.
if (aecm_render_queue_element_max_size_ <
new_aecm_render_queue_element_max_size) {
aecm_render_queue_element_max_size_ =
new_aecm_render_queue_element_max_size;
std::vector<int16_t> template_queue_element(
aecm_render_queue_element_max_size_);
aecm_render_signal_queue_.reset(
new SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>(
kMaxNumFramesToBuffer, template_queue_element,
RenderQueueItemVerifier<int16_t>(
aecm_render_queue_element_max_size_)));
aecm_render_queue_buffer_.resize(aecm_render_queue_element_max_size_);
aecm_capture_queue_buffer_.resize(aecm_render_queue_element_max_size_);
} else {
aecm_render_signal_queue_->Clear();
}
if (agc_render_queue_element_max_size_ <
new_agc_render_queue_element_max_size) {
@ -1181,9 +1154,12 @@ void AudioProcessingImpl::EmptyQueuedRenderAudio() {
}
}
while (aecm_render_signal_queue_->Remove(&aecm_capture_queue_buffer_)) {
private_submodules_->echo_control_mobile->ProcessRenderAudio(
aecm_capture_queue_buffer_);
if (private_submodules_->echo_control_mobile) {
RTC_DCHECK(aecm_render_signal_queue_);
while (aecm_render_signal_queue_->Remove(&aecm_capture_queue_buffer_)) {
private_submodules_->echo_control_mobile->ProcessRenderAudio(
aecm_capture_queue_buffer_);
}
}
while (agc_render_signal_queue_->Remove(&agc_capture_queue_buffer_)) {
@ -1272,9 +1248,10 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() {
// Ensure that not both the AEC and AECM are active at the same time.
// TODO(peah): Simplify once the public API Enable functions for these
// are moved to APM.
RTC_DCHECK(!((private_submodules_->echo_cancellation &&
private_submodules_->echo_cancellation->is_enabled()) &&
private_submodules_->echo_control_mobile->is_enabled()));
RTC_DCHECK_LE(!!private_submodules_->echo_controller +
!!private_submodules_->echo_cancellation +
!!private_submodules_->echo_control_mobile,
1);
MaybeUpdateHistograms();
@ -1356,47 +1333,44 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() {
public_submodules_->gain_control->AnalyzeCaptureAudio(capture_buffer));
public_submodules_->noise_suppression->AnalyzeCaptureAudio(capture_buffer);
// Ensure that the stream delay was set before the call to the
// AEC ProcessCaptureAudio function.
if (private_submodules_->echo_cancellation &&
private_submodules_->echo_cancellation->is_enabled() &&
!private_submodules_->echo_controller && !was_stream_delay_set()) {
return AudioProcessing::kStreamParameterNotSetError;
}
if (private_submodules_->echo_controller) {
data_dumper_->DumpRaw("stream_delay", stream_delay_ms());
if (was_stream_delay_set()) {
private_submodules_->echo_controller->SetAudioBufferDelay(
stream_delay_ms());
if (private_submodules_->echo_control_mobile) {
// Ensure that the stream delay was set before the call to the
// AECM ProcessCaptureAudio function.
if (!was_stream_delay_set()) {
return AudioProcessing::kStreamParameterNotSetError;
}
private_submodules_->echo_controller->ProcessCapture(
capture_buffer, capture_.echo_path_gain_change);
} else if (private_submodules_->echo_cancellation) {
RETURN_ON_ERR(private_submodules_->echo_cancellation->ProcessCaptureAudio(
capture_buffer, stream_delay_ms()));
}
if (public_submodules_->noise_suppression->is_enabled()) {
capture_buffer->CopyLowPassToReference();
}
if (private_submodules_->echo_control_mobile->is_enabled() &&
public_submodules_->noise_suppression->is_enabled()) {
capture_buffer->CopyLowPassToReference();
}
public_submodules_->noise_suppression->ProcessCaptureAudio(capture_buffer);
public_submodules_->noise_suppression->ProcessCaptureAudio(capture_buffer);
// Ensure that the stream delay was set before the call to the
// AECM ProcessCaptureAudio function.
if (private_submodules_->echo_control_mobile->is_enabled() &&
!was_stream_delay_set()) {
return AudioProcessing::kStreamParameterNotSetError;
}
if (!(private_submodules_->echo_controller ||
(private_submodules_->echo_cancellation &&
private_submodules_->echo_cancellation->is_enabled()))) {
RETURN_ON_ERR(private_submodules_->echo_control_mobile->ProcessCaptureAudio(
capture_buffer, stream_delay_ms()));
} else {
if (private_submodules_->echo_controller) {
data_dumper_->DumpRaw("stream_delay", stream_delay_ms());
if (was_stream_delay_set()) {
private_submodules_->echo_controller->SetAudioBufferDelay(
stream_delay_ms());
}
private_submodules_->echo_controller->ProcessCapture(
capture_buffer, capture_.echo_path_gain_change);
} else if (private_submodules_->echo_cancellation) {
// Ensure that the stream delay was set before the call to the
// AEC ProcessCaptureAudio function.
if (!was_stream_delay_set()) {
return AudioProcessing::kStreamParameterNotSetError;
}
RETURN_ON_ERR(private_submodules_->echo_cancellation->ProcessCaptureAudio(
capture_buffer, stream_delay_ms()));
}
public_submodules_->noise_suppression->ProcessCaptureAudio(capture_buffer);
}
public_submodules_->voice_detection->ProcessCaptureAudio(capture_buffer);
@ -1795,7 +1769,8 @@ bool AudioProcessingImpl::UpdateActiveSubmoduleStates() {
config_.high_pass_filter.enabled,
private_submodules_->echo_cancellation &&
private_submodules_->echo_cancellation->is_enabled(),
private_submodules_->echo_control_mobile->is_enabled(),
private_submodules_->echo_control_mobile &&
private_submodules_->echo_control_mobile->is_enabled(),
config_.residual_echo_detector.enabled,
public_submodules_->noise_suppression->is_enabled(),
public_submodules_->gain_control->is_enabled(),
@ -1828,9 +1803,13 @@ void AudioProcessingImpl::InitializeLowCutFilter() {
}
void AudioProcessingImpl::InitializeEchoController() {
if (echo_control_factory_ ||
bool use_echo_controller =
echo_control_factory_ ||
(config_.echo_canceller.enabled && !config_.echo_canceller.mobile_mode &&
!config_.echo_canceller.use_legacy_aec)) {
!config_.echo_canceller.use_legacy_aec);
if (use_echo_controller) {
// Create and activate the echo controller.
if (echo_control_factory_) {
private_submodules_->echo_controller =
echo_control_factory_->Create(proc_sample_rate_hz());
@ -1843,7 +1822,8 @@ void AudioProcessingImpl::InitializeEchoController() {
private_submodules_->echo_cancellation.reset();
aec_render_signal_queue_.reset();
private_submodules_->echo_control_mobile->Enable(false);
private_submodules_->echo_control_mobile.reset();
aecm_render_signal_queue_.reset();
return;
}
@ -1853,13 +1833,35 @@ void AudioProcessingImpl::InitializeEchoController() {
if (!config_.echo_canceller.enabled) {
private_submodules_->echo_cancellation.reset();
aec_render_signal_queue_.reset();
private_submodules_->echo_control_mobile->Enable(false);
private_submodules_->echo_control_mobile.reset();
aecm_render_signal_queue_.reset();
return;
}
if (config_.echo_canceller.mobile_mode) {
// Create and activate AECM.
// TODO(peah): Add an on-demand creation of AECmobile similar to AEC2.
size_t max_element_size =
std::max(static_cast<size_t>(1),
kMaxAllowedValuesOfSamplesPerBand *
EchoControlMobileImpl::NumCancellersRequired(
num_output_channels(), num_reverse_channels()));
std::vector<int16_t> template_queue_element(max_element_size);
aecm_render_signal_queue_.reset(
new SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>(
kMaxNumFramesToBuffer, template_queue_element,
RenderQueueItemVerifier<int16_t>(max_element_size)));
aecm_render_queue_buffer_.resize(max_element_size);
aecm_capture_queue_buffer_.resize(max_element_size);
private_submodules_->echo_control_mobile.reset(new EchoControlMobileImpl());
private_submodules_->echo_control_mobile->Initialize(
proc_split_sample_rate_hz(), num_reverse_channels(),
num_output_channels());
private_submodules_->echo_control_mobile->Enable(true);
private_submodules_->echo_cancellation.reset();
@ -1867,41 +1869,31 @@ void AudioProcessingImpl::InitializeEchoController() {
return;
}
private_submodules_->echo_control_mobile.reset();
aecm_render_signal_queue_.reset();
// Create and activate AEC2.
private_submodules_->echo_control_mobile->Enable(false);
private_submodules_->echo_cancellation.reset(new EchoCancellationImpl());
private_submodules_->echo_cancellation->SetExtraOptions(
capture_nonlocked_.use_aec2_extended_filter,
capture_nonlocked_.use_aec2_delay_agnostic,
capture_nonlocked_.use_aec2_refined_adaptive_filter);
const size_t new_aec_render_queue_element_max_size =
size_t element_max_size =
std::max(static_cast<size_t>(1),
kMaxAllowedValuesOfSamplesPerBand *
EchoCancellationImpl::NumCancellersRequired(
num_output_channels(), num_reverse_channels()));
// Reallocate the queues if the queue item sizes are too small to fit the
// data to put in the queues.
if (aec_render_queue_element_max_size_ <
new_aec_render_queue_element_max_size ||
!aec_render_signal_queue_) {
aec_render_queue_element_max_size_ = new_aec_render_queue_element_max_size;
std::vector<float> template_queue_element(element_max_size);
std::vector<float> template_queue_element(
aec_render_queue_element_max_size_);
aec_render_signal_queue_.reset(
new SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>(
kMaxNumFramesToBuffer, template_queue_element,
RenderQueueItemVerifier<float>(element_max_size)));
aec_render_signal_queue_.reset(
new SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>(
kMaxNumFramesToBuffer, template_queue_element,
RenderQueueItemVerifier<float>(
aec_render_queue_element_max_size_)));
aec_render_queue_buffer_.resize(aec_render_queue_element_max_size_);
aec_capture_queue_buffer_.resize(aec_render_queue_element_max_size_);
} else {
aec_render_signal_queue_->Clear();
}
aec_render_queue_buffer_.resize(element_max_size);
aec_capture_queue_buffer_.resize(element_max_size);
private_submodules_->echo_cancellation->Initialize(
proc_sample_rate_hz(), num_reverse_channels(), num_output_channels(),
@ -1913,9 +1905,6 @@ void AudioProcessingImpl::InitializeEchoController() {
config_.echo_canceller.legacy_moderate_suppression_level
? EchoCancellationImpl::SuppressionLevel::kModerateSuppression
: EchoCancellationImpl::SuppressionLevel::kHighSuppression);
private_submodules_->echo_controller.reset();
capture_nonlocked_.echo_controller_enabled = false;
}
void AudioProcessingImpl::InitializeGainController2() {
@ -2077,11 +2066,16 @@ void AudioProcessingImpl::WriteAecDumpConfigMessage(bool forced) {
: 0;
apm_config.aecm_enabled =
private_submodules_->echo_control_mobile &&
private_submodules_->echo_control_mobile->is_enabled();
apm_config.aecm_comfort_noise_enabled =
private_submodules_->echo_control_mobile &&
private_submodules_->echo_control_mobile->is_comfort_noise_enabled();
apm_config.aecm_routing_mode = static_cast<int>(
private_submodules_->echo_control_mobile->routing_mode());
apm_config.aecm_routing_mode =
private_submodules_->echo_control_mobile
? static_cast<int>(
private_submodules_->echo_control_mobile->routing_mode())
: 0;
apm_config.agc_enabled = public_submodules_->gain_control->is_enabled();
apm_config.agc_mode =