zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Only create AEC2 when needed

Browse Source 
This CL ensures that the AEC2 is only created when needed.
The changes in the CL are bitexact when running AEC2 via
audioproc_f

Bug: webrtc:8671
Change-Id: I5f6d33e45a7031c69ac53098781635c415668e49
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/129740
Reviewed-by: Sam Zackrisson <saza@webrtc.org>
Commit-Queue: Per Åhgren <peah@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#27772}
This commit is contained in:
Per Åhgren
2019-04-25 15:18:06 +02:00
committed by Commit Bot
parent e680c83a41
commit f204fafdb4
6 changed files with 144 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

218
modules/audio_processing/audio_processing_impl.cc
View File

@ -429,7 +429,6 @@ AudioProcessingImpl::AudioProcessingImpl(
new rtc::RefCountedObject<ResidualEchoDetector>();
}
private_submodules_->echo_cancellation.reset(new EchoCancellationImpl());
private_submodules_->echo_control_mobile.reset(new EchoControlMobileImpl());
// TODO(alessiob): Move the injected gain controller once injection is
// implemented.
@ -548,15 +547,8 @@ int AudioProcessingImpl::InitializeLocked() {
formats_.api_format.output_stream().num_channels(),
formats_.api_format.output_stream().num_frames()));
private_submodules_->echo_cancellation->Initialize(
proc_sample_rate_hz(), num_reverse_channels(), num_output_channels(),
num_proc_channels());
AllocateRenderQueue();
int success = private_submodules_->echo_cancellation->enable_metrics(true);
RTC_DCHECK_EQ(0, success);
success = private_submodules_->echo_cancellation->enable_delay_logging(true);
RTC_DCHECK_EQ(0, success);
private_submodules_->echo_control_mobile->Initialize(
proc_split_sample_rate_hz(), num_reverse_channels(),
num_output_channels());
@ -802,7 +794,12 @@ void AudioProcessingImpl::SetExtraOptions(const webrtc::Config& config) {
rtc::CritScope cs_render(&crit_render_);
rtc::CritScope cs_capture(&crit_capture_);
private_submodules_->echo_cancellation->SetExtraOptions(config);
capture_nonlocked_.use_aec2_extended_filter =
config.Get<ExtendedFilter>().enabled;
capture_nonlocked_.use_aec2_delay_agnostic =
config.Get<DelayAgnostic>().enabled;
capture_nonlocked_.use_aec2_refined_adaptive_filter =
config.Get<RefinedAdaptiveFilter>().enabled;
if (capture_.transient_suppressor_enabled !=
config.Get<ExperimentalNs>().enabled) {
@ -1035,13 +1032,16 @@ void AudioProcessingImpl::QueueBandedRenderAudio(AudioBuffer* audio) {
RTC_DCHECK_GE(160, audio->num_frames_per_band());
// Insert the samples into the queue.
if (!aec_render_signal_queue_->Insert(&aec_render_queue_buffer_)) {
// The data queue is full and needs to be emptied.
EmptyQueuedRenderAudio();
if (private_submodules_->echo_cancellation) {
RTC_DCHECK(aec_render_signal_queue_);
if (!aec_render_signal_queue_->Insert(&aec_render_queue_buffer_)) {
// The data queue is full and needs to be emptied.
EmptyQueuedRenderAudio();
// Retry the insert (should always work).
bool result = aec_render_signal_queue_->Insert(&aec_render_queue_buffer_);
RTC_DCHECK(result);
// Retry the insert (should always work).
bool result = aec_render_signal_queue_->Insert(&aec_render_queue_buffer_);
RTC_DCHECK(result);
}
}
EchoControlMobileImpl::PackRenderAudioBuffer(audio, num_output_channels(),
@ -1087,12 +1087,6 @@ void AudioProcessingImpl::QueueNonbandedRenderAudio(AudioBuffer* audio) {
}
void AudioProcessingImpl::AllocateRenderQueue() {
const size_t new_aec_render_queue_element_max_size =
std::max(static_cast<size_t>(1),
kMaxAllowedValuesOfSamplesPerBand *
EchoCancellationImpl::NumCancellersRequired(
num_output_channels(), num_reverse_channels()));
const size_t new_aecm_render_queue_element_max_size =
std::max(static_cast<size_t>(1),
kMaxAllowedValuesOfSamplesPerBand *
@ -1107,25 +1101,6 @@ void AudioProcessingImpl::AllocateRenderQueue() {
// Reallocate the queues if the queue item sizes are too small to fit the
// data to put in the queues.
if (aec_render_queue_element_max_size_ <
new_aec_render_queue_element_max_size) {
aec_render_queue_element_max_size_ = new_aec_render_queue_element_max_size;
std::vector<float> template_queue_element(
aec_render_queue_element_max_size_);
aec_render_signal_queue_.reset(
new SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>(
kMaxNumFramesToBuffer, template_queue_element,
RenderQueueItemVerifier<float>(
aec_render_queue_element_max_size_)));
aec_render_queue_buffer_.resize(aec_render_queue_element_max_size_);
aec_capture_queue_buffer_.resize(aec_render_queue_element_max_size_);
} else {
aec_render_signal_queue_->Clear();
}
if (aecm_render_queue_element_max_size_ <
new_aecm_render_queue_element_max_size) {
aecm_render_queue_element_max_size_ =
@ -1187,9 +1162,12 @@ void AudioProcessingImpl::AllocateRenderQueue() {
void AudioProcessingImpl::EmptyQueuedRenderAudio() {
rtc::CritScope cs_capture(&crit_capture_);
while (aec_render_signal_queue_->Remove(&aec_capture_queue_buffer_)) {
private_submodules_->echo_cancellation->ProcessRenderAudio(
aec_capture_queue_buffer_);
if (private_submodules_->echo_cancellation) {
RTC_DCHECK(aec_render_signal_queue_);
while (aec_render_signal_queue_->Remove(&aec_capture_queue_buffer_)) {
private_submodules_->echo_cancellation->ProcessRenderAudio(
aec_capture_queue_buffer_);
}
}
while (aecm_render_signal_queue_->Remove(&aecm_capture_queue_buffer_)) {
@ -1283,7 +1261,8 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() {
// Ensure that not both the AEC and AECM are active at the same time.
// TODO(peah): Simplify once the public API Enable functions for these
// are moved to APM.
RTC_DCHECK(!(private_submodules_->echo_cancellation->is_enabled() &&
RTC_DCHECK(!((private_submodules_->echo_cancellation &&
private_submodules_->echo_cancellation->is_enabled()) &&
private_submodules_->echo_control_mobile->is_enabled()));
MaybeUpdateHistograms();
@ -1368,7 +1347,8 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() {
// Ensure that the stream delay was set before the call to the
// AEC ProcessCaptureAudio function.
if (private_submodules_->echo_cancellation->is_enabled() &&
if (private_submodules_->echo_cancellation &&
private_submodules_->echo_cancellation->is_enabled() &&
!private_submodules_->echo_controller && !was_stream_delay_set()) {
return AudioProcessing::kStreamParameterNotSetError;
}
@ -1383,7 +1363,7 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() {
private_submodules_->echo_controller->ProcessCapture(
capture_buffer, capture_.echo_path_gain_change);
} else {
} else if (private_submodules_->echo_cancellation) {
RETURN_ON_ERR(private_submodules_->echo_cancellation->ProcessCaptureAudio(
capture_buffer, stream_delay_ms()));
}
@ -1402,7 +1382,8 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() {
}
if (!(private_submodules_->echo_controller ||
private_submodules_->echo_cancellation->is_enabled())) {
(private_submodules_->echo_cancellation &&
private_submodules_->echo_cancellation->is_enabled()))) {
RETURN_ON_ERR(private_submodules_->echo_control_mobile->ProcessCaptureAudio(
capture_buffer, stream_delay_ms()));
}
@ -1426,7 +1407,8 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() {
// TODO(peah): Add reporting from AEC3 whether there is echo.
RETURN_ON_ERR(public_submodules_->gain_control->ProcessCaptureAudio(
capture_buffer,
private_submodules_->echo_cancellation->stream_has_echo()));
private_submodules_->echo_cancellation &&
private_submodules_->echo_cancellation->stream_has_echo()));
if (submodule_states_.CaptureMultiBandProcessingActive() &&
SampleRateSupportsMultiBand(
@ -1756,20 +1738,6 @@ AudioProcessingStats AudioProcessingImpl::GetStatistics(
stats.echo_return_loss_enhancement =
ec_metrics.echo_return_loss_enhancement;
stats.delay_ms = ec_metrics.delay_ms;
} else if (private_submodules_->echo_cancellation->GetMetrics(&metrics) ==
Error::kNoError) {
if (metrics.divergent_filter_fraction != -1.0f) {
stats.divergent_filter_fraction =
absl::optional<double>(metrics.divergent_filter_fraction);
}
if (metrics.echo_return_loss.instant != -100) {
stats.echo_return_loss =
absl::optional<double>(metrics.echo_return_loss.instant);
}
if (metrics.echo_return_loss_enhancement.instant != -100) {
stats.echo_return_loss_enhancement =
absl::optional<double>(metrics.echo_return_loss_enhancement.instant);
}
}
if (config_.residual_echo_detector.enabled) {
RTC_DCHECK(private_submodules_->echo_detector);
@ -1778,18 +1746,6 @@ AudioProcessingStats AudioProcessingImpl::GetStatistics(
stats.residual_echo_likelihood_recent_max =
ed_metrics.echo_likelihood_recent_max;
}
int delay_median, delay_std;
float fraction_poor_delays;
if (private_submodules_->echo_cancellation->GetDelayMetrics(
&delay_median, &delay_std, &fraction_poor_delays) ==
Error::kNoError) {
if (delay_median >= 0) {
stats.delay_median_ms = absl::optional<int32_t>(delay_median);
}
if (delay_std >= 0) {
stats.delay_standard_deviation_ms = absl::optional<int32_t>(delay_std);
}
}
return stats;
}
@ -1826,7 +1782,8 @@ AudioProcessing::Config AudioProcessingImpl::GetConfig() const {
bool AudioProcessingImpl::UpdateActiveSubmoduleStates() {
return submodule_states_.Update(
config_.high_pass_filter.enabled,
private_submodules_->echo_cancellation->is_enabled(),
private_submodules_->echo_cancellation &&
private_submodules_->echo_cancellation->is_enabled(),
private_submodules_->echo_control_mobile->is_enabled(),
config_.residual_echo_detector.enabled,
public_submodules_->noise_suppression->is_enabled(),
@ -1872,20 +1829,82 @@ void AudioProcessingImpl::InitializeEchoController() {
}
capture_nonlocked_.echo_controller_enabled = true;
} else {
private_submodules_->echo_cancellation->Enable(
config_.echo_canceller.enabled && !config_.echo_canceller.mobile_mode);
private_submodules_->echo_control_mobile->Enable(
config_.echo_canceller.enabled && config_.echo_canceller.mobile_mode);
private_submodules_->echo_cancellation->set_suppression_level(
config_.echo_canceller.legacy_moderate_suppression_level
? EchoCancellationImpl::SuppressionLevel::kModerateSuppression
: EchoCancellationImpl::SuppressionLevel::kHighSuppression);
private_submodules_->echo_controller.reset();
capture_nonlocked_.echo_controller_enabled = false;
private_submodules_->echo_cancellation.reset();
aec_render_signal_queue_.reset();
private_submodules_->echo_control_mobile->Enable(false);
return;
}
private_submodules_->echo_controller.reset();
capture_nonlocked_.echo_controller_enabled = false;
if (!config_.echo_canceller.enabled) {
private_submodules_->echo_cancellation.reset();
aec_render_signal_queue_.reset();
private_submodules_->echo_control_mobile->Enable(false);
return;
}
if (config_.echo_canceller.mobile_mode) {
// Create and activate AECM.
// TODO(peah): Add an on-demand creation of AECmobile similar to AEC2.
private_submodules_->echo_control_mobile->Enable(true);
private_submodules_->echo_cancellation.reset();
aec_render_signal_queue_.reset();
return;
}
// Create and activate AEC2.
private_submodules_->echo_control_mobile->Enable(false);
private_submodules_->echo_cancellation.reset(new EchoCancellationImpl());
private_submodules_->echo_cancellation->SetExtraOptions(
capture_nonlocked_.use_aec2_extended_filter,
capture_nonlocked_.use_aec2_delay_agnostic,
capture_nonlocked_.use_aec2_refined_adaptive_filter);
const size_t new_aec_render_queue_element_max_size =
std::max(static_cast<size_t>(1),
kMaxAllowedValuesOfSamplesPerBand *
EchoCancellationImpl::NumCancellersRequired(
num_output_channels(), num_reverse_channels()));
// Reallocate the queues if the queue item sizes are too small to fit the
// data to put in the queues.
if (aec_render_queue_element_max_size_ <
new_aec_render_queue_element_max_size ||
!aec_render_signal_queue_) {
aec_render_queue_element_max_size_ = new_aec_render_queue_element_max_size;
std::vector<float> template_queue_element(
aec_render_queue_element_max_size_);
aec_render_signal_queue_.reset(
new SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>(
kMaxNumFramesToBuffer, template_queue_element,
RenderQueueItemVerifier<float>(
aec_render_queue_element_max_size_)));
aec_render_queue_buffer_.resize(aec_render_queue_element_max_size_);
aec_capture_queue_buffer_.resize(aec_render_queue_element_max_size_);
} else {
aec_render_signal_queue_->Clear();
}
private_submodules_->echo_cancellation->Initialize(
proc_sample_rate_hz(), num_reverse_channels(), num_output_channels(),
num_proc_channels());
private_submodules_->echo_cancellation->Enable(true);
private_submodules_->echo_cancellation->set_suppression_level(
config_.echo_canceller.legacy_moderate_suppression_level
? EchoCancellationImpl::SuppressionLevel::kModerateSuppression
: EchoCancellationImpl::SuppressionLevel::kHighSuppression);
private_submodules_->echo_controller.reset();
capture_nonlocked_.echo_controller_enabled = false;
}
void AudioProcessingImpl::InitializeGainController2() {
@ -1935,7 +1954,8 @@ void AudioProcessingImpl::InitializePreProcessor() {
void AudioProcessingImpl::MaybeUpdateHistograms() {
static const int kMinDiffDelayMs = 60;
if (private_submodules_->echo_cancellation->is_enabled()) {
if (private_submodules_->echo_cancellation &&
private_submodules_->echo_cancellation->is_enabled()) {
// Activate delay_jumps_ counters if we know echo_cancellation is running.
// If a stream has echo we know that the echo_cancellation is in process.
if (capture_.stream_delay_jumps == -1 &&
@ -2009,8 +2029,12 @@ void AudioProcessingImpl::WriteAecDumpConfigMessage(bool forced) {
if (!aec_dump_) {
return;
}
std::string experiments_description =
private_submodules_->echo_cancellation->GetExperimentsDescription();
std::string experiments_description = "";
if (private_submodules_->echo_cancellation) {
experiments_description +=
private_submodules_->echo_cancellation->GetExperimentsDescription();
}
// TODO(peah): Add semicolon-separated concatenations of experiment
// descriptions for other submodules.
if (constants_.agc_clipped_level_min != kClippedLevelMin) {
@ -2027,13 +2051,19 @@ void AudioProcessingImpl::WriteAecDumpConfigMessage(bool forced) {
apm_config.aec_enabled = config_.echo_canceller.enabled;
apm_config.aec_delay_agnostic_enabled =
private_submodules_->echo_cancellation &&
private_submodules_->echo_cancellation->is_delay_agnostic_enabled();
apm_config.aec_drift_compensation_enabled =
private_submodules_->echo_cancellation &&
private_submodules_->echo_cancellation->is_drift_compensation_enabled();
apm_config.aec_extended_filter_enabled =
private_submodules_->echo_cancellation &&
private_submodules_->echo_cancellation->is_extended_filter_enabled();
apm_config.aec_suppression_level = static_cast<int>(
private_submodules_->echo_cancellation->suppression_level());
apm_config.aec_suppression_level =
private_submodules_->echo_cancellation
? static_cast<int>(
private_submodules_->echo_cancellation->suppression_level())
: 0;
apm_config.aecm_enabled =
private_submodules_->echo_control_mobile->is_enabled();
@ -2115,7 +2145,9 @@ void AudioProcessingImpl::RecordAudioProcessingState() {
AecDump::AudioProcessingState audio_proc_state;
audio_proc_state.delay = capture_nonlocked_.stream_delay_ms;
audio_proc_state.drift =
private_submodules_->echo_cancellation->stream_drift_samples();
private_submodules_->echo_cancellation
? private_submodules_->echo_cancellation->stream_drift_samples()
: 0;
audio_proc_state.level = agc1()->stream_analog_level();
audio_proc_state.keypress = capture_.key_pressed;
aec_dump_->AddAudioProcessingState(audio_proc_state);