zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

AEC3: changes the signal used for deciding when to update the erle so the reverb render signal is now used

Browse Source 
In this CL we change the signal that controls the updates of the ERLE estimator. Until now, the render signal was used which is not optimum for reverberant signals. In this CL, a reverberation has been added to the the render signal and this new signal has been used for controlling when to update the ERLE estimator.

Bug: webrtc:9873
Change-Id: I0ebea3fc208f97aa237af015ba543015d49ed978
Reviewed-on: https://webrtc-review.googlesource.com/c/105660
Reviewed-by: Gustaf Ullberg <gustaf@webrtc.org>
Commit-Queue: Jesus de Vicente Pena <devicentepena@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#25285}
This commit is contained in:
Jesús de Vicente Peña
2018-10-22 11:41:05 +02:00
committed by Commit Bot
parent ecdd432a32
commit c98849cf92
13 changed files with 160 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
modules/audio_processing/aec3/BUILD.gn
View File

@ -83,6 +83,8 @@ rtc_static_library("aec3") {
"render_delay_controller2.cc",
"render_delay_controller_metrics.cc",
"render_delay_controller_metrics.h",
"render_reverb_model.cc",
"render_reverb_model.h",
"render_signal_analyzer.cc",
"render_signal_analyzer.h",
"residual_echo_estimator.cc",

26
modules/audio_processing/aec3/aec_state.cc
View File

@ -41,6 +41,10 @@ bool EnableLegacySaturationBehavior() {
bool UseSuppressionGainLimiter() {
return field_trial::IsEnabled("WebRTC-Aec3GainLimiterDeactivationKillSwitch");
}
bool EnableErleUpdatesDuringReverb() {
return !field_trial::IsEnabled(
"WebRTC-Aec3EnableErleUpdatesDuringReverbKillSwitch");
}
constexpr size_t kBlocksSinceConvergencedFilterInit = 10000;
constexpr size_t kBlocksSinceConsistentEstimateInit = 10000;
@ -77,6 +81,7 @@ AecState::AecState(const EchoCanceller3Config& config)
config_(config),
use_legacy_saturation_behavior_(EnableLegacySaturationBehavior()),
enable_erle_resets_at_gain_changes_(EnableErleResetsAtGainChanges()),
enable_erle_updates_during_reverb_(EnableErleUpdatesDuringReverb()),
use_legacy_filter_quality_(UseLegacyFilterQualityState()),
use_suppressor_gain_limiter_(UseSuppressionGainLimiter()),
initial_state_(config_),
@ -182,22 +187,33 @@ void AecState::Update(
}
}
std::array<float, kFftLengthBy2Plus1> X2_reverb;
render_reverb_.Apply(
render_buffer.GetSpectrumBuffer(), delay_state_.DirectPathFilterDelay(),
config_.ep_strength.reverb_based_on_render ? ReverbDecay() : 0.f,
X2_reverb);
if (config_.echo_audibility.use_stationary_properties) {
// Update the echo audibility evaluator.
echo_audibility_.Update(
render_buffer, delay_state_.DirectPathFilterDelay(),
delay_state_.ExternalDelayReported(),
config_.ep_strength.reverb_based_on_render ? ReverbDecay() : 0.f);
echo_audibility_.Update(render_buffer,
render_reverb_.GetReverbContributionPowerSpectrum(),
delay_state_.DirectPathFilterDelay(),
delay_state_.ExternalDelayReported());
}
// Update the ERL and ERLE measures.
if (initial_state_.TransitionTriggered()) {
erle_estimator_.Reset(false);
}
const auto& X2 = render_buffer.Spectrum(delay_state_.DirectPathFilterDelay());
erle_estimator_.Update(X2, Y2, E2_main,
const auto& X2_input_erle =
enable_erle_updates_during_reverb_ ? X2_reverb : X2;
erle_estimator_.Update(X2_input_erle, Y2, E2_main,
subtractor_output_analyzer_.ConvergedFilter(),
config_.erle.onset_detection);
erl_estimator_.Update(subtractor_output_analyzer_.ConvergedFilter(), X2, Y2);
// Detect and flag echo saturation.

3
modules/audio_processing/aec3/aec_state.h
View File

@ -28,6 +28,7 @@
#include "modules/audio_processing/aec3/erle_estimator.h"
#include "modules/audio_processing/aec3/filter_analyzer.h"
#include "modules/audio_processing/aec3/render_buffer.h"
#include "modules/audio_processing/aec3/render_reverb_model.h"
#include "modules/audio_processing/aec3/reverb_model_estimator.h"
#include "modules/audio_processing/aec3/subtractor_output.h"
#include "modules/audio_processing/aec3/subtractor_output_analyzer.h"
@ -172,6 +173,7 @@ class AecState {
const EchoCanceller3Config config_;
const bool use_legacy_saturation_behavior_;
const bool enable_erle_resets_at_gain_changes_;
const bool enable_erle_updates_during_reverb_;
const bool use_legacy_filter_quality_;
const bool use_suppressor_gain_limiter_;
@ -383,6 +385,7 @@ class AecState {
absl::optional<DelayEstimate> external_delay_;
EchoAudibility echo_audibility_;
ReverbModelEstimator reverb_model_estimator_;
RenderReverbModel render_reverb_;
SubtractorOutputAnalyzer subtractor_output_analyzer_;
};

22
modules/audio_processing/aec3/echo_audibility.cc
View File

@ -13,6 +13,7 @@
#include <algorithm>
#include <cmath>
#include "api/array_view.h"
#include "modules/audio_processing/aec3/aec3_common.h"
#include "modules/audio_processing/aec3/matrix_buffer.h"
#include "modules/audio_processing/aec3/stationarity_estimator.h"
@ -27,16 +28,18 @@ EchoAudibility::EchoAudibility(bool use_render_stationarity_at_init)
EchoAudibility::~EchoAudibility() = default;
void EchoAudibility::Update(const RenderBuffer& render_buffer,
int delay_blocks,
bool external_delay_seen,
float reverb_decay) {
void EchoAudibility::Update(
const RenderBuffer& render_buffer,
rtc::ArrayView<const float> render_reverb_contribution_spectrum,
int delay_blocks,
bool external_delay_seen) {
UpdateRenderNoiseEstimator(render_buffer.GetSpectrumBuffer(),
render_buffer.GetBlockBuffer(),
external_delay_seen);
if (external_delay_seen || use_render_stationarity_at_init_) {
UpdateRenderStationarityFlags(render_buffer, delay_blocks, reverb_decay);
UpdateRenderStationarityFlags(
render_buffer, render_reverb_contribution_spectrum, delay_blocks);
}
}
@ -48,8 +51,8 @@ void EchoAudibility::Reset() {
void EchoAudibility::UpdateRenderStationarityFlags(
const RenderBuffer& render_buffer,
int delay_blocks,
float reverb_decay) {
rtc::ArrayView<const float> render_reverb_contribution_spectrum,
int delay_blocks) {
const VectorBuffer& spectrum_buffer = render_buffer.GetSpectrumBuffer();
int idx_at_delay =
spectrum_buffer.OffsetIndex(spectrum_buffer.read, delay_blocks);
@ -57,8 +60,9 @@ void EchoAudibility::UpdateRenderStationarityFlags(
int num_lookahead = render_buffer.Headroom() - delay_blocks + 1;
num_lookahead = std::max(0, num_lookahead);
render_stationarity_.UpdateStationarityFlags(spectrum_buffer, idx_at_delay,
num_lookahead, reverb_decay);
render_stationarity_.UpdateStationarityFlags(
spectrum_buffer, render_reverb_contribution_spectrum, idx_at_delay,
num_lookahead);
}
void EchoAudibility::UpdateRenderNoiseEstimator(

11
modules/audio_processing/aec3/echo_audibility.h
View File

@ -36,9 +36,9 @@ class EchoAudibility {
// Feed new render data to the echo audibility estimator.
void Update(const RenderBuffer& render_buffer,
rtc::ArrayView<const float> render_reverb_contribution_spectrum,
int delay_blocks,
bool external_delay_seen,
float reverb_decay);
bool external_delay_seen);
// Get the residual echo scaling.
void GetResidualEchoScaling(bool filter_has_had_time_to_converge,
rtc::ArrayView<float> residual_scaling) const {
@ -62,9 +62,10 @@ class EchoAudibility {
void Reset();
// Updates the render stationarity flags for the current frame.
void UpdateRenderStationarityFlags(const RenderBuffer& render_buffer,
int delay_blocks,
float reverb_decay);
void UpdateRenderStationarityFlags(
const RenderBuffer& render_buffer,
rtc::ArrayView<const float> render_reverb_contribution_spectrum,
int delay_blocks);
// Updates the noise estimator with the new render data since the previous
// call to this method.

11
modules/audio_processing/aec3/erle_estimator.cc
View File

@ -36,15 +36,15 @@ void ErleEstimator::Reset(bool delay_change) {
}
}
void ErleEstimator::Update(rtc::ArrayView<const float> render_spectrum,
void ErleEstimator::Update(rtc::ArrayView<const float> reverb_render_spectrum,
rtc::ArrayView<const float> capture_spectrum,
rtc::ArrayView<const float> subtractor_spectrum,
bool converged_filter,
bool onset_detection) {
RTC_DCHECK_EQ(kFftLengthBy2Plus1, render_spectrum.size());
RTC_DCHECK_EQ(kFftLengthBy2Plus1, reverb_render_spectrum.size());
RTC_DCHECK_EQ(kFftLengthBy2Plus1, capture_spectrum.size());
RTC_DCHECK_EQ(kFftLengthBy2Plus1, subtractor_spectrum.size());
const auto& X2 = render_spectrum;
const auto& X2_reverb = reverb_render_spectrum;
const auto& Y2 = capture_spectrum;
const auto& E2 = subtractor_spectrum;
@ -52,8 +52,9 @@ void ErleEstimator::Update(rtc::ArrayView<const float> render_spectrum,
return;
}
subband_erle_estimator_.Update(X2, Y2, E2, converged_filter, onset_detection);
fullband_erle_estimator_.Update(X2, Y2, E2, converged_filter);
subband_erle_estimator_.Update(X2_reverb, Y2, E2, converged_filter,
onset_detection);
fullband_erle_estimator_.Update(X2_reverb, Y2, E2, converged_filter);
}
void ErleEstimator::Dump(

2
modules/audio_processing/aec3/erle_estimator.h
View File

@ -37,7 +37,7 @@ class ErleEstimator {
void Reset(bool delay_change);
// Updates the ERLE estimates.
void Update(rtc::ArrayView<const float> render_spectrum,
void Update(rtc::ArrayView<const float> reverb_render_spectrum,
rtc::ArrayView<const float> capture_spectrum,
rtc::ArrayView<const float> subtractor_spectrum,
bool converged_filter,

43
modules/audio_processing/aec3/render_reverb_model.cc Normal file
View File

@ -0,0 +1,43 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <algorithm>
#include "api/array_view.h"
#include "modules/audio_processing/aec3/render_reverb_model.h"
#include "rtc_base/checks.h"
namespace webrtc {
RenderReverbModel::RenderReverbModel() {
Reset();
}
RenderReverbModel::~RenderReverbModel() = default;
void RenderReverbModel::Reset() {
render_reverb_.Reset();
}
void RenderReverbModel::Apply(const VectorBuffer& spectrum_buffer,
int delay_blocks,
float reverb_decay,
rtc::ArrayView<float> reverb_power_spectrum) {
int idx_at_delay =
spectrum_buffer.OffsetIndex(spectrum_buffer.read, delay_blocks);
int idx_past = spectrum_buffer.IncIndex(idx_at_delay);
const auto& X2 = spectrum_buffer.buffer[idx_at_delay];
RTC_DCHECK_EQ(X2.size(), reverb_power_spectrum.size());
std::copy(X2.begin(), X2.end(), reverb_power_spectrum.begin());
render_reverb_.AddReverbNoFreqShaping(spectrum_buffer.buffer[idx_past], 1.0f,
reverb_decay, reverb_power_spectrum);
}
} // namespace webrtc

49
modules/audio_processing/aec3/render_reverb_model.h Normal file
View File

@ -0,0 +1,49 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_AUDIO_PROCESSING_AEC3_RENDER_REVERB_MODEL_H_
#define MODULES_AUDIO_PROCESSING_AEC3_RENDER_REVERB_MODEL_H_
#include "api/array_view.h"
#include "modules/audio_processing/aec3/reverb_model.h"
#include "modules/audio_processing/aec3/vector_buffer.h"
namespace webrtc {
// The RenderReverbModel class applies an exponential reverberant model over the
// render spectrum.
class RenderReverbModel {
public:
RenderReverbModel();
~RenderReverbModel();
// Resets the state.
void Reset();
// Applies the reverberation model over the render spectrum. It also returns
// the reverberation render power spectrum in the array reverb_power_spectrum.
void Apply(const VectorBuffer& spectrum_buffer,
int delay_blocks,
float reverb_decay,
rtc::ArrayView<float> reverb_power_spectrum);
// Gets the reverberation spectrum that was added to the render spectrum for
// computing the reverberation render spectrum.
rtc::ArrayView<const float> GetReverbContributionPowerSpectrum() const {
return render_reverb_.GetPowerSpectrum();
}
private:
ReverbModel render_reverb_;
};
} // namespace webrtc.
#endif // MODULES_AUDIO_PROCESSING_AEC3_RENDER_REVERB_MODEL_H_

3
modules/audio_processing/aec3/residual_echo_estimator.h
View File

@ -39,7 +39,7 @@ class ResidualEchoEstimator {
std::array<float, kFftLengthBy2Plus1>* R2);
// Returns the reverberant power spectrum contributions to the echo residual.
const std::array<float, kFftLengthBy2Plus1>& GetReverbPowerSpectrum() const {
rtc::ArrayView<const float> GetReverbPowerSpectrum() const {
if (echo_reverb_) {
return echo_reverb_->GetPowerSpectrum();
} else {
@ -66,7 +66,6 @@ class ResidualEchoEstimator {
const std::array<float, kFftLengthBy2Plus1>& Y2,
std::array<float, kFftLengthBy2Plus1>* R2);
// Estimates the echo generating signal power as gated maximal power over a
// time window.
void EchoGeneratingPower(const VectorBuffer& spectrum_buffer,

4
modules/audio_processing/aec3/reverb_model.h
View File

@ -50,9 +50,7 @@ class ReverbModel {
float reverb_decay);
// Returns the current power spectrum reverberation contributions.
const std::array<float, kFftLengthBy2Plus1>& GetPowerSpectrum() const {
return reverb_;
}
rtc::ArrayView<const float> GetPowerSpectrum() const { return reverb_; }
private:
// Updates the reverberation contributions.

13
modules/audio_processing/aec3/stationarity_estimator.cc
View File

@ -14,6 +14,7 @@
#include <array>
#include <vector>
#include "api/array_view.h"
#include "modules/audio_processing/aec3/aec3_common.h"
#include "modules/audio_processing/aec3/vector_buffer.h"
#include "modules/audio_processing/logging/apm_data_dumper.h"
@ -40,7 +41,6 @@ void StationarityEstimator::Reset() {
noise_.Reset();
hangovers_.fill(0);
stationarity_flags_.fill(false);
render_reverb_.Reset();
}
// Update just the noise estimator. Usefull until the delay is known
@ -54,9 +54,9 @@ void StationarityEstimator::UpdateNoiseEstimator(
void StationarityEstimator::UpdateStationarityFlags(
const VectorBuffer& spectrum_buffer,
rtc::ArrayView<const float> render_reverb_contribution_spectrum,
int idx_current,
int num_lookahead,
float reverb_decay) {
int num_lookahead) {
std::array<int, kWindowLength> indexes;
int num_lookahead_bounded = std::min(num_lookahead, kWindowLength - 1);
int idx = idx_current;
@ -79,12 +79,9 @@ void StationarityEstimator::UpdateStationarityFlags(
spectrum_buffer.DecIndex(indexes[kWindowLength - 1]),
spectrum_buffer.OffsetIndex(idx_current, -(num_lookahead_bounded + 1)));
int idx_past = spectrum_buffer.IncIndex(idx_current);
render_reverb_.UpdateReverbContributionsNoFreqShaping(
spectrum_buffer.buffer[idx_past], 1.0f, reverb_decay);
for (size_t k = 0; k < stationarity_flags_.size(); ++k) {
stationarity_flags_[k] = EstimateBandStationarity(
spectrum_buffer, render_reverb_.GetPowerSpectrum(), indexes, k);
spectrum_buffer, render_reverb_contribution_spectrum, indexes, k);
}
UpdateHangover();
SmoothStationaryPerFreq();
@ -102,7 +99,7 @@ bool StationarityEstimator::IsBlockStationary() const {
bool StationarityEstimator::EstimateBandStationarity(
const VectorBuffer& spectrum_buffer,
const std::array<float, kFftLengthBy2Plus1>& reverb,
rtc::ArrayView<const float> reverb,
const std::array<int, kWindowLength>& indexes,
size_t band) const {
constexpr float kThrStationarity = 10.f;

19
modules/audio_processing/aec3/stationarity_estimator.h
View File

@ -37,10 +37,11 @@ class StationarityEstimator {
// Update the flag indicating whether this current frame is stationary. For
// getting a more robust estimation, it looks at future and/or past frames.
void UpdateStationarityFlags(const VectorBuffer& spectrum_buffer,
int idx_current,
int num_lookahead,
float reverb_decay);
void UpdateStationarityFlags(
const VectorBuffer& spectrum_buffer,
rtc::ArrayView<const float> render_reverb_contribution_spectrum,
int idx_current,
int num_lookahead);
// Returns true if the current band is stationary.
bool IsBandStationary(size_t band) const {
@ -57,11 +58,10 @@ class StationarityEstimator {
// Get an estimation of the stationarity for the current band by looking
// at the past/present/future available data.
bool EstimateBandStationarity(
const VectorBuffer& spectrum_buffer,
const std::array<float, kFftLengthBy2Plus1>& reverb,
const std::array<int, kWindowLength>& indexes,
size_t band) const;
bool EstimateBandStationarity(const VectorBuffer& spectrum_buffer,
rtc::ArrayView<const float> reverb,
const std::array<int, kWindowLength>& indexes,
size_t band) const;
// True if all bands at the current point are stationary.
bool AreAllBandsStationary();
@ -111,7 +111,6 @@ class StationarityEstimator {
NoiseSpectrum noise_;
std::array<int, kFftLengthBy2Plus1> hangovers_;
std::array<bool, kFftLengthBy2Plus1> stationarity_flags_;
ReverbModel render_reverb_;
};
} // namespace webrtc