zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

AEC3: Decrease latency until the delay has been detected

Browse Source 
This CL utilizes the existing, but unused, ability to set
different histogram thresholds for early and late delay
estimation. It does so by tuning the parameters for these.

On top of that, some corrections are added to correctly
handle resets and the use of the hysteresis thresholds.

Bug: webrtc:19886,chromium:896334
Change-Id: I950ac107c124541af8f02b4403f477dda71cc1a1
Reviewed-on: https://webrtc-review.googlesource.com/c/106706
Reviewed-by: Sam Zackrisson <saza@webrtc.org>
Reviewed-by: Gustaf Ullberg <gustaf@webrtc.org>
Commit-Queue: Per Åhgren <peah@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#25443}
This commit is contained in:
Per Åhgren
2018-10-30 23:44:40 +01:00
committed by Commit Bot
parent f577ab3d38
commit 8b7d206d37
13 changed files with 98 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
api/audio/echo_canceller3_config.h
View File

@ -50,7 +50,7 @@ struct RTC_EXPORT EchoCanceller3Config {
struct DelaySelectionThresholds {
int initial;
int converged;
} delay_selection_thresholds = {25, 25};
} delay_selection_thresholds = {5, 20};
} delay;
struct Filter {

10
modules/audio_processing/aec3/block_processor.cc
View File

@ -108,7 +108,7 @@ void BlockProcessorImpl::ProcessCapture(
if (!capture_properly_started_) {
capture_properly_started_ = true;
render_buffer_->Reset();
delay_controller_->Reset();
delay_controller_->Reset(true);
}
} else {
// If no render data has yet arrived, do not process the capture signal.
@ -123,7 +123,7 @@ void BlockProcessorImpl::ProcessCapture(
render_properly_started_) {
echo_path_variability.delay_change =
EchoPathVariability::DelayAdjustment::kBufferFlush;
delay_controller_->Reset();
delay_controller_->Reset(true);
RTC_LOG(LS_WARNING) << "Reset due to render buffer overrun at block "
<< capture_call_counter_;
}
@ -139,7 +139,7 @@ void BlockProcessorImpl::ProcessCapture(
estimated_delay_->quality == DelayEstimate::Quality::kRefined) {
echo_path_variability.delay_change =
EchoPathVariability::DelayAdjustment::kDelayReset;
delay_controller_->Reset();
delay_controller_->Reset(true);
capture_properly_started_ = false;
render_properly_started_ = false;
@ -151,7 +151,7 @@ void BlockProcessorImpl::ProcessCapture(
// echo.
echo_path_variability.delay_change =
EchoPathVariability::DelayAdjustment::kDelayReset;
delay_controller_->Reset();
delay_controller_->Reset(true);
capture_properly_started_ = false;
render_properly_started_ = false;
RTC_LOG(LS_WARNING) << "Reset due to render buffer api skew at block "
@ -184,7 +184,7 @@ void BlockProcessorImpl::ProcessCapture(
if (estimated_delay_->quality == DelayEstimate::Quality::kRefined) {
echo_path_variability.delay_change =
EchoPathVariability::DelayAdjustment::kDelayReset;
delay_controller_->Reset();
delay_controller_->Reset(true);
render_buffer_->Reset();
capture_properly_started_ = false;
render_properly_started_ = false;

6
modules/audio_processing/aec3/block_processor2.cc
View File

@ -115,7 +115,7 @@ void BlockProcessorImpl2::ProcessCapture(
if (!capture_properly_started_) {
capture_properly_started_ = true;
render_buffer_->Reset();
delay_controller_->Reset();
delay_controller_->Reset(true);
}
} else {
// If no render data has yet arrived, do not process the capture signal.
@ -130,7 +130,7 @@ void BlockProcessorImpl2::ProcessCapture(
render_properly_started_) {
echo_path_variability.delay_change =
EchoPathVariability::DelayAdjustment::kBufferFlush;
delay_controller_->Reset();
delay_controller_->Reset(true);
RTC_LOG(LS_WARNING) << "Reset due to render buffer overrun at block "
<< capture_call_counter_;
}
@ -143,7 +143,7 @@ void BlockProcessorImpl2::ProcessCapture(
render_buffer_->PrepareCaptureProcessing();
// Reset the delay controller at render buffer underrun.
if (buffer_event == RenderDelayBuffer::BufferingEvent::kRenderUnderrun) {
delay_controller_->Reset();
delay_controller_->Reset(false);
}
data_dumper_->DumpWav("aec3_processblock_capture_input2", kBlockSize,

8
modules/audio_processing/aec3/echo_canceller3.cc
View File

@ -82,6 +82,10 @@ bool EnableNewRenderBuffering() {
return !field_trial::IsEnabled("WebRTC-Aec3NewRenderBufferingKillSwitch");
}
bool UseEarlyDelayDetection() {
return !field_trial::IsEnabled("WebRTC-Aec3EarlyDelayDetectionKillSwitch");
}
// Method for adjusting config parameter dependencies..
EchoCanceller3Config AdjustConfig(const EchoCanceller3Config& config) {
EchoCanceller3Config adjusted_cfg = config;
@ -161,6 +165,10 @@ EchoCanceller3Config AdjustConfig(const EchoCanceller3Config& config) {
adjusted_cfg.echo_audibility.use_stationarity_properties_at_init = false;
}
if (!UseEarlyDelayDetection()) {
adjusted_cfg.delay.delay_selection_thresholds = {25, 25};
}
return adjusted_cfg;
}

21
modules/audio_processing/aec3/echo_path_delay_estimator.cc
View File

@ -49,13 +49,8 @@ EchoPathDelayEstimator::EchoPathDelayEstimator(
EchoPathDelayEstimator::~EchoPathDelayEstimator() = default;
void EchoPathDelayEstimator::Reset(bool soft_reset) {
if (!soft_reset) {
matched_filter_lag_aggregator_.Reset();
}
matched_filter_.Reset();
old_aggregated_lag_ = absl::nullopt;
consistent_estimate_counter_ = 0;
void EchoPathDelayEstimator::Reset(bool reset_delay_confidence) {
Reset(true, reset_delay_confidence);
}
absl::optional<DelayEstimate> EchoPathDelayEstimator::EstimateDelay(
@ -102,10 +97,20 @@ absl::optional<DelayEstimate> EchoPathDelayEstimator::EstimateDelay(
old_aggregated_lag_ = aggregated_matched_filter_lag;
constexpr size_t kNumBlocksPerSecondBy2 = kNumBlocksPerSecond / 2;
if (consistent_estimate_counter_ > kNumBlocksPerSecondBy2) {
Reset(true);
Reset(false, false);
}
return aggregated_matched_filter_lag;
}
void EchoPathDelayEstimator::Reset(bool reset_lag_aggregator,
bool reset_delay_confidence) {
if (reset_lag_aggregator) {
matched_filter_lag_aggregator_.Reset(reset_delay_confidence);
}
matched_filter_.Reset();
old_aggregated_lag_ = absl::nullopt;
consistent_estimate_counter_ = 0;
}
} // namespace webrtc

9
modules/audio_processing/aec3/echo_path_delay_estimator.h
View File

@ -34,9 +34,9 @@ class EchoPathDelayEstimator {
const EchoCanceller3Config& config);
~EchoPathDelayEstimator();
// Resets the estimation. If the soft-reset is specified, only the matched
// filters are reset.
void Reset(bool soft_reset);
// Resets the estimation. If the delay confidence is reset, the reset behavior
// is as if the call is restarted.
void Reset(bool reset_delay_confidence);
// Produce a delay estimate if such is avaliable.
absl::optional<DelayEstimate> EstimateDelay(
@ -59,6 +59,9 @@ class EchoPathDelayEstimator {
absl::optional<DelayEstimate> old_aggregated_lag_;
size_t consistent_estimate_counter_ = 0;
// Internal reset method with more granularity.
void Reset(bool reset_lag_aggregator, bool reset_delay_confidence);
RTC_DISALLOW_COPY_AND_ASSIGN(EchoPathDelayEstimator);
};
} // namespace webrtc

12
modules/audio_processing/aec3/matched_filter_lag_aggregator.cc
View File

@ -31,11 +31,13 @@ MatchedFilterLagAggregator::MatchedFilterLagAggregator(
MatchedFilterLagAggregator::~MatchedFilterLagAggregator() = default;
void MatchedFilterLagAggregator::Reset() {
void MatchedFilterLagAggregator::Reset(bool hard_reset) {
std::fill(histogram_.begin(), histogram_.end(), 0);
histogram_data_.fill(0);
histogram_data_index_ = 0;
significant_candidate_found_ = false;
if (hard_reset) {
significant_candidate_found_ = false;
}
}
absl::optional<DelayEstimate> MatchedFilterLagAggregator::Aggregate(
@ -81,9 +83,13 @@ absl::optional<DelayEstimate> MatchedFilterLagAggregator::Aggregate(
if (histogram_[candidate] > thresholds_.converged ||
(histogram_[candidate] > thresholds_.initial &&
!significant_candidate_found_)) {
return DelayEstimate(DelayEstimate::Quality::kRefined, candidate);
DelayEstimate::Quality quality = significant_candidate_found_
? DelayEstimate::Quality::kRefined
: DelayEstimate::Quality::kCoarse;
return DelayEstimate(quality, candidate);
}
}
return absl::nullopt;
}

2
modules/audio_processing/aec3/matched_filter_lag_aggregator.h
View File

@ -34,7 +34,7 @@ class MatchedFilterLagAggregator {
~MatchedFilterLagAggregator();
// Resets the aggregator.
void Reset();
void Reset(bool hard_reset);
// Aggregates the provided lag estimates.
absl::optional<DelayEstimate> Aggregate(

2
modules/audio_processing/aec3/mock/mock_render_delay_controller.h
View File

@ -25,7 +25,7 @@ class MockRenderDelayController : public RenderDelayController {
MockRenderDelayController();
virtual ~MockRenderDelayController();
MOCK_METHOD0(Reset, void());
MOCK_METHOD1(Reset, void(bool reset_delay_statistics));
MOCK_METHOD0(LogRenderCall, void());
MOCK_METHOD4(GetDelay,
absl::optional<DelayEstimate>(

35
modules/audio_processing/aec3/render_delay_controller.cc
View File

@ -45,6 +45,10 @@ bool UseOffsetBlocks() {
return field_trial::IsEnabled("WebRTC-Aec3UseOffsetBlocks");
}
bool UseEarlyDelayDetection() {
return !field_trial::IsEnabled("WebRTC-Aec3EarlyDelayDetectionKillSwitch");
}
constexpr int kSkewHistorySizeLog2 = 8;
class RenderDelayControllerImpl final : public RenderDelayController {
@ -53,7 +57,7 @@ class RenderDelayControllerImpl final : public RenderDelayController {
int non_causal_offset,
int sample_rate_hz);
~RenderDelayControllerImpl() override;
void Reset() override;
void Reset(bool reset_delay_confidence) override;
void LogRenderCall() override;
absl::optional<DelayEstimate> GetDelay(
const DownsampledRenderBuffer& render_buffer,
@ -64,6 +68,7 @@ class RenderDelayControllerImpl final : public RenderDelayController {
private:
static int instance_count_;
std::unique_ptr<ApmDataDumper> data_dumper_;
const bool use_early_delay_detection_;
const int delay_headroom_blocks_;
const int hysteresis_limit_1_blocks_;
const int hysteresis_limit_2_blocks_;
@ -82,6 +87,7 @@ class RenderDelayControllerImpl final : public RenderDelayController {
size_t capture_call_counter_ = 0;
int delay_change_counter_ = 0;
size_t soft_reset_counter_ = 0;
DelayEstimate::Quality last_delay_estimate_quality_;
RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderDelayControllerImpl);
};
@ -130,6 +136,7 @@ RenderDelayControllerImpl::RenderDelayControllerImpl(
int sample_rate_hz)
: data_dumper_(
new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
use_early_delay_detection_(UseEarlyDelayDetection()),
delay_headroom_blocks_(
static_cast<int>(config.delay.delay_headroom_blocks)),
hysteresis_limit_1_blocks_(
@ -140,7 +147,8 @@ RenderDelayControllerImpl::RenderDelayControllerImpl(
use_offset_blocks_(UseOffsetBlocks()),
delay_estimator_(data_dumper_.get(), config),
delay_buf_(kBlockSize * non_causal_offset, 0.f),
skew_estimator_(kSkewHistorySizeLog2) {
skew_estimator_(kSkewHistorySizeLog2),
last_delay_estimate_quality_(DelayEstimate::Quality::kCoarse) {
RTC_DCHECK(ValidFullBandRate(sample_rate_hz));
delay_estimator_.LogDelayEstimationProperties(sample_rate_hz,
delay_buf_.size());
@ -148,16 +156,19 @@ RenderDelayControllerImpl::RenderDelayControllerImpl(
RenderDelayControllerImpl::~RenderDelayControllerImpl() = default;
void RenderDelayControllerImpl::Reset() {
void RenderDelayControllerImpl::Reset(bool reset_delay_confidence) {
delay_ = absl::nullopt;
delay_samples_ = absl::nullopt;
skew_ = absl::nullopt;
previous_offset_blocks_ = 0;
std::fill(delay_buf_.begin(), delay_buf_.end(), 0.f);
delay_estimator_.Reset(false);
delay_estimator_.Reset(reset_delay_confidence);
skew_estimator_.Reset();
delay_change_counter_ = 0;
soft_reset_counter_ = 0;
if (reset_delay_confidence) {
last_delay_estimate_quality_ = DelayEstimate::Quality::kCoarse;
}
}
void RenderDelayControllerImpl::LogRenderCall() {
@ -195,9 +206,6 @@ absl::optional<DelayEstimate> RenderDelayControllerImpl::GetDelay(
absl::optional<int> skew = skew_estimator_.GetSkewFromCapture();
if (delay_samples) {
// TODO(peah): Refactor the rest of the code to assume a kRefined estimate
// quality.
RTC_DCHECK(DelayEstimate::Quality::kRefined == delay_samples->quality);
if (!delay_samples_ || delay_samples->delay != delay_samples_->delay) {
delay_change_counter_ = 0;
}
@ -240,7 +248,7 @@ absl::optional<DelayEstimate> RenderDelayControllerImpl::GetDelay(
} else if (soft_reset_counter_ > 10 * kNumBlocksPerSecond) {
// Soft reset the delay estimator if there is a significant offset
// detected.
delay_estimator_.Reset(true);
delay_estimator_.Reset(false);
soft_reset_counter_ = 0;
}
}
@ -265,9 +273,14 @@ absl::optional<DelayEstimate> RenderDelayControllerImpl::GetDelay(
if (delay_samples_) {
// Compute the render delay buffer delay.
delay_ = ComputeBufferDelay(
delay_, delay_headroom_blocks_, hysteresis_limit_1_blocks_,
hysteresis_limit_2_blocks_, offset_blocks, *delay_samples_);
const bool use_hysteresis =
last_delay_estimate_quality_ == DelayEstimate::Quality::kRefined &&
delay_samples_->quality == DelayEstimate::Quality::kRefined;
delay_ = ComputeBufferDelay(delay_, delay_headroom_blocks_,
use_hysteresis ? hysteresis_limit_1_blocks_ : 0,
use_hysteresis ? hysteresis_limit_2_blocks_ : 0,
offset_blocks, *delay_samples_);
last_delay_estimate_quality_ = delay_samples_->quality;
}
metrics_.Update(delay_samples_ ? absl::optional<size_t>(delay_samples_->delay)

5
modules/audio_processing/aec3/render_delay_controller.h
View File

@ -31,8 +31,9 @@ class RenderDelayController {
int sample_rate_hz);
virtual ~RenderDelayController() = default;
// Resets the delay controller.
virtual void Reset() = 0;
// Resets the delay controller. If the delay confidence is reset, the reset
// behavior is as if the call is restarted.
virtual void Reset(bool reset_delay_confidence) = 0;
// Logs a render call.
virtual void LogRenderCall() = 0;

32
modules/audio_processing/aec3/render_delay_controller2.cc
View File

@ -24,17 +24,22 @@
#include "rtc_base/atomicops.h"
#include "rtc_base/checks.h"
#include "rtc_base/constructormagic.h"
#include "system_wrappers/include/field_trial.h"
namespace webrtc {
namespace {
bool UseEarlyDelayDetection() {
return !field_trial::IsEnabled("WebRTC-Aec3EarlyDelayDetectionKillSwitch");
}
class RenderDelayControllerImpl2 final : public RenderDelayController {
public:
RenderDelayControllerImpl2(const EchoCanceller3Config& config,
int sample_rate_hz);
~RenderDelayControllerImpl2() override;
void Reset() override;
void Reset(bool reset_delay_confidence) override;
void LogRenderCall() override;
absl::optional<DelayEstimate> GetDelay(
const DownsampledRenderBuffer& render_buffer,
@ -45,6 +50,7 @@ class RenderDelayControllerImpl2 final : public RenderDelayController {
private:
static int instance_count_;
std::unique_ptr<ApmDataDumper> data_dumper_;
const bool use_early_delay_detection_;
const int delay_headroom_blocks_;
const int hysteresis_limit_1_blocks_;
const int hysteresis_limit_2_blocks_;
@ -54,6 +60,7 @@ class RenderDelayControllerImpl2 final : public RenderDelayController {
absl::optional<DelayEstimate> delay_samples_;
size_t capture_call_counter_ = 0;
int delay_change_counter_ = 0;
DelayEstimate::Quality last_delay_estimate_quality_;
RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderDelayControllerImpl2);
};
@ -100,24 +107,29 @@ RenderDelayControllerImpl2::RenderDelayControllerImpl2(
int sample_rate_hz)
: data_dumper_(
new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
use_early_delay_detection_(UseEarlyDelayDetection()),
delay_headroom_blocks_(
static_cast<int>(config.delay.delay_headroom_blocks)),
hysteresis_limit_1_blocks_(
static_cast<int>(config.delay.hysteresis_limit_1_blocks)),
hysteresis_limit_2_blocks_(
static_cast<int>(config.delay.hysteresis_limit_2_blocks)),
delay_estimator_(data_dumper_.get(), config) {
delay_estimator_(data_dumper_.get(), config),
last_delay_estimate_quality_(DelayEstimate::Quality::kCoarse) {
RTC_DCHECK(ValidFullBandRate(sample_rate_hz));
delay_estimator_.LogDelayEstimationProperties(sample_rate_hz, 0);
}
RenderDelayControllerImpl2::~RenderDelayControllerImpl2() = default;
void RenderDelayControllerImpl2::Reset() {
void RenderDelayControllerImpl2::Reset(bool reset_delay_confidence) {
delay_ = absl::nullopt;
delay_samples_ = absl::nullopt;
delay_estimator_.Reset(false);
delay_estimator_.Reset(reset_delay_confidence);
delay_change_counter_ = 0;
if (reset_delay_confidence || true) {
last_delay_estimate_quality_ = DelayEstimate::Quality::kCoarse;
}
}
void RenderDelayControllerImpl2::LogRenderCall() {}
@ -141,9 +153,6 @@ absl::optional<DelayEstimate> RenderDelayControllerImpl2::GetDelay(
}
if (delay_samples) {
// TODO(peah): Refactor the rest of the code to assume a kRefined estimate
// quality.
RTC_DCHECK(DelayEstimate::Quality::kRefined == delay_samples->quality);
if (!delay_samples_ || delay_samples->delay != delay_samples_->delay) {
delay_change_counter_ = 0;
}
@ -171,9 +180,14 @@ absl::optional<DelayEstimate> RenderDelayControllerImpl2::GetDelay(
if (delay_samples_) {
// Compute the render delay buffer delay.
const bool use_hysteresis =
last_delay_estimate_quality_ == DelayEstimate::Quality::kRefined &&
delay_samples_->quality == DelayEstimate::Quality::kRefined;
delay_ = ComputeBufferDelay(delay_, delay_headroom_blocks_,
hysteresis_limit_1_blocks_,
hysteresis_limit_2_blocks_, *delay_samples_);
use_hysteresis ? hysteresis_limit_1_blocks_ : 0,
use_hysteresis ? hysteresis_limit_2_blocks_ : 0,
*delay_samples_);
last_delay_estimate_quality_ = delay_samples_->quality;
}
metrics_.Update(delay_samples_ ? absl::optional<size_t>(delay_samples_->delay)

1
test/fuzzers/audio_processing_configs_fuzzer.cc
View File

@ -59,6 +59,7 @@ const std::string kFieldTrialNames[] = {
"WebRTC-Aec3UseStationarityPropertiesKillSwitch",
"WebRTC-Aec3UtilizeShadowFilterOutputKillSwitch",
"WebRTC-Aec3ZeroExternalDelayHeadroomKillSwitch",
"WebRTC-Aec3EarlyDelayDetectionKillSwitch",
};
std::unique_ptr<AudioProcessing> CreateApm(test::FuzzDataHelper* fuzz_data,