AEC3: Decrease latency until the delay has been detected

This CL utilizes the existing, but unused, ability to set
different histogram thresholds for early and late delay
estimation. It does so by tuning the parameters for these.

On top of that, some corrections are added to correctly
handle resets and the use of the hysteresis thresholds.

Bug: webrtc:19886,chromium:896334
Change-Id: I950ac107c124541af8f02b4403f477dda71cc1a1
Reviewed-on: https://webrtc-review.googlesource.com/c/106706
Reviewed-by: Sam Zackrisson <saza@webrtc.org>
Reviewed-by: Gustaf Ullberg <gustaf@webrtc.org>
Commit-Queue: Per Åhgren <peah@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#25443}

api/audio/echo_canceller3_config.h

@@ -50,7 +50,7 @@ struct RTC_EXPORT EchoCanceller3Config {
     struct DelaySelectionThresholds {
       int initial;
       int converged;
-    } delay_selection_thresholds = {25, 25};
+    } delay_selection_thresholds = {5, 20};
   } delay;
 
   struct Filter {

modules/audio_processing/aec3/block_processor.cc

@@ -108,7 +108,7 @@ void BlockProcessorImpl::ProcessCapture(
     if (!capture_properly_started_) {
       capture_properly_started_ = true;
       render_buffer_->Reset();
-      delay_controller_->Reset();
+      delay_controller_->Reset(true);
     }
   } else {
     // If no render data has yet arrived, do not process the capture signal.
@@ -123,7 +123,7 @@ void BlockProcessorImpl::ProcessCapture(
       render_properly_started_) {
     echo_path_variability.delay_change =
         EchoPathVariability::DelayAdjustment::kBufferFlush;
-    delay_controller_->Reset();
+    delay_controller_->Reset(true);
     RTC_LOG(LS_WARNING) << "Reset due to render buffer overrun at block  "
                         << capture_call_counter_;
   }
@@ -139,7 +139,7 @@ void BlockProcessorImpl::ProcessCapture(
         estimated_delay_->quality == DelayEstimate::Quality::kRefined) {
       echo_path_variability.delay_change =
           EchoPathVariability::DelayAdjustment::kDelayReset;
-      delay_controller_->Reset();
+      delay_controller_->Reset(true);
       capture_properly_started_ = false;
       render_properly_started_ = false;
 
@@ -151,7 +151,7 @@ void BlockProcessorImpl::ProcessCapture(
     // echo.
     echo_path_variability.delay_change =
         EchoPathVariability::DelayAdjustment::kDelayReset;
-    delay_controller_->Reset();
+    delay_controller_->Reset(true);
     capture_properly_started_ = false;
     render_properly_started_ = false;
     RTC_LOG(LS_WARNING) << "Reset due to render buffer api skew at block "
@@ -184,7 +184,7 @@ void BlockProcessorImpl::ProcessCapture(
       if (estimated_delay_->quality == DelayEstimate::Quality::kRefined) {
         echo_path_variability.delay_change =
             EchoPathVariability::DelayAdjustment::kDelayReset;
-        delay_controller_->Reset();
+        delay_controller_->Reset(true);
         render_buffer_->Reset();
         capture_properly_started_ = false;
         render_properly_started_ = false;

modules/audio_processing/aec3/block_processor2.cc

@@ -115,7 +115,7 @@ void BlockProcessorImpl2::ProcessCapture(
     if (!capture_properly_started_) {
       capture_properly_started_ = true;
       render_buffer_->Reset();
-      delay_controller_->Reset();
+      delay_controller_->Reset(true);
     }
   } else {
     // If no render data has yet arrived, do not process the capture signal.
@@ -130,7 +130,7 @@ void BlockProcessorImpl2::ProcessCapture(
       render_properly_started_) {
     echo_path_variability.delay_change =
         EchoPathVariability::DelayAdjustment::kBufferFlush;
-    delay_controller_->Reset();
+    delay_controller_->Reset(true);
     RTC_LOG(LS_WARNING) << "Reset due to render buffer overrun at block  "
                         << capture_call_counter_;
   }
@@ -143,7 +143,7 @@ void BlockProcessorImpl2::ProcessCapture(
       render_buffer_->PrepareCaptureProcessing();
   // Reset the delay controller at render buffer underrun.
   if (buffer_event == RenderDelayBuffer::BufferingEvent::kRenderUnderrun) {
-    delay_controller_->Reset();
+    delay_controller_->Reset(false);
   }
 
   data_dumper_->DumpWav("aec3_processblock_capture_input2", kBlockSize,

modules/audio_processing/aec3/echo_canceller3.cc

@@ -82,6 +82,10 @@ bool EnableNewRenderBuffering() {
   return !field_trial::IsEnabled("WebRTC-Aec3NewRenderBufferingKillSwitch");
 }
 
+bool UseEarlyDelayDetection() {
+  return !field_trial::IsEnabled("WebRTC-Aec3EarlyDelayDetectionKillSwitch");
+}
+
 // Method for adjusting config parameter dependencies..
 EchoCanceller3Config AdjustConfig(const EchoCanceller3Config& config) {
   EchoCanceller3Config adjusted_cfg = config;
@@ -161,6 +165,10 @@ EchoCanceller3Config AdjustConfig(const EchoCanceller3Config& config) {
     adjusted_cfg.echo_audibility.use_stationarity_properties_at_init = false;
   }
 
+  if (!UseEarlyDelayDetection()) {
+    adjusted_cfg.delay.delay_selection_thresholds = {25, 25};
+  }
+
   return adjusted_cfg;
 }
 

modules/audio_processing/aec3/echo_path_delay_estimator.cc

@@ -49,13 +49,8 @@ EchoPathDelayEstimator::EchoPathDelayEstimator(
 
 EchoPathDelayEstimator::~EchoPathDelayEstimator() = default;
 
-void EchoPathDelayEstimator::Reset(bool soft_reset) {
+void EchoPathDelayEstimator::Reset(bool reset_delay_confidence) {
-  if (!soft_reset) {
+  Reset(true, reset_delay_confidence);
-    matched_filter_lag_aggregator_.Reset();
-  }
-  matched_filter_.Reset();
-  old_aggregated_lag_ = absl::nullopt;
-  consistent_estimate_counter_ = 0;
 }
 
 absl::optional<DelayEstimate> EchoPathDelayEstimator::EstimateDelay(
@@ -102,10 +97,20 @@ absl::optional<DelayEstimate> EchoPathDelayEstimator::EstimateDelay(
   old_aggregated_lag_ = aggregated_matched_filter_lag;
   constexpr size_t kNumBlocksPerSecondBy2 = kNumBlocksPerSecond / 2;
   if (consistent_estimate_counter_ > kNumBlocksPerSecondBy2) {
-    Reset(true);
+    Reset(false, false);
   }
 
   return aggregated_matched_filter_lag;
 }
 
+void EchoPathDelayEstimator::Reset(bool reset_lag_aggregator,
+                                   bool reset_delay_confidence) {
+  if (reset_lag_aggregator) {
+    matched_filter_lag_aggregator_.Reset(reset_delay_confidence);
+  }
+  matched_filter_.Reset();
+  old_aggregated_lag_ = absl::nullopt;
+  consistent_estimate_counter_ = 0;
+}
+
 }  // namespace webrtc

modules/audio_processing/aec3/echo_path_delay_estimator.h

@@ -34,9 +34,9 @@ class EchoPathDelayEstimator {
                          const EchoCanceller3Config& config);
   ~EchoPathDelayEstimator();
 
-  // Resets the estimation. If the soft-reset is specified, only  the matched
+  // Resets the estimation. If the delay confidence is reset, the reset behavior
-  // filters are reset.
+  // is as if the call is restarted.
-  void Reset(bool soft_reset);
+  void Reset(bool reset_delay_confidence);
 
   // Produce a delay estimate if such is avaliable.
   absl::optional<DelayEstimate> EstimateDelay(
@@ -59,6 +59,9 @@ class EchoPathDelayEstimator {
   absl::optional<DelayEstimate> old_aggregated_lag_;
   size_t consistent_estimate_counter_ = 0;
 
+  // Internal reset method with more granularity.
+  void Reset(bool reset_lag_aggregator, bool reset_delay_confidence);
+
   RTC_DISALLOW_COPY_AND_ASSIGN(EchoPathDelayEstimator);
 };
 }  // namespace webrtc

modules/audio_processing/aec3/matched_filter_lag_aggregator.cc

@@ -31,11 +31,13 @@ MatchedFilterLagAggregator::MatchedFilterLagAggregator(
 
 MatchedFilterLagAggregator::~MatchedFilterLagAggregator() = default;
 
-void MatchedFilterLagAggregator::Reset() {
+void MatchedFilterLagAggregator::Reset(bool hard_reset) {
   std::fill(histogram_.begin(), histogram_.end(), 0);
   histogram_data_.fill(0);
   histogram_data_index_ = 0;
+  if (hard_reset) {
     significant_candidate_found_ = false;
+  }
 }
 
 absl::optional<DelayEstimate> MatchedFilterLagAggregator::Aggregate(
@@ -81,9 +83,13 @@ absl::optional<DelayEstimate> MatchedFilterLagAggregator::Aggregate(
     if (histogram_[candidate] > thresholds_.converged ||
         (histogram_[candidate] > thresholds_.initial &&
          !significant_candidate_found_)) {
-      return DelayEstimate(DelayEstimate::Quality::kRefined, candidate);
+      DelayEstimate::Quality quality = significant_candidate_found_
+                                           ? DelayEstimate::Quality::kRefined
+                                           : DelayEstimate::Quality::kCoarse;
+      return DelayEstimate(quality, candidate);
     }
   }
+
   return absl::nullopt;
 }
 

modules/audio_processing/aec3/matched_filter_lag_aggregator.h

@@ -34,7 +34,7 @@ class MatchedFilterLagAggregator {
   ~MatchedFilterLagAggregator();
 
   // Resets the aggregator.
-  void Reset();
+  void Reset(bool hard_reset);
 
   // Aggregates the provided lag estimates.
   absl::optional<DelayEstimate> Aggregate(

modules/audio_processing/aec3/mock/mock_render_delay_controller.h

@@ -25,7 +25,7 @@ class MockRenderDelayController : public RenderDelayController {
   MockRenderDelayController();
   virtual ~MockRenderDelayController();
 
-  MOCK_METHOD0(Reset, void());
+  MOCK_METHOD1(Reset, void(bool reset_delay_statistics));
   MOCK_METHOD0(LogRenderCall, void());
   MOCK_METHOD4(GetDelay,
                absl::optional<DelayEstimate>(

modules/audio_processing/aec3/render_delay_controller.cc

@@ -45,6 +45,10 @@ bool UseOffsetBlocks() {
   return field_trial::IsEnabled("WebRTC-Aec3UseOffsetBlocks");
 }
 
+bool UseEarlyDelayDetection() {
+  return !field_trial::IsEnabled("WebRTC-Aec3EarlyDelayDetectionKillSwitch");
+}
+
 constexpr int kSkewHistorySizeLog2 = 8;
 
 class RenderDelayControllerImpl final : public RenderDelayController {
@@ -53,7 +57,7 @@ class RenderDelayControllerImpl final : public RenderDelayController {
                             int non_causal_offset,
                             int sample_rate_hz);
   ~RenderDelayControllerImpl() override;
-  void Reset() override;
+  void Reset(bool reset_delay_confidence) override;
   void LogRenderCall() override;
   absl::optional<DelayEstimate> GetDelay(
       const DownsampledRenderBuffer& render_buffer,
@@ -64,6 +68,7 @@ class RenderDelayControllerImpl final : public RenderDelayController {
  private:
   static int instance_count_;
   std::unique_ptr<ApmDataDumper> data_dumper_;
+  const bool use_early_delay_detection_;
   const int delay_headroom_blocks_;
   const int hysteresis_limit_1_blocks_;
   const int hysteresis_limit_2_blocks_;
@@ -82,6 +87,7 @@ class RenderDelayControllerImpl final : public RenderDelayController {
   size_t capture_call_counter_ = 0;
   int delay_change_counter_ = 0;
   size_t soft_reset_counter_ = 0;
+  DelayEstimate::Quality last_delay_estimate_quality_;
   RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderDelayControllerImpl);
 };
 
@@ -130,6 +136,7 @@ RenderDelayControllerImpl::RenderDelayControllerImpl(
     int sample_rate_hz)
     : data_dumper_(
           new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
+      use_early_delay_detection_(UseEarlyDelayDetection()),
       delay_headroom_blocks_(
           static_cast<int>(config.delay.delay_headroom_blocks)),
       hysteresis_limit_1_blocks_(
@@ -140,7 +147,8 @@ RenderDelayControllerImpl::RenderDelayControllerImpl(
       use_offset_blocks_(UseOffsetBlocks()),
       delay_estimator_(data_dumper_.get(), config),
       delay_buf_(kBlockSize * non_causal_offset, 0.f),
-      skew_estimator_(kSkewHistorySizeLog2) {
+      skew_estimator_(kSkewHistorySizeLog2),
+      last_delay_estimate_quality_(DelayEstimate::Quality::kCoarse) {
   RTC_DCHECK(ValidFullBandRate(sample_rate_hz));
   delay_estimator_.LogDelayEstimationProperties(sample_rate_hz,
                                                 delay_buf_.size());
@@ -148,16 +156,19 @@ RenderDelayControllerImpl::RenderDelayControllerImpl(
 
 RenderDelayControllerImpl::~RenderDelayControllerImpl() = default;
 
-void RenderDelayControllerImpl::Reset() {
+void RenderDelayControllerImpl::Reset(bool reset_delay_confidence) {
   delay_ = absl::nullopt;
   delay_samples_ = absl::nullopt;
   skew_ = absl::nullopt;
   previous_offset_blocks_ = 0;
   std::fill(delay_buf_.begin(), delay_buf_.end(), 0.f);
-  delay_estimator_.Reset(false);
+  delay_estimator_.Reset(reset_delay_confidence);
   skew_estimator_.Reset();
   delay_change_counter_ = 0;
   soft_reset_counter_ = 0;
+  if (reset_delay_confidence) {
+    last_delay_estimate_quality_ = DelayEstimate::Quality::kCoarse;
+  }
 }
 
 void RenderDelayControllerImpl::LogRenderCall() {
@@ -195,9 +206,6 @@ absl::optional<DelayEstimate> RenderDelayControllerImpl::GetDelay(
   absl::optional<int> skew = skew_estimator_.GetSkewFromCapture();
 
   if (delay_samples) {
-    // TODO(peah): Refactor the rest of the code to assume a kRefined estimate
-    // quality.
-    RTC_DCHECK(DelayEstimate::Quality::kRefined == delay_samples->quality);
     if (!delay_samples_ || delay_samples->delay != delay_samples_->delay) {
       delay_change_counter_ = 0;
     }
@@ -240,7 +248,7 @@ absl::optional<DelayEstimate> RenderDelayControllerImpl::GetDelay(
     } else if (soft_reset_counter_ > 10 * kNumBlocksPerSecond) {
       // Soft reset the delay estimator if there is a significant offset
       // detected.
-      delay_estimator_.Reset(true);
+      delay_estimator_.Reset(false);
       soft_reset_counter_ = 0;
     }
   }
@@ -265,9 +273,14 @@ absl::optional<DelayEstimate> RenderDelayControllerImpl::GetDelay(
 
   if (delay_samples_) {
     // Compute the render delay buffer delay.
-    delay_ = ComputeBufferDelay(
+    const bool use_hysteresis =
-        delay_, delay_headroom_blocks_, hysteresis_limit_1_blocks_,
+        last_delay_estimate_quality_ == DelayEstimate::Quality::kRefined &&
-        hysteresis_limit_2_blocks_, offset_blocks, *delay_samples_);
+        delay_samples_->quality == DelayEstimate::Quality::kRefined;
+    delay_ = ComputeBufferDelay(delay_, delay_headroom_blocks_,
+                                use_hysteresis ? hysteresis_limit_1_blocks_ : 0,
+                                use_hysteresis ? hysteresis_limit_2_blocks_ : 0,
+                                offset_blocks, *delay_samples_);
+    last_delay_estimate_quality_ = delay_samples_->quality;
   }
 
   metrics_.Update(delay_samples_ ? absl::optional<size_t>(delay_samples_->delay)

modules/audio_processing/aec3/render_delay_controller.h

@@ -31,8 +31,9 @@ class RenderDelayController {
                                         int sample_rate_hz);
   virtual ~RenderDelayController() = default;
 
-  // Resets the delay controller.
+  // Resets the delay controller. If the delay confidence is reset, the reset
-  virtual void Reset() = 0;
+  // behavior is as if the call is restarted.
+  virtual void Reset(bool reset_delay_confidence) = 0;
 
   // Logs a render call.
   virtual void LogRenderCall() = 0;

modules/audio_processing/aec3/render_delay_controller2.cc

@@ -24,17 +24,22 @@
 #include "rtc_base/atomicops.h"
 #include "rtc_base/checks.h"
 #include "rtc_base/constructormagic.h"
+#include "system_wrappers/include/field_trial.h"
 
 namespace webrtc {
 
 namespace {
 
+bool UseEarlyDelayDetection() {
+  return !field_trial::IsEnabled("WebRTC-Aec3EarlyDelayDetectionKillSwitch");
+}
+
 class RenderDelayControllerImpl2 final : public RenderDelayController {
  public:
   RenderDelayControllerImpl2(const EchoCanceller3Config& config,
                              int sample_rate_hz);
   ~RenderDelayControllerImpl2() override;
-  void Reset() override;
+  void Reset(bool reset_delay_confidence) override;
   void LogRenderCall() override;
   absl::optional<DelayEstimate> GetDelay(
       const DownsampledRenderBuffer& render_buffer,
@@ -45,6 +50,7 @@ class RenderDelayControllerImpl2 final : public RenderDelayController {
  private:
   static int instance_count_;
   std::unique_ptr<ApmDataDumper> data_dumper_;
+  const bool use_early_delay_detection_;
   const int delay_headroom_blocks_;
   const int hysteresis_limit_1_blocks_;
   const int hysteresis_limit_2_blocks_;
@@ -54,6 +60,7 @@ class RenderDelayControllerImpl2 final : public RenderDelayController {
   absl::optional<DelayEstimate> delay_samples_;
   size_t capture_call_counter_ = 0;
   int delay_change_counter_ = 0;
+  DelayEstimate::Quality last_delay_estimate_quality_;
   RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderDelayControllerImpl2);
 };
 
@@ -100,24 +107,29 @@ RenderDelayControllerImpl2::RenderDelayControllerImpl2(
     int sample_rate_hz)
     : data_dumper_(
           new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
+      use_early_delay_detection_(UseEarlyDelayDetection()),
       delay_headroom_blocks_(
           static_cast<int>(config.delay.delay_headroom_blocks)),
       hysteresis_limit_1_blocks_(
           static_cast<int>(config.delay.hysteresis_limit_1_blocks)),
       hysteresis_limit_2_blocks_(
           static_cast<int>(config.delay.hysteresis_limit_2_blocks)),
-      delay_estimator_(data_dumper_.get(), config) {
+      delay_estimator_(data_dumper_.get(), config),
+      last_delay_estimate_quality_(DelayEstimate::Quality::kCoarse) {
   RTC_DCHECK(ValidFullBandRate(sample_rate_hz));
   delay_estimator_.LogDelayEstimationProperties(sample_rate_hz, 0);
 }
 
 RenderDelayControllerImpl2::~RenderDelayControllerImpl2() = default;
 
-void RenderDelayControllerImpl2::Reset() {
+void RenderDelayControllerImpl2::Reset(bool reset_delay_confidence) {
   delay_ = absl::nullopt;
   delay_samples_ = absl::nullopt;
-  delay_estimator_.Reset(false);
+  delay_estimator_.Reset(reset_delay_confidence);
   delay_change_counter_ = 0;
+  if (reset_delay_confidence || true) {
+    last_delay_estimate_quality_ = DelayEstimate::Quality::kCoarse;
+  }
 }
 
 void RenderDelayControllerImpl2::LogRenderCall() {}
@@ -141,9 +153,6 @@ absl::optional<DelayEstimate> RenderDelayControllerImpl2::GetDelay(
   }
 
   if (delay_samples) {
-    // TODO(peah): Refactor the rest of the code to assume a kRefined estimate
-    // quality.
-    RTC_DCHECK(DelayEstimate::Quality::kRefined == delay_samples->quality);
     if (!delay_samples_ || delay_samples->delay != delay_samples_->delay) {
       delay_change_counter_ = 0;
     }
@@ -171,9 +180,14 @@ absl::optional<DelayEstimate> RenderDelayControllerImpl2::GetDelay(
 
   if (delay_samples_) {
     // Compute the render delay buffer delay.
+    const bool use_hysteresis =
+        last_delay_estimate_quality_ == DelayEstimate::Quality::kRefined &&
+        delay_samples_->quality == DelayEstimate::Quality::kRefined;
     delay_ = ComputeBufferDelay(delay_, delay_headroom_blocks_,
-                                hysteresis_limit_1_blocks_,
+                                use_hysteresis ? hysteresis_limit_1_blocks_ : 0,
-                                hysteresis_limit_2_blocks_, *delay_samples_);
+                                use_hysteresis ? hysteresis_limit_2_blocks_ : 0,
+                                *delay_samples_);
+    last_delay_estimate_quality_ = delay_samples_->quality;
   }
 
   metrics_.Update(delay_samples_ ? absl::optional<size_t>(delay_samples_->delay)

test/fuzzers/audio_processing_configs_fuzzer.cc

@@ -59,6 +59,7 @@ const std::string kFieldTrialNames[] = {
     "WebRTC-Aec3UseStationarityPropertiesKillSwitch",
     "WebRTC-Aec3UtilizeShadowFilterOutputKillSwitch",
     "WebRTC-Aec3ZeroExternalDelayHeadroomKillSwitch",
+    "WebRTC-Aec3EarlyDelayDetectionKillSwitch",
 };
 
 std::unique_ptr<AudioProcessing> CreateApm(test::FuzzDataHelper* fuzz_data,