Leveraging the skew in API call order to a boost AEC3 signal realignment

This CL resets the AEC3 realignment functionality when a significant
and persistent skew in the number of render and capture API calls is
detected.

Bug: chromium:811658,webrtc:8879
Change-Id: Ib5c727b38f427da2a7d25eac7c939a17bdaabe74
Reviewed-on: https://webrtc-review.googlesource.com/52260
Reviewed-by: Gustaf Ullberg <gustaf@webrtc.org>
Commit-Queue: Gustaf Ullberg <gustaf@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#21997}

modules/audio_processing/aec3/block_processor.cc

@@ -101,6 +101,7 @@ void BlockProcessorImpl::ProcessCapture(
     if (!capture_properly_started_) {
       capture_properly_started_ = true;
       render_buffer_->Reset();
+      delay_controller_->Reset();
     }
   } else {
     // If no render data has yet arrived, do not process the capture signal.
@@ -213,6 +214,7 @@ void BlockProcessorImpl::BufferRender(
                         RenderDelayBuffer::BufferingEvent::kNone);
 
   render_properly_started_ = true;
+  delay_controller_->LogRenderCall();
 }
 
 void BlockProcessorImpl::UpdateEchoLeakageStatus(bool leakage_detected) {

modules/audio_processing/aec3/echo_path_delay_estimator.cc

@@ -43,8 +43,10 @@ EchoPathDelayEstimator::EchoPathDelayEstimator(
 
 EchoPathDelayEstimator::~EchoPathDelayEstimator() = default;
 
-void EchoPathDelayEstimator::Reset() {
-  matched_filter_lag_aggregator_.Reset();
+void EchoPathDelayEstimator::Reset(bool soft_reset) {
+  if (!soft_reset) {
+    matched_filter_lag_aggregator_.Reset();
+  }
   matched_filter_.Reset();
 }
 

modules/audio_processing/aec3/echo_path_delay_estimator.h

@@ -33,8 +33,9 @@ class EchoPathDelayEstimator {
                          const EchoCanceller3Config& config);
   ~EchoPathDelayEstimator();
 
-  // Resets the estimation.
-  void Reset();
+  // Resets the estimation. If the soft-reset is specified, only  the matched
+  // filters are reset.
+  void Reset(bool soft_reset);
 
   // Produce a delay estimate if such is avaliable.
   rtc::Optional<DelayEstimate> EstimateDelay(

modules/audio_processing/aec3/mock/mock_render_delay_controller.h

@@ -25,6 +25,7 @@ class MockRenderDelayController : public RenderDelayController {
   virtual ~MockRenderDelayController() = default;
 
   MOCK_METHOD0(Reset, void());
+  MOCK_METHOD0(LogRenderCall, void());
   MOCK_METHOD2(
       GetDelay,
       rtc::Optional<DelayEstimate>(const DownsampledRenderBuffer& render_buffer,

modules/audio_processing/aec3/render_delay_controller.cc

@@ -11,6 +11,7 @@
 
 #include <algorithm>
 #include <memory>
+#include <numeric>
 #include <string>
 #include <vector>
 
@@ -25,6 +26,47 @@ namespace webrtc {
 
 namespace {
 
+constexpr int kSkewHistorySizeLog2 = 8;
+
+// Estimator of API call skew between render and capture.
+class SkewEstimator {
+ public:
+  // Resets the estimation.
+  void Reset() {
+    skew_ = 0;
+    next_index_ = 0;
+    sufficient_skew_stored_ = false;
+  }
+
+  // Updates the skew data for a render call.
+  void LogRenderCall() { ++skew_; }
+
+  // Updates and computes the skew at a capture call. Returns an optional which
+  // is non-null if a reliable skew has been found.
+  rtc::Optional<int> GetSkewFromCapture() {
+    --skew_;
+
+    skew_history_[next_index_] = skew_;
+    if (++next_index_ == skew_history_.size()) {
+      next_index_ = 0;
+      sufficient_skew_stored_ = true;
+    }
+
+    if (!sufficient_skew_stored_) {
+      return rtc::nullopt;
+    }
+
+    return std::accumulate(skew_history_.begin(), skew_history_.end(), 0) >>
+           kSkewHistorySizeLog2;
+  }
+
+ private:
+  int skew_ = 0;
+  std::array<int, 1 << kSkewHistorySizeLog2> skew_history_;
+  size_t next_index_ = 0;
+  bool sufficient_skew_stored_ = false;
+};
+
 class RenderDelayControllerImpl final : public RenderDelayController {
  public:
   RenderDelayControllerImpl(const EchoCanceller3Config& config,
@@ -32,6 +74,7 @@ class RenderDelayControllerImpl final : public RenderDelayController {
                             int sample_rate_hz);
   ~RenderDelayControllerImpl() override;
   void Reset() override;
+  void LogRenderCall() override;
   rtc::Optional<DelayEstimate> GetDelay(
       const DownsampledRenderBuffer& render_buffer,
       rtc::ArrayView<const float> capture) override;
@@ -47,23 +90,27 @@ class RenderDelayControllerImpl final : public RenderDelayController {
   std::vector<float> delay_buf_;
   int delay_buf_index_ = 0;
   RenderDelayControllerMetrics metrics_;
+  SkewEstimator skew_estimator_;
+  rtc::Optional<DelayEstimate> delay_samples_;
+  rtc::Optional<int> skew_;
+  int delay_change_counter_ = 0;
+  size_t soft_reset_counter_ = 0;
   RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderDelayControllerImpl);
 };
 
-DelayEstimate ComputeNewBufferDelay(
+DelayEstimate ComputeBufferDelay(
     const rtc::Optional<DelayEstimate>& current_delay,
     int delay_headroom_blocks,
     int hysteresis_limit_1_blocks,
     int hysteresis_limit_2_blocks,
+    int offset_blocks,
     DelayEstimate estimated_delay) {
   // The below division is not exact and the truncation is intended.
   const int echo_path_delay_blocks = estimated_delay.delay >> kBlockSizeLog2;
 
   // Compute the buffer delay increase required to achieve the desired latency.
-  size_t new_delay_blocks =
-      std::max(echo_path_delay_blocks - delay_headroom_blocks, 0);
-
-  DelayEstimate new_delay(estimated_delay.quality, new_delay_blocks);
+  size_t new_delay_blocks = std::max(
+      echo_path_delay_blocks + offset_blocks - delay_headroom_blocks, 0);
 
   // Add hysteresis.
   if (current_delay) {
@@ -111,8 +158,17 @@ RenderDelayControllerImpl::~RenderDelayControllerImpl() = default;
 
 void RenderDelayControllerImpl::Reset() {
   delay_ = rtc::nullopt;
+  delay_samples_ = rtc::nullopt;
+  skew_ = rtc::nullopt;
   std::fill(delay_buf_.begin(), delay_buf_.end(), 0.f);
-  delay_estimator_.Reset();
+  delay_estimator_.Reset(false);
+  skew_estimator_.Reset();
+  delay_change_counter_ = 0;
+  soft_reset_counter_ = 0;
+}
+
+void RenderDelayControllerImpl::LogRenderCall() {
+  skew_estimator_.LogRenderCall();
 }
 
 rtc::Optional<DelayEstimate> RenderDelayControllerImpl::GetDelay(
@@ -131,24 +187,64 @@ rtc::Optional<DelayEstimate> RenderDelayControllerImpl::GetDelay(
             delay_buf_.begin() + delay_buf_index_);
   delay_buf_index_ = (delay_buf_index_ + kBlockSize) % delay_buf_.size();
 
+  // Compute the latest skew update.
+  rtc::Optional<int> skew = skew_estimator_.GetSkewFromCapture();
+
   if (delay_samples) {
-    // Compute and set new render delay buffer delay.
-
-    delay_ = ComputeNewBufferDelay(delay_, delay_headroom_blocks_,
-                                   hysteresis_limit_1_blocks_,
-                                   hysteresis_limit_2_blocks_, *delay_samples);
-
-    metrics_.Update(static_cast<int>(delay_samples->delay),
-                    delay_ ? delay_->delay : 0);
-  } else {
-    metrics_.Update(rtc::nullopt, delay_ ? delay_->delay : 0);
+    if (!delay_samples_ || delay_samples->delay != delay_samples_->delay) {
+      delay_change_counter_ = 0;
+    }
+    delay_samples_ = delay_samples;
   }
 
+  if (delay_change_counter_ < 2 * kNumBlocksPerSecond) {
+    ++delay_change_counter_;
+    // If a new delay estimate is recently obtained, store the skew for that.
+    skew_ = skew;
+  } else {
+    // A reliable skew should have been obtained after 2 seconds.
+    RTC_DCHECK(skew_);
+    RTC_DCHECK(skew);
+  }
+
+  ++soft_reset_counter_;
+  int offset_blocks = 0;
+  if (skew_ && skew && delay_samples_ &&
+      delay_samples_->quality == DelayEstimate::Quality::kRefined) {
+    // Compute the skew offset and add a margin.
+    offset_blocks = *skew_ - *skew;
+    if (abs(offset_blocks) <= 1) {
+      offset_blocks = 0;
+    } else if (soft_reset_counter_ > 10 * kNumBlocksPerSecond) {
+      // Soft reset the delay estimator if there is a significant offset
+      // detected.
+      delay_estimator_.Reset(true);
+      soft_reset_counter_ = 0;
+    }
+  }
+
+  if (delay_samples_) {
+    // Compute the render delay buffer delay.
+    delay_ = ComputeBufferDelay(
+        delay_, delay_headroom_blocks_, hysteresis_limit_1_blocks_,
+        hysteresis_limit_2_blocks_, offset_blocks, *delay_samples_);
+  }
+
+  metrics_.Update(delay_samples_ ? rtc::Optional<size_t>(delay_samples_->delay)
+                                 : rtc::nullopt,
+                  delay_ ? delay_->delay : 0);
+
   data_dumper_->DumpRaw("aec3_render_delay_controller_delay",
                         delay_samples ? delay_samples->delay : 0);
   data_dumper_->DumpRaw("aec3_render_delay_controller_buffer_delay",
                         delay_ ? delay_->delay : 0);
 
+  data_dumper_->DumpRaw("aec3_render_delay_controller_new_skew",
+                        skew ? *skew : 0);
+  data_dumper_->DumpRaw("aec3_render_delay_controller_old_skew",
+                        skew_ ? *skew_ : 0);
+  data_dumper_->DumpRaw("aec3_render_delay_controller_offset", offset_blocks);
+
   return delay_;
 }
 

modules/audio_processing/aec3/render_delay_controller.h

@@ -32,6 +32,9 @@ class RenderDelayController {
   // Resets the delay controller.
   virtual void Reset() = 0;
 
+  // Logs a render call.
+  virtual void LogRenderCall() = 0;
+
   // Aligns the render buffer content with the capture signal.
   virtual rtc::Optional<DelayEstimate> GetDelay(
       const DownsampledRenderBuffer& render_buffer,