From 1ba3dbecbb2f7ee3bda107e774a9b08ce69897f2 Mon Sep 17 00:00:00 2001 From: "bjornv@google.com" Date: Mon, 3 Oct 2011 08:18:10 +0000 Subject: [PATCH] Adds possibility to log delay estimates in AEC. Review URL: http://webrtc-codereview.appspot.com/178001 git-svn-id: http://webrtc.googlecode.com/svn/trunk@674 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../aec/main/interface/echo_cancellation.h | 20 +++- .../aec/main/source/aec_core.c | 52 ++++++++++- .../aec/main/source/aec_core.h | 6 ++ .../aec/main/source/echo_cancellation.c | 88 ++++++++++++++++-- .../main/interface/audio_processing.h | 10 ++ .../main/source/echo_cancellation_impl.cc | 37 +++++++- .../main/source/echo_cancellation_impl.h | 4 + .../main/test/process_test/process_test.cc | 24 +++-- .../main/test/unit_test/unit_test.cc | 28 ++++++ .../main/test/unit_test/unittest.proto | 7 ++ .../audio_processing/output_data_float.pb | Bin 1244 -> 1316 bytes 11 files changed, 257 insertions(+), 19 deletions(-) diff --git a/src/modules/audio_processing/aec/main/interface/echo_cancellation.h b/src/modules/audio_processing/aec/main/interface/echo_cancellation.h index 883357da1e..4da6e731a7 100644 --- a/src/modules/audio_processing/aec/main/interface/echo_cancellation.h +++ b/src/modules/audio_processing/aec/main/interface/echo_cancellation.h @@ -38,6 +38,7 @@ typedef struct { WebRtc_Word16 nlpMode; // default kAecNlpModerate WebRtc_Word16 skewMode; // default kAecFalse WebRtc_Word16 metricsMode; // default kAecFalse + int delay_logging; // default kAecFalse //float realSkew; } AecConfig; @@ -66,7 +67,7 @@ extern "C" { * Inputs Description * ------------------------------------------------------------------- * void **aecInst Pointer to the AEC instance to be created - * and initilized + * and initialized * * Outputs Description * ------------------------------------------------------------------- @@ -225,6 +226,23 @@ WebRtc_Word32 WebRtcAec_get_echo_status(void *aecInst, WebRtc_Word16 *status); */ WebRtc_Word32 WebRtcAec_GetMetrics(void *aecInst, AecMetrics *metrics); +/* + * Gets the current delay metrics for the session. + * + * Inputs Description + * ------------------------------------------------------------------- + * void* handle Pointer to the AEC instance + * + * Outputs Description + * ------------------------------------------------------------------- + * int* median Delay median value. + * int* std Delay standard deviation. + * + * int return 0: OK + * -1: error + */ +int WebRtcAec_GetDelayMetrics(void* handle, int* median, int* std); + /* * Gets the last error code. * diff --git a/src/modules/audio_processing/aec/main/source/aec_core.c b/src/modules/audio_processing/aec/main/source/aec_core.c index d41319b9c5..e01728fee7 100644 --- a/src/modules/audio_processing/aec/main/source/aec_core.c +++ b/src/modules/audio_processing/aec/main/source/aec_core.c @@ -12,12 +12,14 @@ * The core AEC algorithm, which is presented with time-aligned signals. */ +#include "aec_core.h" + #include #include #include -#include "aec_core.h" #include "aec_rdft.h" +#include "delay_estimator_float.h" #include "ring_buffer.h" #include "system_wrappers/interface/cpu_features_wrapper.h" @@ -174,6 +176,15 @@ int WebRtcAec_CreateAec(aec_t **aecInst) return -1; } + if (WebRtc_CreateDelayEstimatorFloat(&aec->delay_estimator, + PART_LEN1, + kMaxDelay, + 0) == -1) { + WebRtcAec_FreeAec(aec); + aec = NULL; + return -1; + } + return 0; } @@ -190,6 +201,8 @@ int WebRtcAec_FreeAec(aec_t *aec) WebRtcApm_FreeBuffer(aec->nearFrBufH); WebRtcApm_FreeBuffer(aec->outFrBufH); + WebRtc_FreeDelayEstimatorFloat(aec->delay_estimator); + free(aec); return 0; } @@ -371,6 +384,12 @@ int WebRtcAec_InitAec(aec_t *aec, int sampFreq) return -1; } + if (WebRtc_InitDelayEstimatorFloat(aec->delay_estimator) != 0) { + return -1; + } + aec->delay_logging_enabled = 0; + memset(aec->delay_histogram, 0, sizeof(aec->delay_histogram)); + // Default target suppression level aec->targetSupp = -11.5; aec->minOverDrive = 2.0; @@ -561,6 +580,10 @@ static void ProcessBlock(aec_t *aec, const short *farend, float fft[PART_LEN2]; float xf[2][PART_LEN1], yf[2][PART_LEN1], ef[2][PART_LEN1]; complex_t df[PART_LEN1]; + float far_spectrum = 0.0f; + float near_spectrum = 0.0f; + float abs_far_spectrum[PART_LEN1]; + float abs_near_spectrum[PART_LEN1]; const float gPow[2] = {0.9f, 0.1f}; @@ -625,10 +648,15 @@ static void ProcessBlock(aec_t *aec, const short *farend, // Power smoothing for (i = 0; i < PART_LEN1; i++) { - aec->xPow[i] = gPow[0] * aec->xPow[i] + gPow[1] * NR_PART * - (xf[0][i] * xf[0][i] + xf[1][i] * xf[1][i]); - aec->dPow[i] = gPow[0] * aec->dPow[i] + gPow[1] * - (df[i][0] * df[i][0] + df[i][1] * df[i][1]); + far_spectrum = xf[0][i] * xf[0][i] + xf[1][i] * xf[1][i]; + aec->xPow[i] = gPow[0] * aec->xPow[i] + gPow[1] * NR_PART * far_spectrum; + // Calculate absolute spectra + abs_far_spectrum[i] = sqrtf(far_spectrum); + + near_spectrum = df[i][0] * df[i][0] + df[i][1] * df[i][1]; + aec->dPow[i] = gPow[0] * aec->dPow[i] + gPow[1] * near_spectrum; + // Calculate absolute spectra + abs_near_spectrum[i] = sqrtf(near_spectrum); } // Estimate noise power. Wait until dPow is more stable. @@ -663,6 +691,20 @@ static void ProcessBlock(aec_t *aec, const short *farend, aec->noisePow = aec->dMinPow; } + // Block wise delay estimation used for logging + if (aec->delay_logging_enabled) { + int delay_estimate = 0; + // Estimate the delay + delay_estimate = WebRtc_DelayEstimatorProcessFloat(aec->delay_estimator, + abs_far_spectrum, + abs_near_spectrum, + PART_LEN1, + aec->echoState); + if (delay_estimate >= 0) { + // Update delay estimate buffer + aec->delay_histogram[delay_estimate]++; + } + } // Update the xfBuf block position. aec->xfBufBlockPos--; diff --git a/src/modules/audio_processing/aec/main/source/aec_core.h b/src/modules/audio_processing/aec/main/source/aec_core.h index eb99f9a337..e693425858 100644 --- a/src/modules/audio_processing/aec/main/source/aec_core.h +++ b/src/modules/audio_processing/aec/main/source/aec_core.h @@ -33,6 +33,8 @@ #define PREF_BAND_SIZE 24 #define BLOCKL_MAX FRAME_LEN +// Maximum delay in fixed point delay estimator, used for logging +enum {kMaxDelay = 100}; typedef float complex_t[2]; // For performance reasons, some arrays of complex numbers are replaced by twice @@ -141,6 +143,10 @@ typedef struct { int flag_Hband_cn; //for comfort noise float cn_scale_Hband; //scale for comfort noise in H band + int delay_histogram[kMaxDelay]; + int delay_logging_enabled; + void* delay_estimator; + #ifdef AEC_DEBUG FILE *farFile; FILE *nearFile; diff --git a/src/modules/audio_processing/aec/main/source/echo_cancellation.c b/src/modules/audio_processing/aec/main/source/echo_cancellation.c index f6a2538d98..36738f75b7 100644 --- a/src/modules/audio_processing/aec/main/source/echo_cancellation.c +++ b/src/modules/audio_processing/aec/main/source/echo_cancellation.c @@ -11,16 +11,18 @@ /* * Contains the API functions for the AEC. */ +#include "echo_cancellation.h" + +#include +#ifdef AEC_DEBUG +#include +#endif #include #include -#include "echo_cancellation.h" #include "aec_core.h" -#include "ring_buffer.h" #include "resampler.h" -#ifdef AEC_DEBUG - #include -#endif +#include "ring_buffer.h" #define BUF_SIZE_FRAMES 50 // buffer size (frames) // Maximum length of resampled signal. Must be an integer multiple of frames @@ -215,7 +217,7 @@ WebRtc_Word32 WebRtcAec_Init(void *aecInst, WebRtc_Word32 sampFreq, WebRtc_Word3 return -1; } - aecpc->initFlag = initCheck; // indicates that initilisation has been done + aecpc->initFlag = initCheck; // indicates that initialization has been done if (aecpc->sampFreq == 32000) { aecpc->splitSampFreq = 16000; @@ -254,6 +256,7 @@ WebRtc_Word32 WebRtcAec_Init(void *aecInst, WebRtc_Word32 sampFreq, WebRtc_Word3 aecConfig.nlpMode = kAecNlpModerate; aecConfig.skewMode = kAecFalse; aecConfig.metricsMode = kAecFalse; + aecConfig.delay_logging = kAecFalse; if (WebRtcAec_set_config(aecpc, aecConfig) == -1) { aecpc->lastError = AEC_UNSPECIFIED_ERROR; @@ -566,6 +569,15 @@ WebRtc_Word32 WebRtcAec_set_config(void *aecInst, AecConfig config) WebRtcAec_InitMetrics(aecpc->aec); } + if (config.delay_logging != kAecFalse && config.delay_logging != kAecTrue) { + aecpc->lastError = AEC_BAD_PARAMETER_ERROR; + return -1; + } + aecpc->aec->delay_logging_enabled = config.delay_logging; + if (aecpc->aec->delay_logging_enabled == kAecTrue) { + memset(aecpc->aec->delay_histogram, 0, sizeof(aecpc->aec->delay_histogram)); + } + return 0; } @@ -590,6 +602,7 @@ WebRtc_Word32 WebRtcAec_get_config(void *aecInst, AecConfig *config) config->nlpMode = aecpc->nlpMode; config->skewMode = aecpc->skewMode; config->metricsMode = aecpc->aec->metricsMode; + config->delay_logging = aecpc->aec->delay_logging_enabled; return 0; } @@ -717,6 +730,69 @@ WebRtc_Word32 WebRtcAec_GetMetrics(void *aecInst, AecMetrics *metrics) return 0; } +int WebRtcAec_GetDelayMetrics(void* handle, int* median, int* std) { + aecpc_t* self = handle; + int i = 0; + int delay_values = 0; + int num_delay_values = 0; + int my_median = 0; + float l1_norm = 0; + + if (self == NULL) { + return -1; + } + if (median == NULL) { + self->lastError = AEC_NULL_POINTER_ERROR; + return -1; + } + if (std == NULL) { + self->lastError = AEC_NULL_POINTER_ERROR; + return -1; + } + if (self->initFlag != initCheck) { + self->lastError = AEC_UNINITIALIZED_ERROR; + return -1; + } + if (self->aec->delay_logging_enabled == 0) { + // Logging disabled + self->lastError = AEC_UNSUPPORTED_FUNCTION_ERROR; + return -1; + } + + // Get number of delay values since last update + for (i = 0; i < kMaxDelay; i++) { + num_delay_values += self->aec->delay_histogram[i]; + } + if (num_delay_values == 0) { + // We have no new delay value data + *median = -1; + *std = -1; + return 0; + } + + delay_values = num_delay_values >> 1; // Start value for median count down + // Get median of delay values since last update + for (i = 0; i < kMaxDelay; i++) { + delay_values -= self->aec->delay_histogram[i]; + if (delay_values < 0) { + my_median = i; + break; + } + } + *median = my_median; + + // Calculate the L1 norm, with median value as central moment + for (i = 0; i < kMaxDelay; i++) { + l1_norm += (float) (fabs(i - my_median) * self->aec->delay_histogram[i]); + } + *std = (int) (l1_norm / (float) num_delay_values + 0.5f); + + // Reset histogram + memset(self->aec->delay_histogram, 0, sizeof(self->aec->delay_histogram)); + + return 0; +} + WebRtc_Word32 WebRtcAec_get_version(WebRtc_Word8 *versionStr, WebRtc_Word16 len) { const char version[] = "AEC 2.5.0"; diff --git a/src/modules/audio_processing/main/interface/audio_processing.h b/src/modules/audio_processing/main/interface/audio_processing.h index c8c87127ad..3dc698bf36 100644 --- a/src/modules/audio_processing/main/interface/audio_processing.h +++ b/src/modules/audio_processing/main/interface/audio_processing.h @@ -322,6 +322,16 @@ class EchoCancellation { // TODO(ajm): discuss the metrics update period. virtual int GetMetrics(Metrics* metrics) = 0; + // Enables computation and logging of delay values. Statistics are obtained + // through |GetDelayMetrics()|. + virtual int enable_delay_logging(bool enable) = 0; + virtual bool is_delay_logging_enabled() const = 0; + + // The delay metrics consists of the delay |median| and the delay standard + // deviation |std|. The values are averaged over the time period since the + // last call to |GetDelayMetrics()|. + virtual int GetDelayMetrics(int* median, int* std) = 0; + protected: virtual ~EchoCancellation() {}; }; diff --git a/src/modules/audio_processing/main/source/echo_cancellation_impl.cc b/src/modules/audio_processing/main/source/echo_cancellation_impl.cc index 886d5f158c..61940b1498 100644 --- a/src/modules/audio_processing/main/source/echo_cancellation_impl.cc +++ b/src/modules/audio_processing/main/source/echo_cancellation_impl.cc @@ -66,7 +66,8 @@ EchoCancellationImpl::EchoCancellationImpl(const AudioProcessingImpl* apm) device_sample_rate_hz_(48000), stream_drift_samples_(0), was_stream_drift_set_(false), - stream_has_echo_(false) {} + stream_has_echo_(false), + delay_logging_enabled_(false) {} EchoCancellationImpl::~EchoCancellationImpl() {} @@ -283,6 +284,39 @@ bool EchoCancellationImpl::stream_has_echo() const { return stream_has_echo_; } +int EchoCancellationImpl::enable_delay_logging(bool enable) { + CriticalSectionScoped crit_scoped(*apm_->crit()); + delay_logging_enabled_ = enable; + return Configure(); +} + +bool EchoCancellationImpl::is_delay_logging_enabled() const { + return delay_logging_enabled_; +} + +// TODO(bjornv): How should we handle the multi-channel case? +int EchoCancellationImpl::GetDelayMetrics(int* median, int* std) { + CriticalSectionScoped crit_scoped(*apm_->crit()); + if (median == NULL) { + return apm_->kNullPointerError; + } + if (std == NULL) { + return apm_->kNullPointerError; + } + + if (!is_component_enabled() || !delay_logging_enabled_) { + return apm_->kNotEnabledError; + } + + Handle* my_handle = static_cast(handle(0)); + if (WebRtcAec_GetDelayMetrics(my_handle, median, std) != + apm_->kNoError) { + return GetHandleError(my_handle); + } + + return apm_->kNoError; +} + int EchoCancellationImpl::Initialize() { int err = ProcessingComponent::Initialize(); if (err != apm_->kNoError || !is_component_enabled()) { @@ -332,6 +366,7 @@ int EchoCancellationImpl::ConfigureHandle(void* handle) const { config.metricsMode = metrics_enabled_; config.nlpMode = MapSetting(suppression_level_); config.skewMode = drift_compensation_enabled_; + config.delay_logging = delay_logging_enabled_; return WebRtcAec_set_config(static_cast(handle), config); } diff --git a/src/modules/audio_processing/main/source/echo_cancellation_impl.h b/src/modules/audio_processing/main/source/echo_cancellation_impl.h index 071c18fbfc..a483a3aee4 100644 --- a/src/modules/audio_processing/main/source/echo_cancellation_impl.h +++ b/src/modules/audio_processing/main/source/echo_cancellation_impl.h @@ -49,6 +49,9 @@ class EchoCancellationImpl : public EchoCancellation, virtual bool are_metrics_enabled() const; virtual bool stream_has_echo() const; virtual int GetMetrics(Metrics* metrics); + virtual int enable_delay_logging(bool enable); + virtual bool is_delay_logging_enabled() const; + virtual int GetDelayMetrics(int* median, int* std); // ProcessingComponent implementation. virtual void* CreateHandle() const; @@ -66,6 +69,7 @@ class EchoCancellationImpl : public EchoCancellation, int stream_drift_samples_; bool was_stream_drift_set_; bool stream_has_echo_; + bool delay_logging_enabled_; }; } // namespace webrtc diff --git a/src/modules/audio_processing/main/test/process_test/process_test.cc b/src/modules/audio_processing/main/test/process_test/process_test.cc index eb3174c1ff..130f3ee59e 100644 --- a/src/modules/audio_processing/main/test/process_test/process_test.cc +++ b/src/modules/audio_processing/main/test/process_test/process_test.cc @@ -93,8 +93,8 @@ void usage() { printf("\n -aec Echo cancellation\n"); printf(" --drift_compensation\n"); printf(" --no_drift_compensation\n"); - printf(" --echo_metrics\n"); printf(" --no_echo_metrics\n"); + printf(" --no_delay_logging\n"); printf("\n -aecm Echo control mobile\n"); printf(" --aecm_echo_path_in_file FILE\n"); printf(" --aecm_echo_path_out_file FILE\n"); @@ -218,6 +218,10 @@ void void_main(int argc, char* argv[]) { } else if (strcmp(argv[i], "-aec") == 0) { ASSERT_EQ(apm->kNoError, apm->echo_cancellation()->Enable(true)); + ASSERT_EQ(apm->kNoError, + apm->echo_cancellation()->enable_metrics(true)); + ASSERT_EQ(apm->kNoError, + apm->echo_cancellation()->enable_delay_logging(true)); } else if (strcmp(argv[i], "--drift_compensation") == 0) { ASSERT_EQ(apm->kNoError, apm->echo_cancellation()->Enable(true)); @@ -230,16 +234,16 @@ void void_main(int argc, char* argv[]) { ASSERT_EQ(apm->kNoError, apm->echo_cancellation()->enable_drift_compensation(false)); - } else if (strcmp(argv[i], "--echo_metrics") == 0) { - ASSERT_EQ(apm->kNoError, apm->echo_cancellation()->Enable(true)); - ASSERT_EQ(apm->kNoError, - apm->echo_cancellation()->enable_metrics(true)); - } else if (strcmp(argv[i], "--no_echo_metrics") == 0) { ASSERT_EQ(apm->kNoError, apm->echo_cancellation()->Enable(true)); ASSERT_EQ(apm->kNoError, apm->echo_cancellation()->enable_metrics(false)); + } else if (strcmp(argv[i], "--no_delay_logging") == 0) { + ASSERT_EQ(apm->kNoError, apm->echo_cancellation()->Enable(true)); + ASSERT_EQ(apm->kNoError, + apm->echo_cancellation()->enable_delay_logging(false)); + } else if (strcmp(argv[i], "-aecm") == 0) { ASSERT_EQ(apm->kNoError, apm->echo_control_mobile()->Enable(true)); @@ -884,6 +888,14 @@ void void_main(int argc, char* argv[]) { printf("ANLP: "); PrintStat(metrics.a_nlp); } + if (apm->echo_cancellation()->is_delay_logging_enabled()) { + int median = 0; + int std = 0; + apm->echo_cancellation()->GetDelayMetrics(&median, &std); + printf("\n--Delay metrics--\n"); + printf("Median: %3d\n", median); + printf("Standard deviation: %3d\n", std); + } } if (!pb_file) { diff --git a/src/modules/audio_processing/main/test/unit_test/unit_test.cc b/src/modules/audio_processing/main/test/unit_test/unit_test.cc index 5c9f5afd3e..973bafebfa 100644 --- a/src/modules/audio_processing/main/test/unit_test/unit_test.cc +++ b/src/modules/audio_processing/main/test/unit_test/unit_test.cc @@ -457,6 +457,8 @@ TEST_F(ApmTest, Process) { apm_->echo_cancellation()->enable_drift_compensation(true)); EXPECT_EQ(apm_->kNoError, apm_->echo_cancellation()->enable_metrics(true)); + EXPECT_EQ(apm_->kNoError, + apm_->echo_cancellation()->enable_delay_logging(true)); EXPECT_EQ(apm_->kNoError, apm_->echo_cancellation()->Enable(true)); EXPECT_EQ(apm_->kNoError, @@ -591,6 +593,10 @@ TEST_F(ApmTest, Process) { EchoCancellation::Metrics echo_metrics; EXPECT_EQ(apm_->kNoError, apm_->echo_cancellation()->GetMetrics(&echo_metrics)); + int median = 0; + int std = 0; + EXPECT_EQ(apm_->kNoError, + apm_->echo_cancellation()->GetDelayMetrics(&median, &std)); #endif if (!write_output_data) { @@ -612,6 +618,11 @@ TEST_F(ApmTest, Process) { reference.echo_return_loss_enhancement()); TestStats(echo_metrics.a_nlp, reference.a_nlp()); + + webrtc::audioproc::Test::DelayMetrics reference_delay = + test->delay_metrics(); + EXPECT_EQ(median, reference_delay.median()); + EXPECT_EQ(std, reference_delay.std()); #endif } else { test->set_has_echo_count(has_echo_count); @@ -632,6 +643,11 @@ TEST_F(ApmTest, Process) { message->mutable_echo_return_loss_enhancement()); WriteStatsMessage(echo_metrics.a_nlp, message->mutable_a_nlp()); + + webrtc::audioproc::Test::DelayMetrics* message_delay = + test->mutable_delay_metrics(); + message_delay->set_median(median); + message_delay->set_std(std); #endif } @@ -696,6 +712,18 @@ TEST_F(ApmTest, EchoCancellation) { apm_->echo_cancellation()->enable_metrics(false)); EXPECT_FALSE(apm_->echo_cancellation()->are_metrics_enabled()); + int median = 0; + int std = 0; + EXPECT_EQ(apm_->kNotEnabledError, + apm_->echo_cancellation()->GetDelayMetrics(&median, &std)); + + EXPECT_EQ(apm_->kNoError, + apm_->echo_cancellation()->enable_delay_logging(true)); + EXPECT_TRUE(apm_->echo_cancellation()->is_delay_logging_enabled()); + EXPECT_EQ(apm_->kNoError, + apm_->echo_cancellation()->enable_delay_logging(false)); + EXPECT_FALSE(apm_->echo_cancellation()->is_delay_logging_enabled()); + EXPECT_EQ(apm_->kNoError, apm_->echo_cancellation()->Enable(true)); EXPECT_TRUE(apm_->echo_cancellation()->is_enabled()); EXPECT_EQ(apm_->kNoError, apm_->echo_cancellation()->Enable(false)); diff --git a/src/modules/audio_processing/main/test/unit_test/unittest.proto b/src/modules/audio_processing/main/test/unit_test/unittest.proto index 3dde02b752..cdfacc4627 100644 --- a/src/modules/audio_processing/main/test/unit_test/unittest.proto +++ b/src/modules/audio_processing/main/test/unit_test/unittest.proto @@ -35,6 +35,13 @@ message Test { } optional EchoMetrics echo_metrics = 11; + + message DelayMetrics { + optional int32 median = 1; + optional int32 std = 2; + } + + optional DelayMetrics delay_metrics = 12; } message OutputData { diff --git a/test/data/audio_processing/output_data_float.pb b/test/data/audio_processing/output_data_float.pb index 2fc85bc5ff739a6cd091eb27578f8ebc9dc6aec0..985cad8660b7ffa2d7d8d912c92cfd2716facf78 100644 GIT binary patch delta 162 zcmcb^xrB>Nft7)QD|;eaDn}9v2crOpoeO3&2rzKvOuUi?5(lxfC)+TBCBTX%FNSbH tGC7mEn2?keF(E0t3E@DLIWQwBTfvN^jGqO}fha3uNn=Z5;b0J8008&I9wh(( delta 89 zcmZ3&b%&Gf1`7iNSL#HzR3@&}iN&c*Txk;@q%m=&PWEF22~FM&qS7X7Fo6Z9F@Xh{ YnL!FC7cheb?}2!!lf7BMf;(8!0AuwS?EnA(