From 2224294c52fc0bb95eedb8187a62bedf1b5eb853 Mon Sep 17 00:00:00 2001 From: Karl Wiberg Date: Fri, 3 Jul 2015 04:04:33 +0200 Subject: [PATCH] iSAC: Functions for importing and exporting bandwidth est. info They make it possible to send bandwidth estimation info from decoder to encoder even if they are separate objects (which we want them to be because multithreading). R=henrik.lundin@webrtc.org Review URL: https://codereview.webrtc.org/1208923002. Cr-Commit-Position: refs/heads/master@{#9535} --- .../audio_coding/codecs/isac/bandwidth_info.h | 24 ++ .../fix/interface/audio_encoder_isacfix.h | 9 +- .../codecs/isac/fix/interface/isacfix.h | 11 +- .../isac/fix/source/bandwidth_estimator.c | 72 +++-- .../isac/fix/source/bandwidth_estimator.h | 8 + .../codecs/isac/fix/source/isacfix.c | 27 +- .../codecs/isac/fix/source/structs.h | 5 +- .../isac/main/interface/audio_encoder_isac.h | 9 +- .../codecs/isac/main/interface/isac.h | 10 +- .../isac/main/source/bandwidth_estimator.c | 93 +++--- .../isac/main/source/bandwidth_estimator.h | 21 +- .../codecs/isac/main/source/isac.c | 23 +- .../codecs/isac/main/source/structs.h | 3 + .../audio_coding/codecs/isac/unittest.cc | 271 ++++++++++++++++++ webrtc/modules/modules.gyp | 1 + 15 files changed, 488 insertions(+), 99 deletions(-) create mode 100644 webrtc/modules/audio_coding/codecs/isac/bandwidth_info.h create mode 100644 webrtc/modules/audio_coding/codecs/isac/unittest.cc diff --git a/webrtc/modules/audio_coding/codecs/isac/bandwidth_info.h b/webrtc/modules/audio_coding/codecs/isac/bandwidth_info.h new file mode 100644 index 0000000000..1e3f4c9a86 --- /dev/null +++ b/webrtc/modules/audio_coding/codecs/isac/bandwidth_info.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_AUDIO_CODING_CODECS_ISAC_BANDWIDTH_INFO_H_ +#define WEBRTC_MODULES_AUDIO_CODING_CODECS_ISAC_BANDWIDTH_INFO_H_ + +#include "webrtc/typedefs.h" + +typedef struct { + int in_use; + int32_t send_bw_avg; + int32_t send_max_delay_avg; + int16_t bottleneck_idx; + int16_t jitter_info; +} IsacBandwidthInfo; + +#endif // WEBRTC_MODULES_AUDIO_CODING_CODECS_ISAC_BANDWIDTH_INFO_H_ diff --git a/webrtc/modules/audio_coding/codecs/isac/fix/interface/audio_encoder_isacfix.h b/webrtc/modules/audio_coding/codecs/isac/fix/interface/audio_encoder_isacfix.h index a1eb2714f9..f5f037de3a 100644 --- a/webrtc/modules/audio_coding/codecs/isac/fix/interface/audio_encoder_isacfix.h +++ b/webrtc/modules/audio_coding/codecs/isac/fix/interface/audio_encoder_isacfix.h @@ -68,6 +68,10 @@ struct IsacFix { static inline int16_t Free(instance_type* inst) { return WebRtcIsacfix_Free(inst); } + static inline void GetBandwidthInfo(instance_type* inst, + IsacBandwidthInfo* bwinfo) { + WebRtcIsacfix_GetBandwidthInfo(inst, bwinfo); + } static inline int16_t GetErrorCode(instance_type* inst) { return WebRtcIsacfix_GetErrorCode(inst); } @@ -75,7 +79,10 @@ struct IsacFix { static inline int16_t GetNewFrameLen(instance_type* inst) { return WebRtcIsacfix_GetNewFrameLen(inst); } - + static inline void SetBandwidthInfo(instance_type* inst, + const IsacBandwidthInfo* bwinfo) { + WebRtcIsacfix_SetBandwidthInfo(inst, bwinfo); + } static inline int16_t SetDecSampRate(instance_type* inst, uint16_t sample_rate_hz) { DCHECK_EQ(sample_rate_hz, kFixSampleRate); diff --git a/webrtc/modules/audio_coding/codecs/isac/fix/interface/isacfix.h b/webrtc/modules/audio_coding/codecs/isac/fix/interface/isacfix.h index 4dbc29d88d..a205c6d641 100644 --- a/webrtc/modules/audio_coding/codecs/isac/fix/interface/isacfix.h +++ b/webrtc/modules/audio_coding/codecs/isac/fix/interface/isacfix.h @@ -11,9 +11,7 @@ #ifndef WEBRTC_MODULES_AUDIO_CODING_CODECS_ISAC_FIX_INTERFACE_ISACFIX_H_ #define WEBRTC_MODULES_AUDIO_CODING_CODECS_ISAC_FIX_INTERFACE_ISACFIX_H_ -/* - * Define the fixpoint numeric formats - */ +#include "webrtc/modules/audio_coding/codecs/isac/bandwidth_info.h" #include "webrtc/typedefs.h" typedef struct { @@ -626,6 +624,13 @@ extern "C" { int16_t WebRtcIsacfix_GetNewFrameLen(ISACFIX_MainStruct *ISAC_main_inst); + /* Fills in an IsacBandwidthInfo struct. */ + void WebRtcIsacfix_GetBandwidthInfo(ISACFIX_MainStruct* ISAC_main_inst, + IsacBandwidthInfo* bwinfo); + + /* Uses the values from an IsacBandwidthInfo struct. */ + void WebRtcIsacfix_SetBandwidthInfo(ISACFIX_MainStruct* ISAC_main_inst, + const IsacBandwidthInfo* bwinfo); #if defined(__cplusplus) } diff --git a/webrtc/modules/audio_coding/codecs/isac/fix/source/bandwidth_estimator.c b/webrtc/modules/audio_coding/codecs/isac/fix/source/bandwidth_estimator.c index 4a4cddc3db..d876a3cb83 100644 --- a/webrtc/modules/audio_coding/codecs/isac/fix/source/bandwidth_estimator.c +++ b/webrtc/modules/audio_coding/codecs/isac/fix/source/bandwidth_estimator.c @@ -19,6 +19,8 @@ */ #include "bandwidth_estimator.h" + +#include #include "settings.h" @@ -116,6 +118,8 @@ int32_t WebRtcIsacfix_InitBandwidthEstimator(BwEstimatorstr *bweStr) bweStr->maxBwInv = kInvBandwidth[3]; bweStr->minBwInv = kInvBandwidth[2]; + bweStr->external_bw_info.in_use = 0; + return 0; } @@ -176,6 +180,8 @@ int32_t WebRtcIsacfix_UpdateUplinkBwImpl(BwEstimatorstr *bweStr, int16_t errCode; + assert(!bweStr->external_bw_info.in_use); + /* UPDATE ESTIMATES FROM OTHER SIDE */ /* The function also checks if Index has a valid value */ @@ -545,6 +551,8 @@ int16_t WebRtcIsacfix_UpdateUplinkBwRec(BwEstimatorstr *bweStr, { uint16_t RateInd; + assert(!bweStr->external_bw_info.in_use); + if ( (Index < 0) || (Index > 23) ) { return -ISAC_RANGE_ERROR_BW_ESTIMATOR; } @@ -616,6 +624,9 @@ uint16_t WebRtcIsacfix_GetDownlinkBwIndexImpl(BwEstimatorstr *bweStr) int32_t tempMin; int32_t tempMax; + if (bweStr->external_bw_info.in_use) + return bweStr->external_bw_info.bottleneck_idx; + /* Get Rate Index */ /* Get unquantized rate. Always returns 10000 <= rate <= 32000 */ @@ -721,6 +732,8 @@ uint16_t WebRtcIsacfix_GetDownlinkBandwidth(const BwEstimatorstr *bweStr) int32_t rec_jitter_short_term_abs_inv; /* Q18 */ int32_t temp; + assert(!bweStr->external_bw_info.in_use); + /* Q18 rec jitter short term abs is in Q13, multiply it by 2^13 to save precision 2^18 then needs to be shifted 13 bits to 2^31 */ rec_jitter_short_term_abs_inv = 0x80000000u / bweStr->recJitterShortTermAbs; @@ -777,6 +790,8 @@ int16_t WebRtcIsacfix_GetDownlinkMaxDelay(const BwEstimatorstr *bweStr) { int16_t recMaxDelay = (int16_t)(bweStr->recMaxDelay >> 15); + assert(!bweStr->external_bw_info.in_use); + /* limit range of jitter estimate */ if (recMaxDelay < MIN_ISAC_MD) { recMaxDelay = MIN_ISAC_MD; @@ -787,42 +802,39 @@ int16_t WebRtcIsacfix_GetDownlinkMaxDelay(const BwEstimatorstr *bweStr) return recMaxDelay; } -/* get the bottle neck rate from here to far side, as estimated by far side */ -int16_t WebRtcIsacfix_GetUplinkBandwidth(const BwEstimatorstr *bweStr) -{ - int16_t send_bw; - - send_bw = (int16_t) WEBRTC_SPL_RSHIFT_U32(bweStr->sendBwAvg, 7); - - /* limit range of bottle neck rate */ - if (send_bw < MIN_ISAC_BW) { - send_bw = MIN_ISAC_BW; - } else if (send_bw > MAX_ISAC_BW) { - send_bw = MAX_ISAC_BW; - } - - return send_bw; +/* Clamp val to the closed interval [min,max]. */ +static int16_t clamp(int16_t val, int16_t min, int16_t max) { + assert(min <= max); + return val < min ? min : (val > max ? max : val); } - - -/* Returns the max delay value from the other side in ms */ -int16_t WebRtcIsacfix_GetUplinkMaxDelay(const BwEstimatorstr *bweStr) -{ - int16_t send_max_delay = (int16_t)(bweStr->sendMaxDelayAvg >> 9); - - /* limit range of jitter estimate */ - if (send_max_delay < MIN_ISAC_MD) { - send_max_delay = MIN_ISAC_MD; - } else if (send_max_delay > MAX_ISAC_MD) { - send_max_delay = MAX_ISAC_MD; - } - - return send_max_delay; +int16_t WebRtcIsacfix_GetUplinkBandwidth(const BwEstimatorstr* bweStr) { + return bweStr->external_bw_info.in_use + ? bweStr->external_bw_info.send_bw_avg + : clamp(bweStr->sendBwAvg >> 7, MIN_ISAC_BW, MAX_ISAC_BW); } +int16_t WebRtcIsacfix_GetUplinkMaxDelay(const BwEstimatorstr* bweStr) { + return bweStr->external_bw_info.in_use + ? bweStr->external_bw_info.send_max_delay_avg + : clamp(bweStr->sendMaxDelayAvg >> 9, MIN_ISAC_MD, MAX_ISAC_MD); +} +void WebRtcIsacfixBw_GetBandwidthInfo(BwEstimatorstr* bweStr, + IsacBandwidthInfo* bwinfo) { + assert(!bweStr->external_bw_info.in_use); + bwinfo->in_use = 1; + bwinfo->send_bw_avg = WebRtcIsacfix_GetUplinkBandwidth(bweStr); + bwinfo->send_max_delay_avg = WebRtcIsacfix_GetUplinkMaxDelay(bweStr); + bwinfo->bottleneck_idx = WebRtcIsacfix_GetDownlinkBwIndexImpl(bweStr); + bwinfo->jitter_info = 0; // Not used. +} +void WebRtcIsacfixBw_SetBandwidthInfo(BwEstimatorstr* bweStr, + const IsacBandwidthInfo* bwinfo) { + memcpy(&bweStr->external_bw_info, bwinfo, + sizeof bweStr->external_bw_info); +} /* * update long-term average bitrate and amount of data in buffer diff --git a/webrtc/modules/audio_coding/codecs/isac/fix/source/bandwidth_estimator.h b/webrtc/modules/audio_coding/codecs/isac/fix/source/bandwidth_estimator.h index acd5dd7354..5d8ccbcd7d 100644 --- a/webrtc/modules/audio_coding/codecs/isac/fix/source/bandwidth_estimator.h +++ b/webrtc/modules/audio_coding/codecs/isac/fix/source/bandwidth_estimator.h @@ -95,6 +95,14 @@ int16_t WebRtcIsacfix_GetDownlinkMaxDelay(const BwEstimatorstr *bwest_str); /* Returns the max delay value from the other side in ms */ int16_t WebRtcIsacfix_GetUplinkMaxDelay(const BwEstimatorstr *bwest_str); +/* Fills in an IsacExternalBandwidthInfo struct. */ +void WebRtcIsacfixBw_GetBandwidthInfo(BwEstimatorstr* bwest_str, + IsacBandwidthInfo* bwinfo); + +/* Uses the values from an IsacExternalBandwidthInfo struct. */ +void WebRtcIsacfixBw_SetBandwidthInfo(BwEstimatorstr* bwest_str, + const IsacBandwidthInfo* bwinfo); + /* * update amount of data in bottle neck buffer and burst handling * returns minimum payload size (bytes) diff --git a/webrtc/modules/audio_coding/codecs/isac/fix/source/isacfix.c b/webrtc/modules/audio_coding/codecs/isac/fix/source/isacfix.c index 9b1927bef9..2441e41ccb 100644 --- a/webrtc/modules/audio_coding/codecs/isac/fix/source/isacfix.c +++ b/webrtc/modules/audio_coding/codecs/isac/fix/source/isacfix.c @@ -72,10 +72,12 @@ int16_t WebRtcIsacfix_AssignSize(int *sizeinbytes) { int16_t WebRtcIsacfix_Assign(ISACFIX_MainStruct **inst, void *ISACFIX_inst_Addr) { if (ISACFIX_inst_Addr!=NULL) { - *inst = (ISACFIX_MainStruct*)ISACFIX_inst_Addr; - (*(ISACFIX_SubStruct**)inst)->errorcode = 0; - (*(ISACFIX_SubStruct**)inst)->initflag = 0; - (*(ISACFIX_SubStruct**)inst)->ISACenc_obj.SaveEnc_ptr = NULL; + ISACFIX_SubStruct* self = ISACFIX_inst_Addr; + *inst = (ISACFIX_MainStruct*)self; + self->errorcode = 0; + self->initflag = 0; + self->ISACenc_obj.SaveEnc_ptr = NULL; + WebRtcIsacfix_InitBandwidthEstimator(&self->bwestimator_obj); return(0); } else { return(-1); @@ -108,6 +110,7 @@ int16_t WebRtcIsacfix_Create(ISACFIX_MainStruct **ISAC_main_inst) (*(ISACFIX_SubStruct**)ISAC_main_inst)->initflag = 0; (*(ISACFIX_SubStruct**)ISAC_main_inst)->ISACenc_obj.SaveEnc_ptr = NULL; WebRtcSpl_Init(); + WebRtcIsacfix_InitBandwidthEstimator(&tempo->bwestimator_obj); return(0); } else { return(-1); @@ -293,8 +296,6 @@ int16_t WebRtcIsacfix_EncoderInit(ISACFIX_MainStruct *ISAC_main_inst, WebRtcIsacfix_InitPitchFilter(&ISAC_inst->ISACenc_obj.pitchfiltstr_obj); WebRtcIsacfix_InitPitchAnalysis(&ISAC_inst->ISACenc_obj.pitchanalysisstr_obj); - - WebRtcIsacfix_InitBandwidthEstimator(&ISAC_inst->bwestimator_obj); WebRtcIsacfix_InitRateModel(&ISAC_inst->ISACenc_obj.rate_data_obj); @@ -1526,3 +1527,17 @@ void WebRtcIsacfix_version(char *version) { strcpy(version, "3.6.0"); } + +void WebRtcIsacfix_GetBandwidthInfo(ISACFIX_MainStruct* ISAC_main_inst, + IsacBandwidthInfo* bwinfo) { + ISACFIX_SubStruct* inst = (ISACFIX_SubStruct*)ISAC_main_inst; + assert(inst->initflag & 1); // Decoder initialized. + WebRtcIsacfixBw_GetBandwidthInfo(&inst->bwestimator_obj, bwinfo); +} + +void WebRtcIsacfix_SetBandwidthInfo(ISACFIX_MainStruct* ISAC_main_inst, + const IsacBandwidthInfo* bwinfo) { + ISACFIX_SubStruct* inst = (ISACFIX_SubStruct*)ISAC_main_inst; + assert(inst->initflag & 2); // Encoder initialized. + WebRtcIsacfixBw_SetBandwidthInfo(&inst->bwestimator_obj, bwinfo); +} diff --git a/webrtc/modules/audio_coding/codecs/isac/fix/source/structs.h b/webrtc/modules/audio_coding/codecs/isac/fix/source/structs.h index 5f3e0c3036..5abbd7ad44 100644 --- a/webrtc/modules/audio_coding/codecs/isac/fix/source/structs.h +++ b/webrtc/modules/audio_coding/codecs/isac/fix/source/structs.h @@ -20,6 +20,7 @@ #include "webrtc/common_audio/signal_processing/include/signal_processing_library.h" +#include "webrtc/modules/audio_coding/codecs/isac/bandwidth_info.h" #include "webrtc/modules/audio_coding/codecs/isac/fix/source/settings.h" #include "webrtc/typedefs.h" @@ -245,9 +246,7 @@ typedef struct { bwe will assume the connection is over broadband network */ int16_t highSpeedSend; - - - + IsacBandwidthInfo external_bw_info; } BwEstimatorstr; diff --git a/webrtc/modules/audio_coding/codecs/isac/main/interface/audio_encoder_isac.h b/webrtc/modules/audio_coding/codecs/isac/main/interface/audio_encoder_isac.h index 8c7053359d..1fe5d312b8 100644 --- a/webrtc/modules/audio_coding/codecs/isac/main/interface/audio_encoder_isac.h +++ b/webrtc/modules/audio_coding/codecs/isac/main/interface/audio_encoder_isac.h @@ -68,6 +68,10 @@ struct IsacFloat { static inline int16_t Free(instance_type* inst) { return WebRtcIsac_Free(inst); } + static inline void GetBandwidthInfo(instance_type* inst, + IsacBandwidthInfo* bwinfo) { + WebRtcIsac_GetBandwidthInfo(inst, bwinfo); + } static inline int16_t GetErrorCode(instance_type* inst) { return WebRtcIsac_GetErrorCode(inst); } @@ -75,7 +79,10 @@ struct IsacFloat { static inline int16_t GetNewFrameLen(instance_type* inst) { return WebRtcIsac_GetNewFrameLen(inst); } - + static inline void SetBandwidthInfo(instance_type* inst, + const IsacBandwidthInfo* bwinfo) { + WebRtcIsac_SetBandwidthInfo(inst, bwinfo); + } static inline int16_t SetDecSampRate(instance_type* inst, uint16_t sample_rate_hz) { return WebRtcIsac_SetDecSampRate(inst, sample_rate_hz); diff --git a/webrtc/modules/audio_coding/codecs/isac/main/interface/isac.h b/webrtc/modules/audio_coding/codecs/isac/main/interface/isac.h index 1a83d722b4..1fe11bcef0 100644 --- a/webrtc/modules/audio_coding/codecs/isac/main/interface/isac.h +++ b/webrtc/modules/audio_coding/codecs/isac/main/interface/isac.h @@ -11,9 +11,7 @@ #ifndef WEBRTC_MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_INTERFACE_ISAC_H_ #define WEBRTC_MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_INTERFACE_ISAC_H_ -/* - * Define the fixed-point numeric formats - */ +#include "webrtc/modules/audio_coding/codecs/isac/bandwidth_info.h" #include "webrtc/typedefs.h" typedef struct WebRtcISACStruct ISACStruct; @@ -708,6 +706,12 @@ extern "C" { int16_t* decoded, int16_t* speechType); + /* Fills in an IsacBandwidthInfo struct. */ + void WebRtcIsac_GetBandwidthInfo(ISACStruct* inst, IsacBandwidthInfo* bwinfo); + + /* Uses the values from an IsacBandwidthInfo struct. */ + void WebRtcIsac_SetBandwidthInfo(ISACStruct* inst, + const IsacBandwidthInfo* bwinfo); #if defined(__cplusplus) } diff --git a/webrtc/modules/audio_coding/codecs/isac/main/source/bandwidth_estimator.c b/webrtc/modules/audio_coding/codecs/isac/main/source/bandwidth_estimator.c index ce8ceb217a..940e8f50c7 100644 --- a/webrtc/modules/audio_coding/codecs/isac/main/source/bandwidth_estimator.c +++ b/webrtc/modules/audio_coding/codecs/isac/main/source/bandwidth_estimator.c @@ -20,7 +20,9 @@ #include "settings.h" #include "isac.h" +#include #include +#include /* array of quantization levels for bottle neck info; Matlab code: */ /* sprintf('%4.1ff, ', logspace(log10(5000), log10(40000), 12)) */ @@ -119,6 +121,9 @@ int32_t WebRtcIsac_InitBandwidthEstimator( bwest_str->inWaitLatePkts = 0; bwest_str->senderTimestamp = 0; bwest_str->receiverTimestamp = 0; + + bwest_str->external_bw_info.in_use = 0; + return 0; } @@ -154,6 +159,7 @@ int16_t WebRtcIsac_UpdateBandwidthEstimator( int immediate_set = 0; int num_pkts_expected; + assert(!bwest_str->external_bw_info.in_use); // We have to adjust the header-rate if the first packet has a // frame-size different than the initialized value. @@ -508,6 +514,8 @@ int16_t WebRtcIsac_UpdateUplinkBwImpl( int16_t index, enum IsacSamplingRate encoderSamplingFreq) { + assert(!bwest_str->external_bw_info.in_use); + if((index < 0) || (index > 23)) { return -ISAC_RANGE_ERROR_BW_ESTIMATOR; @@ -564,6 +572,8 @@ int16_t WebRtcIsac_UpdateUplinkJitter( BwEstimatorstr* bwest_str, int32_t index) { + assert(!bwest_str->external_bw_info.in_use); + if((index < 0) || (index > 23)) { return -ISAC_RANGE_ERROR_BW_ESTIMATOR; @@ -589,7 +599,7 @@ int16_t WebRtcIsac_UpdateUplinkJitter( // Returns the bandwidth/jitter estimation code (integer 0...23) // to put in the sending iSAC payload -uint16_t +void WebRtcIsac_GetDownlinkBwJitIndexImpl( BwEstimatorstr* bwest_str, int16_t* bottleneckIndex, @@ -609,6 +619,12 @@ WebRtcIsac_GetDownlinkBwJitIndexImpl( int16_t maxInd; int16_t midInd; + if (bwest_str->external_bw_info.in_use) { + *bottleneckIndex = bwest_str->external_bw_info.bottleneck_idx; + *jitterInfo = bwest_str->external_bw_info.jitter_info; + return; + } + /* Get Max Delay Bit */ /* get unquantized max delay */ MaxDelay = (float)WebRtcIsac_GetDownlinkMaxDelay(bwest_str); @@ -684,8 +700,6 @@ WebRtcIsac_GetDownlinkBwJitIndexImpl( bwest_str->rec_bw_avg = (1 - weight) * bwest_str->rec_bw_avg + weight * (rate + bwest_str->rec_header_rate); - - return 0; } @@ -697,6 +711,8 @@ int32_t WebRtcIsac_GetDownlinkBandwidth( const BwEstimatorstr *bwest_str) float jitter_sign; float bw_adjust; + assert(!bwest_str->external_bw_info.in_use); + /* create a value between -1.0 and 1.0 indicating "average sign" of jitter */ jitter_sign = bwest_str->rec_jitter_short_term / bwest_str->rec_jitter_short_term_abs; @@ -725,6 +741,8 @@ WebRtcIsac_GetDownlinkMaxDelay(const BwEstimatorstr *bwest_str) { int32_t rec_max_delay; + assert(!bwest_str->external_bw_info.in_use); + rec_max_delay = (int32_t)(bwest_str->rec_max_delay); /* limit range of jitter estimate */ @@ -739,48 +757,41 @@ WebRtcIsac_GetDownlinkMaxDelay(const BwEstimatorstr *bwest_str) return rec_max_delay; } -/* get the bottle neck rate from here to far side, as estimated by far side */ -void -WebRtcIsac_GetUplinkBandwidth( - const BwEstimatorstr* bwest_str, - int32_t* bitRate) -{ - /* limit range of bottle neck rate */ - if (bwest_str->send_bw_avg < MIN_ISAC_BW) - { - *bitRate = MIN_ISAC_BW; - } - else if (bwest_str->send_bw_avg > MAX_ISAC_BW) - { - *bitRate = MAX_ISAC_BW; - } - else - { - *bitRate = (int32_t)(bwest_str->send_bw_avg); - } - return; +/* Clamp val to the closed interval [min,max]. */ +static int32_t clamp(int32_t val, int32_t min, int32_t max) { + assert(min <= max); + return val < min ? min : (val > max ? max : val); } -/* Returns the max delay value from the other side in ms */ -int32_t -WebRtcIsac_GetUplinkMaxDelay(const BwEstimatorstr *bwest_str) -{ - int32_t send_max_delay; - - send_max_delay = (int32_t)(bwest_str->send_max_delay_avg); - - /* limit range of jitter estimate */ - if (send_max_delay < MIN_ISAC_MD) - { - send_max_delay = MIN_ISAC_MD; - } - else if (send_max_delay > MAX_ISAC_MD) - { - send_max_delay = MAX_ISAC_MD; - } - return send_max_delay; +int32_t WebRtcIsac_GetUplinkBandwidth(const BwEstimatorstr* bwest_str) { + return bwest_str->external_bw_info.in_use + ? bwest_str->external_bw_info.send_bw_avg + : clamp(bwest_str->send_bw_avg, MIN_ISAC_BW, MAX_ISAC_BW); } +int32_t WebRtcIsac_GetUplinkMaxDelay(const BwEstimatorstr* bwest_str) { + return bwest_str->external_bw_info.in_use + ? bwest_str->external_bw_info.send_max_delay_avg + : clamp(bwest_str->send_max_delay_avg, MIN_ISAC_MD, MAX_ISAC_MD); +} + +void WebRtcIsacBw_GetBandwidthInfo(BwEstimatorstr* bwest_str, + enum IsacSamplingRate decoder_sample_rate_hz, + IsacBandwidthInfo* bwinfo) { + assert(!bwest_str->external_bw_info.in_use); + bwinfo->in_use = 1; + bwinfo->send_bw_avg = WebRtcIsac_GetUplinkBandwidth(bwest_str); + bwinfo->send_max_delay_avg = WebRtcIsac_GetUplinkMaxDelay(bwest_str); + WebRtcIsac_GetDownlinkBwJitIndexImpl(bwest_str, &bwinfo->bottleneck_idx, + &bwinfo->jitter_info, + decoder_sample_rate_hz); +} + +void WebRtcIsacBw_SetBandwidthInfo(BwEstimatorstr* bwest_str, + const IsacBandwidthInfo* bwinfo) { + memcpy(&bwest_str->external_bw_info, bwinfo, + sizeof bwest_str->external_bw_info); +} /* * update long-term average bitrate and amount of data in buffer diff --git a/webrtc/modules/audio_coding/codecs/isac/main/source/bandwidth_estimator.h b/webrtc/modules/audio_coding/codecs/isac/main/source/bandwidth_estimator.h index 8482a8c70e..2916876082 100644 --- a/webrtc/modules/audio_coding/codecs/isac/main/source/bandwidth_estimator.h +++ b/webrtc/modules/audio_coding/codecs/isac/main/source/bandwidth_estimator.h @@ -104,10 +104,10 @@ extern "C" { enum IsacSamplingRate encoderSamplingFreq); /* Returns the bandwidth/jitter estimation code (integer 0...23) to put in the sending iSAC payload */ - uint16_t WebRtcIsac_GetDownlinkBwJitIndexImpl( - BwEstimatorstr* bwest_str, - int16_t* bottleneckIndex, - int16_t* jitterInfo, + void WebRtcIsac_GetDownlinkBwJitIndexImpl( + BwEstimatorstr* bwest_str, + int16_t* bottleneckIndex, + int16_t* jitterInfo, enum IsacSamplingRate decoderSamplingFreq); /* Returns the bandwidth estimation (in bps) */ @@ -119,14 +119,21 @@ extern "C" { const BwEstimatorstr *bwest_str); /* Returns the bandwidth that iSAC should send with in bps */ - void WebRtcIsac_GetUplinkBandwidth( - const BwEstimatorstr* bwest_str, - int32_t* bitRate); + int32_t WebRtcIsac_GetUplinkBandwidth(const BwEstimatorstr* bwest_str); /* Returns the max delay value from the other side in ms */ int32_t WebRtcIsac_GetUplinkMaxDelay( const BwEstimatorstr *bwest_str); + /* Fills in an IsacExternalBandwidthInfo struct. */ + void WebRtcIsacBw_GetBandwidthInfo( + BwEstimatorstr* bwest_str, + enum IsacSamplingRate decoder_sample_rate_hz, + IsacBandwidthInfo* bwinfo); + + /* Uses the values from an IsacExternalBandwidthInfo struct. */ + void WebRtcIsacBw_SetBandwidthInfo(BwEstimatorstr* bwest_str, + const IsacBandwidthInfo* bwinfo); /* * update amount of data in bottle neck buffer and burst handling diff --git a/webrtc/modules/audio_coding/codecs/isac/main/source/isac.c b/webrtc/modules/audio_coding/codecs/isac/main/source/isac.c index d278c7fd8b..a19fd01167 100644 --- a/webrtc/modules/audio_coding/codecs/isac/main/source/isac.c +++ b/webrtc/modules/audio_coding/codecs/isac/main/source/isac.c @@ -17,6 +17,7 @@ #include "webrtc/modules/audio_coding/codecs/isac/main/interface/isac.h" +#include #include #include #include @@ -113,9 +114,8 @@ static void UpdateBottleneck(ISACMainStruct* instISAC) { if ((instISAC->codingMode == 0) && (instISAC->instLB.ISACencLB_obj.buffer_index == 0) && (instISAC->instLB.ISACencLB_obj.frame_nb == 0)) { - int32_t bottleneck; - WebRtcIsac_GetUplinkBandwidth(&(instISAC->bwestimator_obj), - &bottleneck); + int32_t bottleneck = + WebRtcIsac_GetUplinkBandwidth(&instISAC->bwestimator_obj); /* Adding hysteresis when increasing signal bandwidth. */ if ((instISAC->bandwidthKHz == isac8kHz) @@ -670,7 +670,7 @@ int WebRtcIsac_Encode(ISACStruct* ISAC_main_inst, } /* Add Garbage if required. */ - WebRtcIsac_GetUplinkBandwidth(&instISAC->bwestimator_obj, &bottleneck); + bottleneck = WebRtcIsac_GetUplinkBandwidth(&instISAC->bwestimator_obj); if (instISAC->codingMode == 0) { int minBytes; int limit; @@ -2384,3 +2384,18 @@ uint16_t WebRtcIsac_DecSampRate(ISACStruct* ISAC_main_inst) { ISACMainStruct* instISAC = (ISACMainStruct*)ISAC_main_inst; return instISAC->decoderSamplingRateKHz == kIsacWideband ? 16000 : 32000; } + +void WebRtcIsac_GetBandwidthInfo(ISACStruct* inst, + IsacBandwidthInfo* bwinfo) { + ISACMainStruct* instISAC = (ISACMainStruct*)inst; + assert(instISAC->initFlag & BIT_MASK_DEC_INIT); + WebRtcIsacBw_GetBandwidthInfo(&instISAC->bwestimator_obj, + instISAC->decoderSamplingRateKHz, bwinfo); +} + +void WebRtcIsac_SetBandwidthInfo(ISACStruct* inst, + const IsacBandwidthInfo* bwinfo) { + ISACMainStruct* instISAC = (ISACMainStruct*)inst; + assert(instISAC->initFlag & BIT_MASK_ENC_INIT); + WebRtcIsacBw_SetBandwidthInfo(&instISAC->bwestimator_obj, bwinfo); +} diff --git a/webrtc/modules/audio_coding/codecs/isac/main/source/structs.h b/webrtc/modules/audio_coding/codecs/isac/main/source/structs.h index c4062d753d..84428788d7 100644 --- a/webrtc/modules/audio_coding/codecs/isac/main/source/structs.h +++ b/webrtc/modules/audio_coding/codecs/isac/main/source/structs.h @@ -18,6 +18,7 @@ #ifndef WEBRTC_MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_STRUCTS_H_ #define WEBRTC_MODULES_AUDIO_CODING_CODECS_ISAC_MAIN_SOURCE_STRUCTS_H_ +#include "webrtc/modules/audio_coding/codecs/isac/bandwidth_info.h" #include "webrtc/modules/audio_coding/codecs/isac/main/interface/isac.h" #include "webrtc/modules/audio_coding/codecs/isac/main/source/settings.h" #include "webrtc/typedefs.h" @@ -223,6 +224,8 @@ typedef struct { uint16_t numConsecLatePkts; float consecLatency; int16_t inWaitLatePkts; + + IsacBandwidthInfo external_bw_info; } BwEstimatorstr; diff --git a/webrtc/modules/audio_coding/codecs/isac/unittest.cc b/webrtc/modules/audio_coding/codecs/isac/unittest.cc new file mode 100644 index 0000000000..a80fd08bcf --- /dev/null +++ b/webrtc/modules/audio_coding/codecs/isac/unittest.cc @@ -0,0 +1,271 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#include +#include + +#include "testing/gtest/include/gtest/gtest.h" +#include "webrtc/base/buffer.h" +#include "webrtc/modules/audio_coding/codecs/isac/fix/interface/audio_encoder_isacfix.h" +#include "webrtc/modules/audio_coding/codecs/isac/main/interface/audio_encoder_isac.h" +#include "webrtc/modules/audio_coding/neteq/tools/input_audio_file.h" +#include "webrtc/test/testsupport/fileutils.h" + +namespace webrtc { + +namespace { + +std::vector LoadSpeechData() { + webrtc::test::InputAudioFile input_file( + webrtc::test::ResourcePath("audio_coding/testfile32kHz", "pcm")); + static const int kIsacNumberOfSamples = 32 * 60; // 60 ms at 32 kHz + std::vector speech_data(kIsacNumberOfSamples); + input_file.Read(kIsacNumberOfSamples, speech_data.data()); + return speech_data; +} + +template +IsacBandwidthInfo GetBwInfo(typename T::instance_type* inst) { + IsacBandwidthInfo bi; + T::GetBandwidthInfo(inst, &bi); + EXPECT_TRUE(bi.in_use); + return bi; +} + +template +rtc::Buffer EncodePacket(typename T::instance_type* inst, + const IsacBandwidthInfo* bi, + const int16_t* speech_data, + int framesize_ms) { + rtc::Buffer output(1000); + for (int i = 0;; ++i) { + if (bi) + T::SetBandwidthInfo(inst, bi); + int encoded_bytes = T::Encode(inst, speech_data, output.data()); + if (i + 1 == framesize_ms / 10) { + EXPECT_GT(encoded_bytes, 0); + EXPECT_LE(static_cast(encoded_bytes), output.size()); + output.SetSize(encoded_bytes); + return output; + } + EXPECT_EQ(0, encoded_bytes); + } +} + +class BoundedCapacityChannel final { + public: + BoundedCapacityChannel(int rate_bits_per_second) + : current_time_rtp_(0), + channel_rate_bytes_per_sample_(rate_bits_per_second / + (8.0 * kSamplesPerSecond)) {} + + // Simulate sending the given number of bytes at the given RTP time. Returns + // the new current RTP time after the sending is done. + int Send(int send_time_rtp, int nbytes) { + current_time_rtp_ = std::max(current_time_rtp_, send_time_rtp) + + nbytes / channel_rate_bytes_per_sample_; + return current_time_rtp_; + } + + private: + int current_time_rtp_; + // The somewhat strange unit for channel rate, bytes per sample, is because + // RTP time is measured in samples: + const double channel_rate_bytes_per_sample_; + static const int kSamplesPerSecond = 16000; +}; + +template +struct TestParam {}; + +template <> +struct TestParam { + static const int time_to_settle = 200; + static int ExpectedRateBitsPerSecond(int rate_bits_per_second) { + return rate_bits_per_second; + } +}; + +template <> +struct TestParam { + static const int time_to_settle = 350; + static int ExpectedRateBitsPerSecond(int rate_bits_per_second) { + // For some reason, IsacFix fails to adapt to the channel's actual + // bandwidth. Instead, it settles on a few hundred packets at 10kbit/s, + // then a few hundred at 5kbit/s, then a few hundred at 10kbit/s, and so + // on. The 200 packets starting at 350 are in the middle of the first + // 10kbit/s run. + return 10000; + } +}; + +template <> +struct TestParam { + static const int time_to_settle = 0; + static int ExpectedRateBitsPerSecond(int rate_bits_per_second) { + return 32000; + } +}; + +template <> +struct TestParam { + static const int time_to_settle = 0; + static int ExpectedRateBitsPerSecond(int rate_bits_per_second) { + return 16000; + } +}; + +// Test that the iSAC encoder produces identical output whether or not we use a +// conjoined encoder+decoder pair or a separate encoder and decoder that +// communicate BW estimation info explicitly. +template +void TestGetSetBandwidthInfo(const int16_t* speech_data, + int rate_bits_per_second) { + using Param = TestParam; + const int framesize_ms = adaptive ? 60 : 30; + + // Conjoined encoder/decoder pair: + typename T::instance_type* encdec; + ASSERT_EQ(0, T::Create(&encdec)); + ASSERT_EQ(0, T::EncoderInit(encdec, adaptive ? 0 : 1)); + ASSERT_EQ(0, T::DecoderInit(encdec)); + + // Disjoint encoder/decoder pair: + typename T::instance_type* enc; + ASSERT_EQ(0, T::Create(&enc)); + ASSERT_EQ(0, T::EncoderInit(enc, adaptive ? 0 : 1)); + typename T::instance_type* dec; + ASSERT_EQ(0, T::Create(&dec)); + ASSERT_EQ(0, T::DecoderInit(dec)); + + // 0. Get initial BW info from decoder. + auto bi = GetBwInfo(dec); + + BoundedCapacityChannel channel1(rate_bits_per_second), + channel2(rate_bits_per_second); + std::vector packet_sizes; + for (int i = 0; i < Param::time_to_settle + 200; ++i) { + std::ostringstream ss; + ss << " i = " << i; + SCOPED_TRACE(ss.str()); + + // 1. Encode 6 * 10 ms (adaptive) or 3 * 10 ms (nonadaptive). The separate + // encoder is given the BW info before each encode call. + auto bitstream1 = + EncodePacket(encdec, nullptr, speech_data, framesize_ms); + auto bitstream2 = EncodePacket(enc, &bi, speech_data, framesize_ms); + EXPECT_EQ(bitstream1, bitstream2); + if (i > Param::time_to_settle) + packet_sizes.push_back(bitstream1.size()); + + // 2. Deliver the encoded data to the decoders (but don't actually ask them + // to decode it; that's not necessary). Then get new BW info from the + // separate decoder. + const int samples_per_packet = 16 * framesize_ms; + const int send_time = i * samples_per_packet; + EXPECT_EQ(0, T::UpdateBwEstimate( + encdec, bitstream1.data(), bitstream1.size(), i, send_time, + channel1.Send(send_time, bitstream1.size()))); + EXPECT_EQ(0, T::UpdateBwEstimate( + dec, bitstream2.data(), bitstream2.size(), i, send_time, + channel2.Send(send_time, bitstream2.size()))); + bi = GetBwInfo(dec); + } + + EXPECT_EQ(0, T::Free(encdec)); + EXPECT_EQ(0, T::Free(enc)); + EXPECT_EQ(0, T::Free(dec)); + + // The average send bitrate is close to the channel's capacity. + double avg_size = + std::accumulate(packet_sizes.begin(), packet_sizes.end(), 0) / + static_cast(packet_sizes.size()); + double avg_rate_bits_per_second = 8.0 * avg_size / (framesize_ms * 1e-3); + double expected_rate_bits_per_second = + Param::ExpectedRateBitsPerSecond(rate_bits_per_second); + EXPECT_GT(avg_rate_bits_per_second / expected_rate_bits_per_second, 0.95); + EXPECT_LT(avg_rate_bits_per_second / expected_rate_bits_per_second, 1.06); + + // The largest packet isn't that large, and the smallest not that small. + size_t min_size = *std::min_element(packet_sizes.begin(), packet_sizes.end()); + size_t max_size = *std::max_element(packet_sizes.begin(), packet_sizes.end()); + double size_range = max_size - min_size; + EXPECT_LE(size_range / avg_size, 0.16); +} + +} // namespace + +TEST(IsacCommonTest, GetSetBandwidthInfoFloat12kAdaptive) { + TestGetSetBandwidthInfo(LoadSpeechData().data(), 12000); +} + +TEST(IsacCommonTest, GetSetBandwidthInfoFloat15kAdaptive) { + TestGetSetBandwidthInfo(LoadSpeechData().data(), 15000); +} + +TEST(IsacCommonTest, GetSetBandwidthInfoFloat19kAdaptive) { + TestGetSetBandwidthInfo(LoadSpeechData().data(), 19000); +} + +TEST(IsacCommonTest, GetSetBandwidthInfoFloat22kAdaptive) { + TestGetSetBandwidthInfo(LoadSpeechData().data(), 22000); +} + +TEST(IsacCommonTest, GetSetBandwidthInfoFix12kAdaptive) { + TestGetSetBandwidthInfo(LoadSpeechData().data(), 12000); +} + +TEST(IsacCommonTest, GetSetBandwidthInfoFix15kAdaptive) { + TestGetSetBandwidthInfo(LoadSpeechData().data(), 15000); +} + +TEST(IsacCommonTest, GetSetBandwidthInfoFix19kAdaptive) { + TestGetSetBandwidthInfo(LoadSpeechData().data(), 19000); +} + +TEST(IsacCommonTest, GetSetBandwidthInfoFix22kAdaptive) { + TestGetSetBandwidthInfo(LoadSpeechData().data(), 22000); +} + +TEST(IsacCommonTest, GetSetBandwidthInfoFloat12k) { + TestGetSetBandwidthInfo(LoadSpeechData().data(), 12000); +} + +TEST(IsacCommonTest, GetSetBandwidthInfoFloat15k) { + TestGetSetBandwidthInfo(LoadSpeechData().data(), 15000); +} + +TEST(IsacCommonTest, GetSetBandwidthInfoFloat19k) { + TestGetSetBandwidthInfo(LoadSpeechData().data(), 19000); +} + +TEST(IsacCommonTest, GetSetBandwidthInfoFloat22k) { + TestGetSetBandwidthInfo(LoadSpeechData().data(), 22000); +} + +TEST(IsacCommonTest, GetSetBandwidthInfoFix12k) { + TestGetSetBandwidthInfo(LoadSpeechData().data(), 12000); +} + +TEST(IsacCommonTest, GetSetBandwidthInfoFix15k) { + TestGetSetBandwidthInfo(LoadSpeechData().data(), 15000); +} + +TEST(IsacCommonTest, GetSetBandwidthInfoFix19k) { + TestGetSetBandwidthInfo(LoadSpeechData().data(), 19000); +} + +TEST(IsacCommonTest, GetSetBandwidthInfoFix22k) { + TestGetSetBandwidthInfo(LoadSpeechData().data(), 22000); +} + +} // namespace webrtc diff --git a/webrtc/modules/modules.gyp b/webrtc/modules/modules.gyp index e44cfcc3c0..b06ecc5685 100644 --- a/webrtc/modules/modules.gyp +++ b/webrtc/modules/modules.gyp @@ -113,6 +113,7 @@ 'audio_coding/codecs/isac/fix/source/transform_unittest.cc', 'audio_coding/codecs/isac/main/source/audio_encoder_isac_unittest.cc', 'audio_coding/codecs/isac/main/source/isac_unittest.cc', + 'audio_coding/codecs/isac/unittest.cc', 'audio_coding/codecs/opus/audio_encoder_opus_unittest.cc', 'audio_coding/codecs/opus/opus_unittest.cc', 'audio_coding/codecs/red/audio_encoder_copy_red_unittest.cc',