Avoiding the noise pumping during DTX regions by just forwarding the refresh DTX packets that decrease the comfort noise level at the decoder.

Bug: webrtc:12380
Change-Id: I60e4684150cb4880224f402a9bf42a72811863b3
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/202920
Commit-Queue: Jesus de Vicente Pena <devicentepena@webrtc.org>
Reviewed-by: Minyue Li <minyue@webrtc.org>
Reviewed-by: Henrik Lundin <henrik.lundin@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#33174}

modules/audio_coding/codecs/opus/opus_interface.cc

@@ -12,6 +12,9 @@
 
 #include <cstdlib>
 
+#include <numeric>
+
+#include "api/array_view.h"
 #include "rtc_base/checks.h"
 #include "system_wrappers/include/field_trial.h"
 
@@ -36,6 +39,9 @@ enum {
 constexpr char kPlcUsePrevDecodedSamplesFieldTrial[] =
     "WebRTC-Audio-OpusPlcUsePrevDecodedSamples";
 
+constexpr char kAvoidNoisePumpingDuringDtxFieldTrial[] =
+    "WebRTC-Audio-OpusAvoidNoisePumpingDuringDtx";
+
 static int FrameSizePerChannel(int frame_size_ms, int sample_rate_hz) {
   RTC_DCHECK_GT(frame_size_ms, 0);
   RTC_DCHECK_EQ(frame_size_ms % 10, 0);
@@ -54,6 +60,46 @@ static int DefaultFrameSizePerChannel(int sample_rate_hz) {
   return FrameSizePerChannel(20, sample_rate_hz);
 }
 
+// Returns true if the `encoded` payload corresponds to a refresh DTX packet
+// whose energy is larger than the expected for non activity packets.
+static bool WebRtcOpus_IsHighEnergyRefreshDtxPacket(
+    OpusEncInst* inst,
+    rtc::ArrayView<const int16_t> frame,
+    rtc::ArrayView<const uint8_t> encoded) {
+  if (encoded.size() <= 2) {
+    return false;
+  }
+  int number_frames =
+      frame.size() / DefaultFrameSizePerChannel(inst->sample_rate_hz);
+  if (number_frames > 0 &&
+      WebRtcOpus_PacketHasVoiceActivity(encoded.data(), encoded.size()) == 0) {
+    const float average_frame_energy =
+        std::accumulate(frame.begin(), frame.end(), 0.0f,
+                        [](float a, int32_t b) { return a + b * b; }) /
+        number_frames;
+    if (WebRtcOpus_GetInDtx(inst) == 1 &&
+        average_frame_energy >= inst->smooth_energy_non_active_frames * 0.5f) {
+      // This is a refresh DTX packet as the encoder is in DTX and has
+      // produced a payload > 2 bytes. This refresh packet has a higher energy
+      // than the smooth energy of non activity frames (with a 3 dB negative
+      // margin) and, therefore, it is flagged as a high energy refresh DTX
+      // packet.
+      return true;
+    }
+    // The average energy is tracked in a similar way as the modeling of the
+    // comfort noise in the Silk decoder in Opus
+    // (third_party/opus/src/silk/CNG.c).
+    if (average_frame_energy < inst->smooth_energy_non_active_frames * 0.5f) {
+      inst->smooth_energy_non_active_frames = average_frame_energy;
+    } else {
+      inst->smooth_energy_non_active_frames +=
+          (average_frame_energy - inst->smooth_energy_non_active_frames) *
+          0.25f;
+    }
+  }
+  return false;
+}
+
 int16_t WebRtcOpus_EncoderCreate(OpusEncInst** inst,
                                  size_t channels,
                                  int32_t application,
@@ -88,6 +134,10 @@ int16_t WebRtcOpus_EncoderCreate(OpusEncInst** inst,
 
   state->in_dtx_mode = 0;
   state->channels = channels;
+  state->sample_rate_hz = sample_rate_hz;
+  state->smooth_energy_non_active_frames = 0.0f;
+  state->avoid_noise_pumping_during_dtx =
+      webrtc::field_trial::IsEnabled(kAvoidNoisePumpingDuringDtxFieldTrial);
 
   *inst = state;
   return 0;
@@ -120,9 +170,10 @@ int16_t WebRtcOpus_MultistreamEncoderCreate(
   RTC_DCHECK(state);
 
   int error;
-  state->multistream_encoder =
-      opus_multistream_encoder_create(48000, channels, streams, coupled_streams,
-                                      channel_mapping, opus_app, &error);
+  const int sample_rate_hz = 48000;
+  state->multistream_encoder = opus_multistream_encoder_create(
+      sample_rate_hz, channels, streams, coupled_streams, channel_mapping,
+      opus_app, &error);
 
   if (error != OPUS_OK || (!state->encoder && !state->multistream_encoder)) {
     WebRtcOpus_EncoderFree(state);
@@ -131,6 +182,9 @@ int16_t WebRtcOpus_MultistreamEncoderCreate(
 
   state->in_dtx_mode = 0;
   state->channels = channels;
+  state->sample_rate_hz = sample_rate_hz;
+  state->smooth_energy_non_active_frames = 0.0f;
+  state->avoid_noise_pumping_during_dtx = false;
 
   *inst = state;
   return 0;
@@ -188,6 +242,15 @@ int WebRtcOpus_Encode(OpusEncInst* inst,
     }
   }
 
+  if (inst->avoid_noise_pumping_during_dtx && WebRtcOpus_GetUseDtx(inst) == 1 &&
+      WebRtcOpus_IsHighEnergyRefreshDtxPacket(
+          inst, rtc::MakeArrayView(audio_in, samples),
+          rtc::MakeArrayView(encoded, res))) {
+    // This packet is a high energy refresh DTX packet. For avoiding an increase
+    // of the energy in the DTX region at the decoder, this packet is dropped.
+    inst->in_dtx_mode = 0;
+    return 0;
+  }
   inst->in_dtx_mode = 0;
   return res;
 }
@@ -316,6 +379,16 @@ int16_t WebRtcOpus_DisableDtx(OpusEncInst* inst) {
   }
 }
 
+int16_t WebRtcOpus_GetUseDtx(OpusEncInst* inst) {
+  if (inst) {
+    opus_int32 use_dtx;
+    if (ENCODER_CTL(inst, OPUS_GET_DTX(&use_dtx)) == 0) {
+      return use_dtx;
+    }
+  }
+  return -1;
+}
+
 int16_t WebRtcOpus_EnableCbr(OpusEncInst* inst) {
   if (inst) {
     return ENCODER_CTL(inst, OPUS_SET_VBR(0));