Adds trial to calculate audio overhead based on available data.

This adds the ability to disable legacy overhead calculation so we'll
use the available data on per packet over head and frame length range
to set the min and max total  allocatable bitrate.

Bug: webrtc:11001
Change-Id: I2a94499433e15bad11a08f81fe7f1dfc27982cdf
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/155175
Reviewed-by: Oskar Sundbom <ossu@webrtc.org>
Commit-Queue: Sebastian Jansson <srte@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#29368}

api/audio_codecs/BUILD.gn

@@ -37,6 +37,7 @@ rtc_source_set("audio_codecs_api") {
     "../../rtc_base:rtc_base_approved",
     "../../rtc_base:sanitizer",
     "../../rtc_base/system:rtc_export",
+    "../units:time_delta",
     "//third_party/abseil-cpp/absl/strings",
     "//third_party/abseil-cpp/absl/types:optional",
   ]

api/audio_codecs/audio_encoder.cc

@@ -108,4 +108,9 @@ ANAStats AudioEncoder::GetANAStats() const {
   return ANAStats();
 }
 
+absl::optional<std::pair<TimeDelta, TimeDelta>>
+AudioEncoder::GetFrameLengthRange() const {
+  return absl::nullopt;
+}
+
 }  // namespace webrtc

api/audio_codecs/audio_encoder.h

@@ -13,11 +13,13 @@
 
 #include <memory>
 #include <string>
+#include <utility>
 #include <vector>
 
 #include "absl/types/optional.h"
 #include "api/array_view.h"
 #include "api/call/bitrate_allocation.h"
+#include "api/units/time_delta.h"
 #include "rtc_base/buffer.h"
 #include "rtc_base/deprecation.h"
 
@@ -241,6 +243,12 @@ class AudioEncoder {
   // Get statistics related to audio network adaptation.
   virtual ANAStats GetANAStats() const;
 
+  // The range of frame lengths that are supported or nullopt if there's no sch
+  // information. This is used to calculated the full bitrate range, including
+  // overhead.
+  virtual absl::optional<std::pair<TimeDelta, TimeDelta>> GetFrameLengthRange()
+      const;
+
  protected:
   // Subclasses implement this to perform the actual encoding. Called by
   // Encode().

audio/audio_send_stream.cc

@@ -133,6 +133,8 @@ AudioSendStream::AudioSendStream(
       audio_state_(audio_state),
       channel_send_(std::move(channel_send)),
       event_log_(event_log),
+      use_legacy_overhead_calculation_(
+          !field_trial::IsDisabled("WebRTC-Audio-LegacyOverhead")),
       bitrate_allocator_(bitrate_allocator),
       rtp_transport_(rtp_transport),
       packet_loss_tracker_(kPacketLossTrackerMaxWindowSizeMs,
@@ -481,6 +483,7 @@ void AudioSendStream::DeliverRtcp(const uint8_t* packet, size_t length) {
 }
 
 uint32_t AudioSendStream::OnBitrateUpdated(BitrateAllocationUpdate update) {
+  RTC_DCHECK_RUN_ON(worker_queue_);
   // Pick a target bitrate between the constraints. Overrules the allocator if
   // it 1) allocated a bitrate of zero to disable the stream or 2) allocated a
   // higher than max to allow for e.g. extra FEC.
@@ -666,6 +669,11 @@ bool AudioSendStream::SetupSendCodec(AudioSendStream* stream,
       encoder->OnReceivedOverhead(stream->GetPerPacketOverheadBytes());
     }
   }
+  stream->worker_queue_->PostTask(
+      [stream, length_range = encoder->GetFrameLengthRange()] {
+        RTC_DCHECK_RUN_ON(stream->worker_queue_);
+        stream->frame_length_range_ = length_range;
+      });
 
   stream->StoreEncoderProperties(encoder->SampleRateHz(),
                                  encoder->NumChannels());
@@ -872,20 +880,25 @@ AudioSendStream::GetMinMaxBitrateConstraints() const {
   if (allocation_settings_.MaxBitrate())
     constraints.max = *allocation_settings_.MaxBitrate();
 
-  RTC_DCHECK_GE(constraints.min.bps(), 0);
+  RTC_DCHECK_GE(constraints.min, DataRate::Zero());
-  RTC_DCHECK_GE(constraints.max.bps(), 0);
+  RTC_DCHECK_GE(constraints.max, DataRate::Zero());
-  RTC_DCHECK_GE(constraints.max.bps(), constraints.min.bps());
+  RTC_DCHECK_GE(constraints.max, constraints.min);
-
-  // TODO(srte,dklee): Replace these with proper overhead calculations.
   if (allocation_settings_.IncludeOverheadInAudioAllocation()) {
+    if (use_legacy_overhead_calculation_) {
       // OverheadPerPacket = Ipv4(20B) + UDP(8B) + SRTP(10B) + RTP(12)
       const DataSize kOverheadPerPacket = DataSize::bytes(20 + 8 + 10 + 12);
-    const TimeDelta kMaxFrameLength = TimeDelta::ms(60);  // Based on Opus spec
+      const TimeDelta kMaxFrameLength =
+          TimeDelta::ms(60);  // Based on Opus spec
       const DataRate kMinOverhead = kOverheadPerPacket / kMaxFrameLength;
       constraints.min += kMinOverhead;
-    // TODO(dklee): This is obviously overly conservative to avoid exceeding max
-    // bitrate. Carefully reconsider the logic when addressing todo above.
       constraints.max += kMinOverhead;
+    } else {
+      RTC_DCHECK(frame_length_range_);
+      const DataSize kOverheadPerPacket =
+          DataSize::bytes(total_packet_overhead_bytes_);
+      constraints.min += kOverheadPerPacket / frame_length_range_->second;
+      constraints.max += kOverheadPerPacket / frame_length_range_->first;
+    }
   }
   return constraints;
 }

audio/audio_send_stream.h

@@ -12,6 +12,7 @@
 #define AUDIO_AUDIO_SEND_STREAM_H_
 
 #include <memory>
+#include <utility>
 #include <vector>
 
 #include "audio/audio_level.h"
@@ -133,7 +134,8 @@ class AudioSendStream final : public webrtc::AudioSendStream,
 
   // Returns bitrate constraints, maybe including overhead when enabled by
   // field trial.
-  TargetAudioBitrateConstraints GetMinMaxBitrateConstraints() const;
+  TargetAudioBitrateConstraints GetMinMaxBitrateConstraints() const
+      RTC_RUN_ON(worker_queue_);
 
   // Sets per-packet overhead on encoded (for ANA) based on current known values
   // of transport and packetization overheads.
@@ -157,6 +159,7 @@ class AudioSendStream final : public webrtc::AudioSendStream,
   rtc::scoped_refptr<webrtc::AudioState> audio_state_;
   const std::unique_ptr<voe::ChannelSendInterface> channel_send_;
   RtcEventLog* const event_log_;
+  const bool use_legacy_overhead_calculation_;
 
   int encoder_sample_rate_hz_ = 0;
   size_t encoder_num_channels_ = 0;
@@ -204,6 +207,8 @@ class AudioSendStream final : public webrtc::AudioSendStream,
 
   bool registered_with_allocator_ RTC_GUARDED_BY(worker_queue_) = false;
   size_t total_packet_overhead_bytes_ RTC_GUARDED_BY(worker_queue_) = 0;
+  absl::optional<std::pair<TimeDelta, TimeDelta>> frame_length_range_
+      RTC_GUARDED_BY(worker_queue_);
 
   RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(AudioSendStream);
 };

audio/audio_send_stream_unittest.cc

@@ -83,8 +83,10 @@ const AudioCodecSpec kCodecSpecs[] = {
 // should be made more precise in the future. This can be changed when that
 // logic is more accurate.
 const DataSize kOverheadPerPacket = DataSize::bytes(20 + 8 + 10 + 12);
-const TimeDelta kMaxFrameLength = TimeDelta::ms(60);
+const TimeDelta kMinFrameLength = TimeDelta::ms(20);
-const DataRate kOverheadRate = kOverheadPerPacket / kMaxFrameLength;
+const TimeDelta kMaxFrameLength = TimeDelta::ms(120);
+const DataRate kMinOverheadRate = kOverheadPerPacket / kMaxFrameLength;
+const DataRate kMaxOverheadRate = kOverheadPerPacket / kMinFrameLength;
 
 class MockLimitObserver : public BitrateAllocator::LimitObserver {
  public:
@@ -104,6 +106,9 @@ std::unique_ptr<MockAudioEncoder> SetupAudioEncoderMock(
           .WillByDefault(Return(spec.info.num_channels));
       ON_CALL(*encoder.get(), RtpTimestampRateHz())
           .WillByDefault(Return(spec.format.clockrate_hz));
+      ON_CALL(*encoder.get(), GetFrameLengthRange())
+          .WillByDefault(Return(absl::optional<std::pair<TimeDelta, TimeDelta>>{
+              {TimeDelta::ms(20), TimeDelta::ms(120)}}));
       return encoder;
     }
   }
@@ -299,6 +304,7 @@ struct ConfigHelper {
     EXPECT_CALL(*audio_processing_, GetStatistics(true))
         .WillRepeatedly(Return(audio_processing_stats_));
   }
+  TaskQueueForTest* worker() { return &worker_queue_; }
 
  private:
   SimulatedClock clock_;
@@ -546,7 +552,7 @@ TEST(AudioSendStreamTest, DoesNotPassHigherBitrateThanMaxBitrate) {
   update.packet_loss_ratio = 0;
   update.round_trip_time = TimeDelta::ms(50);
   update.bwe_period = TimeDelta::ms(6000);
-  send_stream->OnBitrateUpdated(update);
+  helper.worker()->SendTask([&] { send_stream->OnBitrateUpdated(update); });
 }
 
 TEST(AudioSendStreamTest, SSBweTargetInRangeRespected) {
@@ -559,7 +565,7 @@ TEST(AudioSendStreamTest, SSBweTargetInRangeRespected) {
                   Eq(DataRate::bps(helper.config().max_bitrate_bps - 5000)))));
   BitrateAllocationUpdate update;
   update.target_bitrate = DataRate::bps(helper.config().max_bitrate_bps - 5000);
-  send_stream->OnBitrateUpdated(update);
+  helper.worker()->SendTask([&] { send_stream->OnBitrateUpdated(update); });
 }
 
 TEST(AudioSendStreamTest, SSBweFieldTrialMinRespected) {
@@ -574,7 +580,7 @@ TEST(AudioSendStreamTest, SSBweFieldTrialMinRespected) {
                                 Eq(DataRate::kbps(6)))));
   BitrateAllocationUpdate update;
   update.target_bitrate = DataRate::kbps(1);
-  send_stream->OnBitrateUpdated(update);
+  helper.worker()->SendTask([&] { send_stream->OnBitrateUpdated(update); });
 }
 
 TEST(AudioSendStreamTest, SSBweFieldTrialMaxRespected) {
@@ -589,55 +595,64 @@ TEST(AudioSendStreamTest, SSBweFieldTrialMaxRespected) {
                                 Eq(DataRate::kbps(64)))));
   BitrateAllocationUpdate update;
   update.target_bitrate = DataRate::kbps(128);
-  send_stream->OnBitrateUpdated(update);
+  helper.worker()->SendTask([&] { send_stream->OnBitrateUpdated(update); });
 }
 
 TEST(AudioSendStreamTest, SSBweWithOverhead) {
   ScopedFieldTrials field_trials(
       "WebRTC-Audio-SendSideBwe/Enabled/"
-      "WebRTC-SendSideBwe-WithOverhead/Enabled/");
+      "WebRTC-SendSideBwe-WithOverhead/Enabled/"
+      "WebRTC-Audio-LegacyOverhead/Disabled/");
   ConfigHelper helper(true, true);
   auto send_stream = helper.CreateAudioSendStream();
+  EXPECT_CALL(*helper.channel_send(), CallEncoder(_)).Times(1);
+  send_stream->OnOverheadChanged(kOverheadPerPacket.bytes<size_t>());
   const DataRate bitrate =
-      DataRate::bps(helper.config().max_bitrate_bps) + kOverheadRate;
+      DataRate::bps(helper.config().max_bitrate_bps) + kMaxOverheadRate;
   EXPECT_CALL(*helper.channel_send(),
               OnBitrateAllocation(Field(
                   &BitrateAllocationUpdate::target_bitrate, Eq(bitrate))));
   BitrateAllocationUpdate update;
   update.target_bitrate = bitrate;
-  send_stream->OnBitrateUpdated(update);
+  helper.worker()->SendTask([&] { send_stream->OnBitrateUpdated(update); });
 }
 
 TEST(AudioSendStreamTest, SSBweWithOverheadMinRespected) {
   ScopedFieldTrials field_trials(
       "WebRTC-Audio-SendSideBwe/Enabled/"
       "WebRTC-SendSideBwe-WithOverhead/Enabled/"
+      "WebRTC-Audio-LegacyOverhead/Disabled/"
       "WebRTC-Audio-Allocation/min:6kbps,max:64kbps/");
   ConfigHelper helper(true, true);
   auto send_stream = helper.CreateAudioSendStream();
-  const DataRate bitrate = DataRate::kbps(6) + kOverheadRate;
+  EXPECT_CALL(*helper.channel_send(), CallEncoder(_)).Times(1);
+  send_stream->OnOverheadChanged(kOverheadPerPacket.bytes<size_t>());
+  const DataRate bitrate = DataRate::kbps(6) + kMinOverheadRate;
   EXPECT_CALL(*helper.channel_send(),
               OnBitrateAllocation(Field(
                   &BitrateAllocationUpdate::target_bitrate, Eq(bitrate))));
   BitrateAllocationUpdate update;
   update.target_bitrate = DataRate::kbps(1);
-  send_stream->OnBitrateUpdated(update);
+  helper.worker()->SendTask([&] { send_stream->OnBitrateUpdated(update); });
 }
 
 TEST(AudioSendStreamTest, SSBweWithOverheadMaxRespected) {
   ScopedFieldTrials field_trials(
       "WebRTC-Audio-SendSideBwe/Enabled/"
       "WebRTC-SendSideBwe-WithOverhead/Enabled/"
+      "WebRTC-Audio-LegacyOverhead/Disabled/"
       "WebRTC-Audio-Allocation/min:6kbps,max:64kbps/");
   ConfigHelper helper(true, true);
   auto send_stream = helper.CreateAudioSendStream();
-  const DataRate bitrate = DataRate::kbps(64) + kOverheadRate;
+  EXPECT_CALL(*helper.channel_send(), CallEncoder(_)).Times(1);
+  send_stream->OnOverheadChanged(kOverheadPerPacket.bytes<size_t>());
+  const DataRate bitrate = DataRate::kbps(64) + kMaxOverheadRate;
   EXPECT_CALL(*helper.channel_send(),
               OnBitrateAllocation(Field(
                   &BitrateAllocationUpdate::target_bitrate, Eq(bitrate))));
   BitrateAllocationUpdate update;
   update.target_bitrate = DataRate::kbps(128);
-  send_stream->OnBitrateUpdated(update);
+  helper.worker()->SendTask([&] { send_stream->OnBitrateUpdated(update); });
 }
 
 TEST(AudioSendStreamTest, ProbingIntervalOnBitrateUpdated) {
@@ -652,7 +667,7 @@ TEST(AudioSendStreamTest, ProbingIntervalOnBitrateUpdated) {
   update.packet_loss_ratio = 0;
   update.round_trip_time = TimeDelta::ms(50);
   update.bwe_period = TimeDelta::ms(5000);
-  send_stream->OnBitrateUpdated(update);
+  helper.worker()->SendTask([&] { send_stream->OnBitrateUpdated(update); });
 }
 
 // Test that AudioSendStream doesn't recreate the encoder unnecessarily.

modules/audio_coding/codecs/opus/audio_encoder_opus.cc

@@ -879,4 +879,17 @@ ANAStats AudioEncoderOpusImpl::GetANAStats() const {
   return ANAStats();
 }
 
+absl::optional<std::pair<TimeDelta, TimeDelta> >
+AudioEncoderOpusImpl::GetFrameLengthRange() const {
+  if (config_.supported_frame_lengths_ms.empty()) {
+    return absl::nullopt;
+  } else if (audio_network_adaptor_) {
+    return {{TimeDelta::ms(config_.supported_frame_lengths_ms.front()),
+             TimeDelta::ms(config_.supported_frame_lengths_ms.back())}};
+  } else {
+    return {{TimeDelta::ms(config_.frame_size_ms),
+             TimeDelta::ms(config_.frame_size_ms)}};
+  }
+}
+
 }  // namespace webrtc

modules/audio_coding/codecs/opus/audio_encoder_opus.h

@@ -115,6 +115,8 @@ class AudioEncoderOpusImpl final : public AudioEncoder {
   void SetReceiverFrameLengthRange(int min_frame_length_ms,
                                    int max_frame_length_ms) override;
   ANAStats GetANAStats() const override;
+  absl::optional<std::pair<TimeDelta, TimeDelta> > GetFrameLengthRange()
+      const override;
   rtc::ArrayView<const int> supported_frame_lengths_ms() const {
     return config_.supported_frame_lengths_ms;
   }

test/mock_audio_encoder.h

@@ -34,6 +34,9 @@ class MockAudioEncoder : public AudioEncoder {
   MOCK_CONST_METHOD0(Num10MsFramesInNextPacket, size_t());
   MOCK_CONST_METHOD0(Max10MsFramesInAPacket, size_t());
   MOCK_CONST_METHOD0(GetTargetBitrate, int());
+  MOCK_CONST_METHOD0(GetFrameLengthRange,
+                     absl::optional<std::pair<TimeDelta, TimeDelta>>());
+
   MOCK_METHOD0(Reset, void());
   MOCK_METHOD1(SetFec, bool(bool enable));
   MOCK_METHOD1(SetDtx, bool(bool enable));