diff --git a/webrtc/modules/audio_coding/codecs/audio_encoder.cc b/webrtc/modules/audio_coding/codecs/audio_encoder.cc index 16b470b8b7..7b7325dc55 100644 --- a/webrtc/modules/audio_coding/codecs/audio_encoder.cc +++ b/webrtc/modules/audio_coding/codecs/audio_encoder.cc @@ -81,6 +81,8 @@ void AudioEncoder::OnReceivedTargetAudioBitrate(int target_audio_bitrate_bps) {} void AudioEncoder::OnReceivedRtt(int rtt_ms) {} +void AudioEncoder::OnReceivedOverhead(size_t overhead_bytes_per_packet) {} + void AudioEncoder::SetReceiverFrameLengthRange(int min_frame_length_ms, int max_frame_length_ms) {} diff --git a/webrtc/modules/audio_coding/codecs/audio_encoder.h b/webrtc/modules/audio_coding/codecs/audio_encoder.h index c913765fe4..5132c2ed8b 100644 --- a/webrtc/modules/audio_coding/codecs/audio_encoder.h +++ b/webrtc/modules/audio_coding/codecs/audio_encoder.h @@ -181,6 +181,10 @@ class AudioEncoder { // Provides RTT to this encoder to allow it to adapt. virtual void OnReceivedRtt(int rtt_ms); + // Provides overhead to this encoder to adapt. The overhead is the number of + // bytes that will be added to each packet the encoder generates. + virtual void OnReceivedOverhead(size_t overhead_bytes_per_packet); + // To allow encoder to adapt its frame length, it must be provided the frame // length range that receivers can accept. virtual void SetReceiverFrameLengthRange(int min_frame_length_ms, diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc index 40ad595e5c..af66cd3634 100644 --- a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc +++ b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc @@ -15,12 +15,14 @@ #include "webrtc/base/analytics/exp_filter.h" #include "webrtc/base/checks.h" +#include "webrtc/base/logging.h" #include "webrtc/base/safe_conversions.h" #include "webrtc/common_types.h" #include "webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h" #include "webrtc/modules/audio_coding/audio_network_adaptor/controller_manager.h" #include "webrtc/modules/audio_coding/codecs/opus/opus_interface.h" #include "webrtc/system_wrappers/include/clock.h" +#include "webrtc/system_wrappers/include/field_trial.h" namespace webrtc { @@ -291,10 +293,25 @@ void AudioEncoderOpus::OnReceivedUplinkPacketLossFraction( void AudioEncoderOpus::OnReceivedTargetAudioBitrate( int target_audio_bitrate_bps) { - if (!audio_network_adaptor_) - return SetTargetBitrate(target_audio_bitrate_bps); - audio_network_adaptor_->SetTargetAudioBitrate(target_audio_bitrate_bps); - ApplyAudioNetworkAdaptor(); + if (audio_network_adaptor_) { + audio_network_adaptor_->SetTargetAudioBitrate(target_audio_bitrate_bps); + ApplyAudioNetworkAdaptor(); + } else if (webrtc::field_trial::FindFullName( + "WebRTC-SendSideBwe-WithOverhead") == "Enabled") { + if (!overhead_bytes_per_packet_) { + LOG(LS_INFO) + << "AudioEncoderOpus: Overhead unknown, target audio bitrate " + << target_audio_bitrate_bps << " bps is ignored."; + return; + } + const int overhead_bps = static_cast( + *overhead_bytes_per_packet_ * 8 * 100 / Num10MsFramesInNextPacket()); + SetTargetBitrate(std::min( + kMaxBitrateBps, + std::max(kMinBitrateBps, target_audio_bitrate_bps - overhead_bps))); + } else { + SetTargetBitrate(target_audio_bitrate_bps); + } } void AudioEncoderOpus::OnReceivedRtt(int rtt_ms) { @@ -304,6 +321,16 @@ void AudioEncoderOpus::OnReceivedRtt(int rtt_ms) { ApplyAudioNetworkAdaptor(); } +void AudioEncoderOpus::OnReceivedOverhead(size_t overhead_bytes_per_packet) { + if (audio_network_adaptor_) { + audio_network_adaptor_->SetOverhead(overhead_bytes_per_packet); + ApplyAudioNetworkAdaptor(); + } else { + overhead_bytes_per_packet_ = + rtc::Optional(overhead_bytes_per_packet); + } +} + void AudioEncoderOpus::SetReceiverFrameLengthRange(int min_frame_length_ms, int max_frame_length_ms) { // Ensure that |SetReceiverFrameLengthRange| is called before diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h index 92f2126b98..0864c7bfb5 100644 --- a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h +++ b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h @@ -110,6 +110,7 @@ class AudioEncoderOpus final : public AudioEncoder { float uplink_packet_loss_fraction) override; void OnReceivedTargetAudioBitrate(int target_audio_bitrate_bps) override; void OnReceivedRtt(int rtt_ms) override; + void OnReceivedOverhead(size_t overhead_bytes_per_packet) override; void SetReceiverFrameLengthRange(int min_frame_length_ms, int max_frame_length_ms) override; rtc::ArrayView supported_frame_lengths_ms() const { @@ -159,6 +160,7 @@ class AudioEncoderOpus final : public AudioEncoder { std::unique_ptr packet_loss_fraction_smoother_; AudioNetworkAdaptorCreator audio_network_adaptor_creator_; std::unique_ptr audio_network_adaptor_; + rtc::Optional overhead_bytes_per_packet_; RTC_DISALLOW_COPY_AND_ASSIGN(AudioEncoderOpus); }; diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc index f07279fac7..9f2c3186bd 100644 --- a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc +++ b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc @@ -14,6 +14,7 @@ #include "webrtc/common_types.h" #include "webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h" #include "webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h" +#include "webrtc/test/field_trial.h" #include "webrtc/test/gmock.h" #include "webrtc/test/gtest.h" #include "webrtc/system_wrappers/include/clock.h" @@ -249,7 +250,7 @@ TEST(AudioEncoderOpusTest, SetReceiverFrameLengthRange) { EXPECT_THAT(states.encoder->supported_frame_lengths_ms(), ElementsAre(20)); } -TEST(AudioEncoderOpusTest, InvokeAudioNetworkAdaptorOnSetUplinkBandwidth) { +TEST(AudioEncoderOpusTest, InvokeAudioNetworkAdaptorOnReceivedUplinkBandwidth) { auto states = CreateCodec(2); states.encoder->EnableAudioNetworkAdaptor("", nullptr); @@ -267,7 +268,7 @@ TEST(AudioEncoderOpusTest, InvokeAudioNetworkAdaptorOnSetUplinkBandwidth) { } TEST(AudioEncoderOpusTest, - InvokeAudioNetworkAdaptorOnSetUplinkPacketLossFraction) { + InvokeAudioNetworkAdaptorOnReceivedUplinkPacketLossFraction) { auto states = CreateCodec(2); states.encoder->EnableAudioNetworkAdaptor("", nullptr); @@ -284,7 +285,8 @@ TEST(AudioEncoderOpusTest, CheckEncoderRuntimeConfig(states.encoder.get(), config); } -TEST(AudioEncoderOpusTest, InvokeAudioNetworkAdaptorOnSetTargetAudioBitrate) { +TEST(AudioEncoderOpusTest, + InvokeAudioNetworkAdaptorOnReceivedTargetAudioBitrate) { auto states = CreateCodec(2); states.encoder->EnableAudioNetworkAdaptor("", nullptr); @@ -301,7 +303,7 @@ TEST(AudioEncoderOpusTest, InvokeAudioNetworkAdaptorOnSetTargetAudioBitrate) { CheckEncoderRuntimeConfig(states.encoder.get(), config); } -TEST(AudioEncoderOpusTest, InvokeAudioNetworkAdaptorOnSetRtt) { +TEST(AudioEncoderOpusTest, InvokeAudioNetworkAdaptorOnReceivedRtt) { auto states = CreateCodec(2); states.encoder->EnableAudioNetworkAdaptor("", nullptr); @@ -317,6 +319,22 @@ TEST(AudioEncoderOpusTest, InvokeAudioNetworkAdaptorOnSetRtt) { CheckEncoderRuntimeConfig(states.encoder.get(), config); } +TEST(AudioEncoderOpusTest, InvokeAudioNetworkAdaptorOnReceivedOverhead) { + auto states = CreateCodec(2); + states.encoder->EnableAudioNetworkAdaptor("", nullptr); + + auto config = CreateEncoderRuntimeConfig(); + EXPECT_CALL(**states.mock_audio_network_adaptor, GetEncoderRuntimeConfig()) + .WillOnce(Return(config)); + + // Since using mock audio network adaptor, any overhead is fine. + constexpr size_t kOverhead = 64; + EXPECT_CALL(**states.mock_audio_network_adaptor, SetOverhead(kOverhead)); + states.encoder->OnReceivedOverhead(kOverhead); + + CheckEncoderRuntimeConfig(states.encoder.get(), config); +} + TEST(AudioEncoderOpusTest, PacketLossFractionSmoothedOnSetUplinkPacketLossFraction) { auto states = CreateCodec(2); @@ -343,4 +361,64 @@ TEST(AudioEncoderOpusTest, EXPECT_FLOAT_EQ(0.05f, states.encoder->packet_loss_rate()); } +TEST(AudioEncoderOpusTest, DoNotInvokeSetTargetBitrateIfOverheadUnknown) { + test::ScopedFieldTrials override_field_trials( + "WebRTC-SendSideBwe-WithOverhead/Enabled/"); + + auto states = CreateCodec(2); + + states.encoder->OnReceivedTargetAudioBitrate(kDefaultOpusSettings.rate * 2); + + // Since |OnReceivedOverhead| has not been called, the codec bitrate should + // not change. + EXPECT_EQ(kDefaultOpusSettings.rate, states.encoder->GetTargetBitrate()); +} + +TEST(AudioEncoderOpusTest, OverheadRemovedFromTargetAudioBitrate) { + test::ScopedFieldTrials override_field_trials( + "WebRTC-SendSideBwe-WithOverhead/Enabled/"); + + auto states = CreateCodec(2); + + constexpr size_t kOverheadBytesPerPacket = 64; + states.encoder->OnReceivedOverhead(kOverheadBytesPerPacket); + + constexpr int kTargetBitrateBps = 40000; + states.encoder->OnReceivedTargetAudioBitrate(kTargetBitrateBps); + + int packet_rate = rtc::CheckedDivExact(48000, kDefaultOpusSettings.pacsize); + EXPECT_EQ(kTargetBitrateBps - + 8 * static_cast(kOverheadBytesPerPacket) * packet_rate, + states.encoder->GetTargetBitrate()); +} + +TEST(AudioEncoderOpusTest, BitrateBounded) { + test::ScopedFieldTrials override_field_trials( + "WebRTC-SendSideBwe-WithOverhead/Enabled/"); + + constexpr int kMinBitrateBps = 500; + constexpr int kMaxBitrateBps = 512000; + + auto states = CreateCodec(2); + + constexpr size_t kOverheadBytesPerPacket = 64; + states.encoder->OnReceivedOverhead(kOverheadBytesPerPacket); + + int packet_rate = rtc::CheckedDivExact(48000, kDefaultOpusSettings.pacsize); + + // Set a target rate that is smaller than |kMinBitrateBps| when overhead is + // subtracted. The eventual codec rate should be bounded by |kMinBitrateBps|. + int target_bitrate = + kOverheadBytesPerPacket * 8 * packet_rate + kMinBitrateBps - 1; + states.encoder->OnReceivedTargetAudioBitrate(target_bitrate); + EXPECT_EQ(kMinBitrateBps, states.encoder->GetTargetBitrate()); + + // Set a target rate that is greater than |kMaxBitrateBps| when overhead is + // subtracted. The eventual codec rate should be bounded by |kMaxBitrateBps|. + target_bitrate = + kOverheadBytesPerPacket * 8 * packet_rate + kMaxBitrateBps + 1; + states.encoder->OnReceivedTargetAudioBitrate(target_bitrate); + EXPECT_EQ(kMaxBitrateBps, states.encoder->GetTargetBitrate()); +} + } // namespace webrtc