diff --git a/modules/audio_coding/neteq/decision_logic_unittest.cc b/modules/audio_coding/neteq/decision_logic_unittest.cc index c29350e80b..fc580350d2 100644 --- a/modules/audio_coding/neteq/decision_logic_unittest.cc +++ b/modules/audio_coding/neteq/decision_logic_unittest.cc @@ -64,7 +64,8 @@ class DecisionLogicTest : public ::testing::Test { std::unique_ptr histogram = std::make_unique(200, 12345, 2); auto delay_manager = std::make_unique( - 200, 0, 12300, config.tick_timer, std::move(histogram)); + 200, 0, 12300, absl::nullopt, 2000, config.tick_timer, + std::move(histogram)); mock_delay_manager_ = delay_manager.get(); auto buffer_level_filter = std::make_unique(); mock_buffer_level_filter_ = buffer_level_filter.get(); diff --git a/modules/audio_coding/neteq/delay_manager.cc b/modules/audio_coding/neteq/delay_manager.cc index 6b10d33a5b..33eeb96f19 100644 --- a/modules/audio_coding/neteq/delay_manager.cc +++ b/modules/audio_coding/neteq/delay_manager.cc @@ -22,6 +22,7 @@ #include "modules/audio_coding/neteq/histogram.h" #include "modules/include/module_common_types_public.h" #include "rtc_base/checks.h" +#include "rtc_base/experiments/struct_parameters_parser.h" #include "rtc_base/logging.h" #include "rtc_base/numerics/safe_conversions.h" #include "rtc_base/numerics/safe_minmax.h" @@ -32,29 +33,34 @@ namespace { constexpr int kMinBaseMinimumDelayMs = 0; constexpr int kMaxBaseMinimumDelayMs = 10000; -constexpr int kMaxHistoryMs = 2000; // Oldest packet to include in history to - // calculate relative packet arrival delay. constexpr int kDelayBuckets = 100; constexpr int kBucketSizeMs = 20; constexpr int kStartDelayMs = 80; constexpr int kMaxNumReorderedPackets = 5; -int PercentileToQuantile(double percentile) { - return static_cast((1 << 30) * percentile / 100.0 + 0.5); -} - -struct DelayHistogramConfig { - int quantile = 1041529569; // 0.97 in Q30. - int forget_factor = 32745; // 0.9993 in Q15. +struct DelayManagerConfig { + double quantile = 0.97; + double forget_factor = 0.9993; absl::optional start_forget_weight = 2; -}; + absl::optional resample_interval_ms; + int max_history_ms = 2000; -// TODO(jakobi): Remove legacy field trial. -DelayHistogramConfig GetDelayHistogramConfig() { - constexpr char kDelayHistogramFieldTrial[] = - "WebRTC-Audio-NetEqDelayHistogram"; - DelayHistogramConfig config; - if (webrtc::field_trial::IsEnabled(kDelayHistogramFieldTrial)) { + std::unique_ptr Parser() { + return webrtc::StructParametersParser::Create( // + "quantile", &quantile, // + "forget_factor", &forget_factor, // + "start_forget_weight", &start_forget_weight, // + "resample_interval_ms", &resample_interval_ms, // + "max_history_ms", &max_history_ms); + } + + // TODO(jakobi): remove legacy field trial. + void MaybeUpdateFromLegacyFieldTrial() { + constexpr char kDelayHistogramFieldTrial[] = + "WebRTC-Audio-NetEqDelayHistogram"; + if (!webrtc::field_trial::IsEnabled(kDelayHistogramFieldTrial)) { + return; + } const auto field_trial_string = webrtc::field_trial::FindFullName(kDelayHistogramFieldTrial); double percentile = -1.0; @@ -64,27 +70,36 @@ DelayHistogramConfig GetDelayHistogramConfig() { &forget_factor, &start_forget_weight) >= 2 && percentile >= 0.0 && percentile <= 100.0 && forget_factor >= 0.0 && forget_factor <= 1.0) { - config.quantile = PercentileToQuantile(percentile); - config.forget_factor = (1 << 15) * forget_factor; - config.start_forget_weight = - start_forget_weight >= 1 ? absl::make_optional(start_forget_weight) - : absl::nullopt; + this->quantile = percentile / 100; + this->forget_factor = forget_factor; + this->start_forget_weight = start_forget_weight >= 1 + ? absl::make_optional(start_forget_weight) + : absl::nullopt; } } - RTC_LOG(LS_INFO) << "Delay histogram config:" - " quantile=" - << config.quantile - << " forget_factor=" << config.forget_factor - << " start_forget_weight=" - << config.start_forget_weight.value_or(0); - return config; -} + + explicit DelayManagerConfig() { + Parser()->Parse(webrtc::field_trial::FindFullName( + "WebRTC-Audio-NetEqDelayManagerConfig")); + MaybeUpdateFromLegacyFieldTrial(); + RTC_LOG(LS_INFO) << "Delay manager config:" + " quantile=" + << quantile << " forget_factor=" << forget_factor + << " start_forget_weight=" + << start_forget_weight.value_or(0) + << " resample_interval_ms=" + << resample_interval_ms.value_or(0) + << " max_history_ms=" << max_history_ms; + } +}; } // namespace DelayManager::DelayManager(int max_packets_in_buffer, int base_minimum_delay_ms, int histogram_quantile, + absl::optional resample_interval_ms, + int max_history_ms, const TickTimer* tick_timer, std::unique_ptr histogram) : first_packet_received_(false), @@ -92,6 +107,8 @@ DelayManager::DelayManager(int max_packets_in_buffer, histogram_(std::move(histogram)), histogram_quantile_(histogram_quantile), tick_timer_(tick_timer), + resample_interval_ms_(resample_interval_ms), + max_history_ms_(max_history_ms), base_minimum_delay_ms_(base_minimum_delay_ms), effective_minimum_delay_ms_(base_minimum_delay_ms), minimum_delay_ms_(0), @@ -108,12 +125,15 @@ std::unique_ptr DelayManager::Create( int max_packets_in_buffer, int base_minimum_delay_ms, const TickTimer* tick_timer) { - auto config = GetDelayHistogramConfig(); + DelayManagerConfig config; + int forget_factor_q15 = (1 << 15) * config.forget_factor; + int quantile_q30 = (1 << 30) * config.quantile; std::unique_ptr histogram = std::make_unique( - kDelayBuckets, config.forget_factor, config.start_forget_weight); - return std::make_unique(max_packets_in_buffer, - base_minimum_delay_ms, config.quantile, - tick_timer, std::move(histogram)); + kDelayBuckets, forget_factor_q15, config.start_forget_weight); + return std::make_unique( + max_packets_in_buffer, base_minimum_delay_ms, quantile_q30, + config.resample_interval_ms, config.max_history_ms, tick_timer, + std::move(histogram)); } DelayManager::~DelayManager() {} @@ -132,6 +152,8 @@ absl::optional DelayManager::Update(uint32_t timestamp, last_timestamp_ = timestamp; first_packet_received_ = true; num_reordered_packets_ = 0; + resample_stopwatch_ = tick_timer_->GetNewStopwatch(); + max_delay_in_interval_ms_ = 0; return absl::nullopt; } @@ -139,7 +161,7 @@ absl::optional DelayManager::Update(uint32_t timestamp, 1000 * static_cast(timestamp - last_timestamp_) / sample_rate_hz; const int iat_ms = packet_iat_stopwatch_->ElapsedMs(); const int iat_delay_ms = iat_ms - expected_iat_ms; - absl::optional relative_delay; + int relative_delay; bool reordered = !IsNewerTimestamp(timestamp, last_timestamp_); if (reordered) { relative_delay = std::max(iat_delay_ms, 0); @@ -147,11 +169,28 @@ absl::optional DelayManager::Update(uint32_t timestamp, UpdateDelayHistory(iat_delay_ms, timestamp, sample_rate_hz); relative_delay = CalculateRelativePacketArrivalDelay(); } - const int index = relative_delay.value() / kBucketSizeMs; - if (index < histogram_->NumBuckets()) { - // Maximum delay to register is 2000 ms. - histogram_->Add(index); + + absl::optional histogram_update; + if (resample_interval_ms_) { + if (static_cast(resample_stopwatch_->ElapsedMs()) > + *resample_interval_ms_) { + histogram_update = max_delay_in_interval_ms_; + resample_stopwatch_ = tick_timer_->GetNewStopwatch(); + max_delay_in_interval_ms_ = 0; + } + max_delay_in_interval_ms_ = + std::max(max_delay_in_interval_ms_, relative_delay); + } else { + histogram_update = relative_delay; } + if (histogram_update) { + const int index = *histogram_update / kBucketSizeMs; + if (index < histogram_->NumBuckets()) { + // Maximum delay to register is 2000 ms. + histogram_->Add(index); + } + } + // Calculate new |target_level_ms_| based on updated statistics. int bucket_index = histogram_->Quantile(histogram_quantile_); target_level_ms_ = (1 + bucket_index) * kBucketSizeMs; @@ -191,7 +230,7 @@ void DelayManager::UpdateDelayHistory(int iat_delay_ms, delay.timestamp = timestamp; delay_history_.push_back(delay); while (timestamp - delay_history_.front().timestamp > - static_cast(kMaxHistoryMs * sample_rate_hz / 1000)) { + static_cast(max_history_ms_ * sample_rate_hz / 1000)) { delay_history_.pop_front(); } } @@ -226,6 +265,8 @@ void DelayManager::Reset() { packet_iat_stopwatch_ = tick_timer_->GetNewStopwatch(); first_packet_received_ = false; num_reordered_packets_ = 0; + resample_stopwatch_ = tick_timer_->GetNewStopwatch(); + max_delay_in_interval_ms_ = 0; } int DelayManager::TargetDelayMs() const { diff --git a/modules/audio_coding/neteq/delay_manager.h b/modules/audio_coding/neteq/delay_manager.h index d5cea45296..9832ceda22 100644 --- a/modules/audio_coding/neteq/delay_manager.h +++ b/modules/audio_coding/neteq/delay_manager.h @@ -28,6 +28,8 @@ class DelayManager { DelayManager(int max_packets_in_buffer, int base_minimum_delay_ms, int histogram_quantile, + absl::optional resample_interval_ms, + int max_history_ms, const TickTimer* tick_timer, std::unique_ptr histogram); @@ -105,6 +107,9 @@ class DelayManager { std::unique_ptr histogram_; const int histogram_quantile_; const TickTimer* tick_timer_; + const absl::optional resample_interval_ms_; + const int max_history_ms_; + int base_minimum_delay_ms_; int effective_minimum_delay_ms_; // Used as lower bound for target delay. int minimum_delay_ms_; // Externally set minimum delay. @@ -116,6 +121,8 @@ class DelayManager { int target_level_ms_; // Currently preferred buffer level. uint32_t last_timestamp_; // Timestamp for the last received packet. int num_reordered_packets_ = 0; + int max_delay_in_interval_ms_ = 0; + std::unique_ptr resample_stopwatch_; struct PacketDelay { int iat_delay_ms; diff --git a/modules/audio_coding/neteq/delay_manager_unittest.cc b/modules/audio_coding/neteq/delay_manager_unittest.cc index a6a38b692e..9678151dfa 100644 --- a/modules/audio_coding/neteq/delay_manager_unittest.cc +++ b/modules/audio_coding/neteq/delay_manager_unittest.cc @@ -16,6 +16,7 @@ #include +#include "absl/types/optional.h" #include "modules/audio_coding/neteq/histogram.h" #include "modules/audio_coding/neteq/mock/mock_histogram.h" #include "modules/audio_coding/neteq/mock/mock_statistics_calculator.h" @@ -29,6 +30,7 @@ namespace webrtc { namespace { constexpr int kMaxNumberOfPackets = 240; constexpr int kMinDelayMs = 0; +constexpr int kMaxHistoryMs = 2000; constexpr int kTimeStepMs = 10; constexpr int kFs = 8000; constexpr int kFrameSizeMs = 20; @@ -53,6 +55,7 @@ class DelayManagerTest : public ::testing::Test { MockHistogram* mock_histogram_; uint32_t ts_; bool use_mock_histogram_ = false; + absl::optional resample_interval_ms_; }; DelayManagerTest::DelayManagerTest() @@ -69,6 +72,7 @@ void DelayManagerTest::RecreateDelayManager() { std::unique_ptr histogram(mock_histogram_); dm_ = std::make_unique(kMaxNumberOfPackets, kMinDelayMs, kDefaultHistogramQuantile, + resample_interval_ms_, kMaxHistoryMs, &tick_timer_, std::move(histogram)); } else { dm_ = DelayManager::Create(kMaxNumberOfPackets, kMinDelayMs, &tick_timer_); @@ -455,4 +459,31 @@ TEST_F(DelayManagerTest, RelativeArrivalDelayStatistic) { EXPECT_EQ(20, InsertNextPacket()); } +TEST_F(DelayManagerTest, ResamplePacketDelays) { + use_mock_histogram_ = true; + resample_interval_ms_ = 500; + RecreateDelayManager(); + + // The histogram should be updated once with the maximum delay observed for + // the following sequence of packets. + EXPECT_CALL(*mock_histogram_, Add(5)).Times(1); + + EXPECT_EQ(absl::nullopt, InsertNextPacket()); + + IncreaseTime(kFrameSizeMs); + EXPECT_EQ(0, InsertNextPacket()); + IncreaseTime(3 * kFrameSizeMs); + EXPECT_EQ(2 * kFrameSizeMs, InsertNextPacket()); + IncreaseTime(4 * kFrameSizeMs); + EXPECT_EQ(5 * kFrameSizeMs, InsertNextPacket()); + + for (int i = 4; i >= 0; --i) { + EXPECT_EQ(i * kFrameSizeMs, InsertNextPacket()); + } + for (int i = 0; i < *resample_interval_ms_ / kFrameSizeMs; ++i) { + IncreaseTime(kFrameSizeMs); + EXPECT_EQ(0, InsertNextPacket()); + } +} + } // namespace webrtc diff --git a/modules/audio_coding/neteq/mock/mock_delay_manager.h b/modules/audio_coding/neteq/mock/mock_delay_manager.h index 0631f6fcc9..5b5133eceb 100644 --- a/modules/audio_coding/neteq/mock/mock_delay_manager.h +++ b/modules/audio_coding/neteq/mock/mock_delay_manager.h @@ -25,11 +25,15 @@ class MockDelayManager : public DelayManager { MockDelayManager(size_t max_packets_in_buffer, int base_minimum_delay_ms, int histogram_quantile, + absl::optional resample_interval_ms, + int max_history_ms, const TickTimer* tick_timer, std::unique_ptr histogram) : DelayManager(max_packets_in_buffer, base_minimum_delay_ms, histogram_quantile, + resample_interval_ms, + max_history_ms, tick_timer, std::move(histogram)) {} MOCK_METHOD(int, TargetDelayMs, (), (const));