From f1efc5713907e019207fbf7522706321a8c4647f Mon Sep 17 00:00:00 2001 From: "turaj@webrtc.org" Date: Fri, 16 Aug 2013 23:44:24 +0000 Subject: [PATCH] Implementing APIs to set maximum and minimum for latency. cpplint warnning fixed Ready for review BUG= R=minyue@webrtc.org Review URL: https://webrtc-codereview.appspot.com/1971004 git-svn-id: http://webrtc.googlecode.com/svn/trunk@4563 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../audio_coding/neteq4/delay_manager.cc | 93 +++++++++++++------ .../audio_coding/neteq4/delay_manager.h | 13 ++- .../neteq4/delay_manager_unittest.cc | 82 ++++++++++++++-- .../audio_coding/neteq4/interface/neteq.h | 20 +++- .../modules/audio_coding/neteq4/neteq_impl.cc | 22 ++++- .../modules/audio_coding/neteq4/neteq_impl.h | 9 +- 6 files changed, 188 insertions(+), 51 deletions(-) diff --git a/webrtc/modules/audio_coding/neteq4/delay_manager.cc b/webrtc/modules/audio_coding/neteq4/delay_manager.cc index 736f60c048..ff65bb7ecc 100644 --- a/webrtc/modules/audio_coding/neteq4/delay_manager.cc +++ b/webrtc/modules/audio_coding/neteq4/delay_manager.cc @@ -34,7 +34,9 @@ DelayManager::DelayManager(int max_packets_in_buffer, streaming_mode_(false), last_seq_no_(0), last_timestamp_(0), - extra_delay_ms_(0), + minimum_delay_ms_(0), + least_required_delay_ms_(target_level_), + maximum_delay_ms_(target_level_), iat_cumulative_sum_(0), max_iat_cumulative_sum_(0), max_timer_ms_(0), @@ -218,21 +220,34 @@ void DelayManager::UpdateHistogram(size_t iat_packets) { iat_factor_ += (kIatFactor_ - iat_factor_ + 3) >> 2; } -// Enforces upper limit for |target_level_|. The limit is chosen to be -// 75% of |max_packets_in_buffer_|, to leave some headroom for natural -// fluctuations around the target. If an extra delay is requested, the -// cap is lowered even further. Note that in practice, this does not have -// any impact, since the target level is far below the buffer capacity in -// all reasonable cases. +// Enforces upper and lower limits for |target_level_|. The upper limit is +// chosen to be minimum of i) 75% of |max_packets_in_buffer_|, to leave some +// headroom for natural fluctuations around the target, and ii) equivalent of +// |maximum_delay_ms_| in packets. Note that in practice, if no +// |maximum_delay_ms_| is specified, this does not have any impact, since the +// target level is far below the buffer capacity in all reasonable cases. +// The lower limit is equivalent of |minimum_delay_ms_| in packets. We update +// |least_required_level_| while the above limits are applied. // TODO(hlundin): Move this check to the buffer logistics class. void DelayManager::LimitTargetLevel() { - int max_buffer_len = max_packets_in_buffer_; - if (extra_delay_ms_ > 0 && packet_len_ms_ > 0) { - max_buffer_len -= extra_delay_ms_ / packet_len_ms_; - max_buffer_len = std::max(max_buffer_len, 1); // Sanity check. + least_required_delay_ms_ = (target_level_ * packet_len_ms_) >> 8; + + if (packet_len_ms_ > 0 && minimum_delay_ms_ > 0) { + int minimum_delay_packet_q8 = (minimum_delay_ms_ << 8) / packet_len_ms_; + target_level_ = std::max(target_level_, minimum_delay_packet_q8); } - max_buffer_len = (3 * (max_buffer_len << 8)) / 4; // Shift to Q8, then 75%. - target_level_ = std::min(target_level_, max_buffer_len); + + if (maximum_delay_ms_ > 0 && packet_len_ms_ > 0) { + int maximum_delay_packet_q8 = (maximum_delay_ms_ << 8) / packet_len_ms_; + target_level_ = std::min(target_level_, maximum_delay_packet_q8); + } + + // Shift to Q8, then 75%.; + int max_buffer_packets_q8 = (3 * (max_packets_in_buffer_ << 8)) / 4; + target_level_ = std::min(target_level_, max_buffer_packets_q8); + + // Sanity check, at least 1 packet (in Q8). + target_level_ = std::max(target_level_, 1 << 8); } int DelayManager::CalculateTargetLevel(int iat_packets) { @@ -331,6 +346,9 @@ void DelayManager::UpdateCounters(int elapsed_time_ms) { void DelayManager::ResetPacketIatCount() { packet_iat_count_ms_ = 0; } +// Note that |low_limit| and |higher_limit| are not assigned to +// |minimum_delay_ms_| and |maximum_delay_ms_| defined by the client of this +// class. They are computed from |target_level_| and used for decision making. void DelayManager::BufferLimits(int* lower_limit, int* higher_limit) const { if (!lower_limit || !higher_limit) { LOG_F(LS_ERROR) << "NULL pointers supplied as input"; @@ -338,29 +356,20 @@ void DelayManager::BufferLimits(int* lower_limit, int* higher_limit) const { return; } - int extra_delay_packets_q8 = 0; int window_20ms = 0x7FFF; // Default large value for legacy bit-exactness. if (packet_len_ms_ > 0) { - extra_delay_packets_q8 = (extra_delay_ms_ << 8) / packet_len_ms_; window_20ms = (20 << 8) / packet_len_ms_; } - // |lower_limit| is 75% of |target_level_| + extra delay. + // |target_level_| is in Q8 already. - *lower_limit = (target_level_ * 3) / 4 + extra_delay_packets_q8; - // |higher_limit| is equal to |target_level_| + extra delay, but should at + *lower_limit = (target_level_ * 3) / 4; + // |higher_limit| is equal to |target_level_|, but should at // least be 20 ms higher than |lower_limit_|. - *higher_limit = std::max(target_level_ + extra_delay_packets_q8, - *lower_limit + window_20ms); + *higher_limit = std::max(target_level_, *lower_limit + window_20ms); } int DelayManager::TargetLevel() const { - if (packet_len_ms_ > 0) { - // Add |extra_delay_ms_| converted to packets in Q8. - return target_level_ + (extra_delay_ms_ << 8) / packet_len_ms_; - } else { - // Cannot convert |extra_delay_ms_|; simply return |target_level_|. - return target_level_; - } + return target_level_; } void DelayManager::LastDecoderType(NetEqDecoder decoder_type) { @@ -375,8 +384,34 @@ void DelayManager::LastDecoderType(NetEqDecoder decoder_type) { } } -void DelayManager::set_extra_delay_ms(int16_t delay) { - extra_delay_ms_ = delay; +bool DelayManager::SetMinimumDelay(int delay_ms) { + // Minimum delay shouldn't be more than maximum delay, if any maximum is set. + // Also, if possible check |delay| to less than 75% of + // |max_packets_in_buffer_|. + if ((maximum_delay_ms_ > 0 && delay_ms > maximum_delay_ms_) || + (packet_len_ms_ > 0 && + delay_ms > 3 * max_packets_in_buffer_ * packet_len_ms_ / 4)) { + return false; + } + minimum_delay_ms_ = delay_ms; + return true; +} + +bool DelayManager::SetMaximumDelay(int delay_ms) { + if (delay_ms == 0) { + // Zero input unsets the maximum delay. + maximum_delay_ms_ = 0; + return true; + } else if (delay_ms < minimum_delay_ms_ || delay_ms < packet_len_ms_) { + // Maximum delay shouldn't be less than minimum delay or less than a packet. + return false; + } + maximum_delay_ms_ = delay_ms; + return true; +} + +int DelayManager::least_required_delay_ms() const { + return least_required_delay_ms_; } int DelayManager::base_target_level() const { return base_target_level_; } diff --git a/webrtc/modules/audio_coding/neteq4/delay_manager.h b/webrtc/modules/audio_coding/neteq4/delay_manager.h index d46924f787..ed1e87b190 100644 --- a/webrtc/modules/audio_coding/neteq4/delay_manager.h +++ b/webrtc/modules/audio_coding/neteq4/delay_manager.h @@ -94,7 +94,10 @@ class DelayManager { virtual void LastDecoderType(NetEqDecoder decoder_type); // Accessors and mutators. - virtual void set_extra_delay_ms(int16_t delay); + // Assuming |delay| is in valid range. + virtual bool SetMinimumDelay(int delay_ms); + virtual bool SetMaximumDelay(int delay_ms); + virtual int least_required_delay_ms() const; virtual int base_target_level() const; virtual void set_streaming_mode(bool value); virtual int last_pack_cng_or_dtmf() const; @@ -135,13 +138,19 @@ class DelayManager { int packet_iat_count_ms_; // Milliseconds elapsed since last packet. int base_target_level_; // Currently preferred buffer level before peak // detection and streaming mode (Q0). + // TODO(turajs) change the comment according to the implementation of + // minimum-delay. int target_level_; // Currently preferred buffer level in (fractions) // of packets (Q8), before adding any extra delay. int packet_len_ms_; // Length of audio in each incoming packet [ms]. bool streaming_mode_; uint16_t last_seq_no_; // Sequence number for last received packet. uint32_t last_timestamp_; // Timestamp for the last received packet. - int extra_delay_ms_; // Externally set extra delay. + int minimum_delay_ms_; // Externally set minimum delay. + int least_required_delay_ms_; // Smallest preferred buffer level (same unit + // as |target_level_|), before applying + // |minimum_delay_ms_| and/or |maximum_delay_ms_|. + int maximum_delay_ms_; // Externally set maximum allowed delay. int iat_cumulative_sum_; // Cumulative sum of delta inter-arrival times. int max_iat_cumulative_sum_; // Max of |iat_cumulative_sum_|. int max_timer_ms_; // Time elapsed since maximum was observed. diff --git a/webrtc/modules/audio_coding/neteq4/delay_manager_unittest.cc b/webrtc/modules/audio_coding/neteq4/delay_manager_unittest.cc index 7c08f340c8..482a65c9a4 100644 --- a/webrtc/modules/audio_coding/neteq4/delay_manager_unittest.cc +++ b/webrtc/modules/audio_coding/neteq4/delay_manager_unittest.cc @@ -21,6 +21,7 @@ namespace webrtc { using ::testing::Return; +using ::testing::_; class DelayManagerTest : public ::testing::Test { protected: @@ -193,9 +194,7 @@ TEST_F(DelayManagerTest, UpdatePeakFound) { EXPECT_EQ(5 << 8, higher); } -TEST_F(DelayManagerTest, ExtraDelay) { - const int kExtraDelayMs = 200; - dm_->set_extra_delay_ms(kExtraDelayMs); +TEST_F(DelayManagerTest, TargetDelay) { SetPacketAudioLength(kFrameSizeMs); // First packet arrival. InsertNextPacket(); @@ -208,24 +207,91 @@ TEST_F(DelayManagerTest, ExtraDelay) { EXPECT_CALL(detector_, Update(1, 1)) .WillOnce(Return(false)); InsertNextPacket(); - const int kExpectedTarget = 1 + kExtraDelayMs / kFrameSizeMs; + const int kExpectedTarget = 1; EXPECT_EQ(kExpectedTarget << 8, dm_->TargetLevel()); // In Q8. EXPECT_EQ(1, dm_->base_target_level()); int lower, higher; dm_->BufferLimits(&lower, &higher); - // Expect |lower| to be 75% of base target level + extra delay, and |higher| - // to be target level + extra delay, but at least leave 20 ms headroom from - // lower. - EXPECT_EQ((1 << 8) * 3 / 4 + (kExtraDelayMs << 8) / kFrameSizeMs, lower); + // Expect |lower| to be 75% of base target level, and |higher| to be + // lower + 20 ms headroom. + EXPECT_EQ((1 << 8) * 3 / 4, lower); EXPECT_EQ(lower + (20 << 8) / kFrameSizeMs, higher); } +TEST_F(DelayManagerTest, MaxAndRequiredDelay) { + const int kExpectedTarget = 5; + const int kTimeIncrement = kExpectedTarget * kFrameSizeMs; + SetPacketAudioLength(kFrameSizeMs); + // First packet arrival. + InsertNextPacket(); + // Second packet arrival. + // Expect detector update method to be called once with inter-arrival time + // equal to |kExpectedTarget| packet. Return true to indicate peaks found. + EXPECT_CALL(detector_, Update(kExpectedTarget, _)) + .WillRepeatedly(Return(true)); + EXPECT_CALL(detector_, MaxPeakHeight()) + .WillRepeatedly(Return(kExpectedTarget)); + IncreaseTime(kTimeIncrement); + InsertNextPacket(); + + // No limit is set. + EXPECT_EQ(kExpectedTarget << 8, dm_->TargetLevel()); + + int kMaxDelayPackets = kExpectedTarget - 2; + int kMaxDelayMs = kMaxDelayPackets * kFrameSizeMs; + EXPECT_TRUE(dm_->SetMaximumDelay(kMaxDelayMs)); + IncreaseTime(kTimeIncrement); + InsertNextPacket(); + EXPECT_EQ(kExpectedTarget * kFrameSizeMs, dm_->least_required_delay_ms()); + EXPECT_EQ(kMaxDelayPackets << 8, dm_->TargetLevel()); + + // Target level at least should be one packet. + EXPECT_FALSE(dm_->SetMaximumDelay(kFrameSizeMs - 1)); +} + +TEST_F(DelayManagerTest, MinAndRequiredDelay) { + const int kExpectedTarget = 5; + const int kTimeIncrement = kExpectedTarget * kFrameSizeMs; + SetPacketAudioLength(kFrameSizeMs); + // First packet arrival. + InsertNextPacket(); + // Second packet arrival. + // Expect detector update method to be called once with inter-arrival time + // equal to |kExpectedTarget| packet. Return true to indicate peaks found. + EXPECT_CALL(detector_, Update(kExpectedTarget, _)) + .WillRepeatedly(Return(true)); + EXPECT_CALL(detector_, MaxPeakHeight()) + .WillRepeatedly(Return(kExpectedTarget)); + IncreaseTime(kTimeIncrement); + InsertNextPacket(); + + // No limit is applied. + EXPECT_EQ(kExpectedTarget << 8, dm_->TargetLevel()); + + int kMinDelayPackets = kExpectedTarget + 2; + int kMinDelayMs = kMinDelayPackets * kFrameSizeMs; + dm_->SetMinimumDelay(kMinDelayMs); + IncreaseTime(kTimeIncrement); + InsertNextPacket(); + EXPECT_EQ(kExpectedTarget * kFrameSizeMs, dm_->least_required_delay_ms()); + EXPECT_EQ(kMinDelayPackets << 8, dm_->TargetLevel()); +} + TEST_F(DelayManagerTest, Failures) { // Wrong sample rate. EXPECT_EQ(-1, dm_->Update(0, 0, -1)); // Wrong packet size. EXPECT_EQ(-1, dm_->SetPacketAudioLength(0)); EXPECT_EQ(-1, dm_->SetPacketAudioLength(-1)); + + // Minimum delay higher than a maximum delay is not accepted. + EXPECT_TRUE(dm_->SetMaximumDelay(10)); + EXPECT_FALSE(dm_->SetMinimumDelay(20)); + + // Maximum delay less than minimum delay is not accepted. + EXPECT_TRUE(dm_->SetMaximumDelay(100)); + EXPECT_TRUE(dm_->SetMinimumDelay(80)); + EXPECT_FALSE(dm_->SetMaximumDelay(60)); } } // namespace webrtc diff --git a/webrtc/modules/audio_coding/neteq4/interface/neteq.h b/webrtc/modules/audio_coding/neteq4/interface/neteq.h index 6243bce1e5..824b17b0a0 100644 --- a/webrtc/modules/audio_coding/neteq4/interface/neteq.h +++ b/webrtc/modules/audio_coding/neteq4/interface/neteq.h @@ -151,10 +151,22 @@ class NetEq { // -1 on failure. virtual int RemovePayloadType(uint8_t rtp_payload_type) = 0; - // Sets the desired extra delay on top of what NetEq already applies due to - // current network situation. Used for synchronization with video. Returns - // true if successful, otherwise false. - virtual bool SetExtraDelay(int extra_delay_ms) = 0; + // Sets a minimum delay in millisecond for packet buffer. The minimum is + // maintained unless a higher latency is dictated by channel condition. + // Returns true if the minimum is successfully applied, otherwise false is + // returned. + virtual bool SetMinimumDelay(int delay_ms) = 0; + + // Sets a maximum delay in milliseconds for packet buffer. The latency will + // not exceed the given value, even required delay (given the channel + // conditions) is higher. + virtual bool SetMaximumDelay(int delay_ms) = 0; + + // The smallest latency required. This is computed bases on inter-arrival + // time and internal NetEq logic. Note that in computing this latency none of + // the user defined limits (applied by calling setMinimumDelay() and/or + // SetMaximumDelay()) are applied. + virtual int LeastRequiredDelayMs() const = 0; // Not implemented. virtual int SetTargetDelay() = 0; diff --git a/webrtc/modules/audio_coding/neteq4/neteq_impl.cc b/webrtc/modules/audio_coding/neteq4/neteq_impl.cc index 9323b20ac0..403fccb37a 100644 --- a/webrtc/modules/audio_coding/neteq4/neteq_impl.cc +++ b/webrtc/modules/audio_coding/neteq4/neteq_impl.cc @@ -236,16 +236,30 @@ int NetEqImpl::RemovePayloadType(uint8_t rtp_payload_type) { return kFail; } -bool NetEqImpl::SetExtraDelay(int extra_delay_ms) { +bool NetEqImpl::SetMinimumDelay(int delay_ms) { CriticalSectionScoped lock(crit_sect_); - if (extra_delay_ms >= 0 && extra_delay_ms < 10000) { + if (delay_ms >= 0 && delay_ms < 10000) { assert(delay_manager_.get()); - delay_manager_->set_extra_delay_ms(extra_delay_ms); - return true; + return delay_manager_->SetMinimumDelay(delay_ms); } return false; } +bool NetEqImpl::SetMaximumDelay(int delay_ms) { + CriticalSectionScoped lock(crit_sect_); + if (delay_ms >= 0 && delay_ms < 10000) { + assert(delay_manager_.get()); + return delay_manager_->SetMaximumDelay(delay_ms); + } + return false; +} + +int NetEqImpl::LeastRequiredDelayMs() const { + CriticalSectionScoped lock(crit_sect_); + assert(delay_manager_.get()); + return delay_manager_->least_required_delay_ms(); +} + void NetEqImpl::SetPlayoutMode(NetEqPlayoutMode mode) { CriticalSectionScoped lock(crit_sect_); if (!decision_logic_.get() || mode != decision_logic_->playout_mode()) { diff --git a/webrtc/modules/audio_coding/neteq4/neteq_impl.h b/webrtc/modules/audio_coding/neteq4/neteq_impl.h index e0e31cef76..0d81208447 100644 --- a/webrtc/modules/audio_coding/neteq4/neteq_impl.h +++ b/webrtc/modules/audio_coding/neteq4/neteq_impl.h @@ -102,10 +102,11 @@ class NetEqImpl : public webrtc::NetEq { // -1 on failure. virtual int RemovePayloadType(uint8_t rtp_payload_type); - // Sets the desired extra delay on top of what NetEq already applies due to - // current network situation. Used for synchronization with video. Returns - // true if successful, otherwise false. - virtual bool SetExtraDelay(int extra_delay_ms); + virtual bool SetMinimumDelay(int delay_ms); + + virtual bool SetMaximumDelay(int delay_ms); + + virtual int LeastRequiredDelayMs() const; virtual int SetTargetDelay() { return kNotImplemented; }