diff --git a/modules/audio_coding/BUILD.gn b/modules/audio_coding/BUILD.gn index 3b7c5f0d80..29aa1e77e7 100644 --- a/modules/audio_coding/BUILD.gn +++ b/modules/audio_coding/BUILD.gn @@ -969,6 +969,7 @@ rtc_library("neteq") { "neteq/nack_tracker.cc", "neteq/nack_tracker.h", "neteq/neteq.cc", + "neteq/neteq_controller.h", "neteq/neteq_impl.cc", "neteq/neteq_impl.h", "neteq/normal.cc", @@ -1955,14 +1956,13 @@ if (rtc_include_tests) { "neteq/expand_unittest.cc", "neteq/histogram_unittest.cc", "neteq/merge_unittest.cc", - "neteq/mock/mock_buffer_level_filter.h", "neteq/mock/mock_decoder_database.h", - "neteq/mock/mock_delay_manager.h", "neteq/mock/mock_delay_peak_detector.h", "neteq/mock/mock_dtmf_buffer.h", "neteq/mock/mock_dtmf_tone_generator.h", "neteq/mock/mock_expand.h", "neteq/mock/mock_histogram.h", + "neteq/mock/mock_neteq_controller.h", "neteq/mock/mock_packet_buffer.h", "neteq/mock/mock_red_payload_splitter.h", "neteq/mock/mock_statistics_calculator.h", diff --git a/modules/audio_coding/neteq/decision_logic.cc b/modules/audio_coding/neteq/decision_logic.cc index fefad689ed..1c887b11b5 100644 --- a/modules/audio_coding/neteq/decision_logic.cc +++ b/modules/audio_coding/neteq/decision_logic.cc @@ -16,12 +16,7 @@ #include #include "absl/types/optional.h" -#include "modules/audio_coding/neteq/buffer_level_filter.h" -#include "modules/audio_coding/neteq/decoder_database.h" -#include "modules/audio_coding/neteq/delay_manager.h" -#include "modules/audio_coding/neteq/expand.h" #include "modules/audio_coding/neteq/packet_buffer.h" -#include "modules/audio_coding/neteq/sync_buffer.h" #include "rtc_base/checks.h" #include "rtc_base/experiments/field_trial_parser.h" #include "rtc_base/logging.h" @@ -37,48 +32,23 @@ constexpr int kDefaultTargetLevelWindowMs = 100; namespace webrtc { -DecisionLogic* DecisionLogic::Create(int fs_hz, - size_t output_size_samples, - bool disallow_time_stretching, - DecoderDatabase* decoder_database, - const PacketBuffer& packet_buffer, - DelayManager* delay_manager, - BufferLevelFilter* buffer_level_filter, - const TickTimer* tick_timer) { - return new DecisionLogic(fs_hz, output_size_samples, disallow_time_stretching, - decoder_database, packet_buffer, delay_manager, - buffer_level_filter, tick_timer); -} - -DecisionLogic::DecisionLogic(int fs_hz, - size_t output_size_samples, - bool disallow_time_stretching, - DecoderDatabase* decoder_database, - const PacketBuffer& packet_buffer, - DelayManager* delay_manager, - BufferLevelFilter* buffer_level_filter, - const TickTimer* tick_timer) - : decoder_database_(decoder_database), - packet_buffer_(packet_buffer), - delay_manager_(delay_manager), - buffer_level_filter_(buffer_level_filter), - tick_timer_(tick_timer), - cng_state_(kCngOff), - packet_length_samples_(0), - sample_memory_(0), - prev_time_scale_(false), - disallow_time_stretching_(disallow_time_stretching), +DecisionLogic::DecisionLogic(NetEqController::Config config) + : delay_peak_detector_(config.tick_timer, config.enable_rtx_handling), + delay_manager_(DelayManager::Create(config.max_packets_in_buffer, + config.base_min_delay_ms, + config.enable_rtx_handling, + &delay_peak_detector_, + config.tick_timer)), + tick_timer_(config.tick_timer), + disallow_time_stretching_(!config.allow_time_stretching), timescale_countdown_( tick_timer_->GetNewCountdown(kMinTimescaleInterval + 1)), - num_consecutive_expands_(0), - time_stretched_cn_samples_(0), estimate_dtx_delay_("estimate_dtx_delay", false), time_stretch_cn_("time_stretch_cn", false), target_level_window_ms_("target_level_window", kDefaultTargetLevelWindowMs, 0, absl::nullopt) { - SetSampleRate(fs_hz, output_size_samples); const std::string field_trial_name = field_trial::FindFullName("WebRTC-Audio-NetEqDecisionLogicSettings"); ParseFieldTrial( @@ -110,6 +80,8 @@ void DecisionLogic::SoftReset() { timescale_countdown_ = tick_timer_->GetNewCountdown(kMinTimescaleInterval + 1); time_stretched_cn_samples_ = 0; + delay_manager_->Reset(); + buffer_level_filter_.Reset(); } void DecisionLogic::SetSampleRate(int fs_hz, size_t output_size_samples) { @@ -119,69 +91,54 @@ void DecisionLogic::SetSampleRate(int fs_hz, size_t output_size_samples) { output_size_samples_ = output_size_samples; } -Operations DecisionLogic::GetDecision(const SyncBuffer& sync_buffer, - const Expand& expand, - size_t decoder_frame_length, - const Packet* next_packet, - Modes prev_mode, - bool play_dtmf, - size_t generated_noise_samples, +Operations DecisionLogic::GetDecision(const NetEqStatus& status, bool* reset_decoder) { // If last mode was CNG (or Expand, since this could be covering up for // a lost CNG packet), remember that CNG is on. This is needed if comfort // noise is interrupted by DTMF. - if (prev_mode == kModeRfc3389Cng) { + if (status.last_mode == kModeRfc3389Cng) { cng_state_ = kCngRfc3389On; - } else if (prev_mode == kModeCodecInternalCng) { + } else if (status.last_mode == kModeCodecInternalCng) { cng_state_ = kCngInternalOn; } - size_t cur_size_samples = - estimate_dtx_delay_ - ? packet_buffer_.GetSpanSamples(decoder_frame_length, sample_rate_, - true) - : packet_buffer_.NumSamplesInBuffer(decoder_frame_length); + size_t cur_size_samples = estimate_dtx_delay_ + ? status.packet_buffer_info.span_samples + : status.packet_buffer_info.num_samples; prev_time_scale_ = - prev_time_scale_ && (prev_mode == kModeAccelerateSuccess || - prev_mode == kModeAccelerateLowEnergy || - prev_mode == kModePreemptiveExpandSuccess || - prev_mode == kModePreemptiveExpandLowEnergy); + prev_time_scale_ && (status.last_mode == kModeAccelerateSuccess || + status.last_mode == kModeAccelerateLowEnergy || + status.last_mode == kModePreemptiveExpandSuccess || + status.last_mode == kModePreemptiveExpandLowEnergy); // Do not update buffer history if currently playing CNG since it will bias // the filtered buffer level. - if (prev_mode != kModeRfc3389Cng && prev_mode != kModeCodecInternalCng && - !(next_packet && next_packet->frame && - next_packet->frame->IsDtxPacket() && !estimate_dtx_delay_)) { + if (status.last_mode != kModeRfc3389Cng && + status.last_mode != kModeCodecInternalCng && + !(status.next_packet && status.next_packet->is_dtx && + !estimate_dtx_delay_)) { FilterBufferLevel(cur_size_samples); } // Guard for errors, to avoid getting stuck in error mode. - if (prev_mode == kModeError) { - if (!next_packet) { + if (status.last_mode == kModeError) { + if (!status.next_packet) { return kExpand; } else { return kUndefined; // Use kUndefined to flag for a reset. } } - uint32_t target_timestamp = sync_buffer.end_timestamp(); - uint32_t available_timestamp = 0; - bool is_cng_packet = false; - if (next_packet) { - available_timestamp = next_packet->timestamp; - is_cng_packet = - decoder_database_->IsComfortNoise(next_packet->payload_type); - } - - if (is_cng_packet) { - return CngOperation(prev_mode, target_timestamp, available_timestamp, - generated_noise_samples); + if (status.next_packet && status.next_packet->is_cng) { + return CngOperation(status.last_mode, status.target_timestamp, + status.next_packet->timestamp, + status.generated_noise_samples); } // Handle the case with no packet at all available (except maybe DTMF). - if (!next_packet) { - return NoPacket(play_dtmf); + if (!status.next_packet) { + return NoPacket(status.play_dtmf); } // If the expand period was very long, reset NetEQ since it is likely that the @@ -198,26 +155,30 @@ Operations DecisionLogic::GetDecision(const SyncBuffer& sync_buffer, // if the mute factor is low enough (otherwise the expansion was short enough // to not be noticable). // Note that the MuteFactor is in Q14, so a value of 16384 corresponds to 1. - size_t current_span = packet_buffer_.GetSpanSamples( - decoder_frame_length, sample_rate_, estimate_dtx_delay_); - if ((prev_mode == kModeExpand || prev_mode == kModeCodecPlc) && - expand.MuteFactor(0) < 16384 / 2 && + const size_t current_span = + estimate_dtx_delay_ ? status.packet_buffer_info.span_samples + : status.packet_buffer_info.span_samples_no_dtx; + if ((status.last_mode == kModeExpand || status.last_mode == kModeCodecPlc) && + status.expand_mutefactor < 16384 / 2 && current_span(delay_manager_->TargetLevel() * packet_length_samples_ * kPostponeDecodingLevel / 100)>> 8 && - !packet_buffer_.ContainsDtxOrCngPacket(decoder_database_)) { + !status.packet_buffer_info.dtx_or_cng) { return kExpand; } const uint32_t five_seconds_samples = static_cast(5 * sample_rate_); // Check if the required packet is available. - if (target_timestamp == available_timestamp) { - return ExpectedPacketAvailable(prev_mode, play_dtmf); - } else if (!PacketBuffer::IsObsoleteTimestamp( - available_timestamp, target_timestamp, five_seconds_samples)) { - return FuturePacketAvailable(decoder_frame_length, prev_mode, - target_timestamp, available_timestamp, - play_dtmf, generated_noise_samples); + if (status.target_timestamp == status.next_packet->timestamp) { + return ExpectedPacketAvailable(status.last_mode, status.play_dtmf); + } else if (!PacketBuffer::IsObsoleteTimestamp(status.next_packet->timestamp, + status.target_timestamp, + five_seconds_samples)) { + return FuturePacketAvailable( + status.last_packet_samples, status.last_mode, status.target_timestamp, + status.next_packet->timestamp, status.play_dtmf, + status.generated_noise_samples, status.packet_buffer_info.span_samples, + status.packet_buffer_info.num_packets); } else { // This implies that available_timestamp < target_timestamp, which can // happen when a new stream or codec is received. Signal for a reset. @@ -233,8 +194,40 @@ void DecisionLogic::ExpandDecision(Operations operation) { } } +absl::optional DecisionLogic::PacketArrived(bool last_cng_or_dtmf, + size_t packet_length_samples, + bool should_update_stats, + uint16_t main_sequence_number, + uint32_t main_timestamp, + int fs_hz) { + delay_manager_->LastDecodedWasCngOrDtmf(last_cng_or_dtmf); + absl::optional relative_delay; + if (delay_manager_->last_pack_cng_or_dtmf() == 0) { + // Calculate the total speech length carried in each packet. + if (packet_length_samples > 0 && + packet_length_samples != packet_length_samples_) { + packet_length_samples_ = packet_length_samples; + delay_manager_->SetPacketAudioLength( + rtc::dchecked_cast((1000 * packet_length_samples) / fs_hz)); + } + + // Update statistics. + if (should_update_stats) { + relative_delay = + delay_manager_->Update(main_sequence_number, main_timestamp, fs_hz); + } + } else if (delay_manager_->last_pack_cng_or_dtmf() == -1) { + // This is first "normal" packet after CNG or DTMF. + // Reset packet time counter and measure time until next packet, + // but don't update statistics. + delay_manager_->set_last_pack_cng_or_dtmf(0); + delay_manager_->ResetPacketIatCount(); + } + return relative_delay; +} + void DecisionLogic::FilterBufferLevel(size_t buffer_size_samples) { - buffer_level_filter_->SetTargetBufferLevel( + buffer_level_filter_.SetTargetBufferLevel( delay_manager_->base_target_level()); int time_stretched_samples = time_stretched_cn_samples_; @@ -243,7 +236,7 @@ void DecisionLogic::FilterBufferLevel(size_t buffer_size_samples) { timescale_countdown_ = tick_timer_->GetNewCountdown(kMinTimescaleInterval); } - buffer_level_filter_->Update(buffer_size_samples, time_stretched_samples); + buffer_level_filter_.Update(buffer_size_samples, time_stretched_samples); prev_time_scale_ = false; time_stretched_cn_samples_ = 0; } @@ -307,7 +300,7 @@ Operations DecisionLogic::ExpectedPacketAvailable(Modes prev_mode, int buffer_level_packets = 0; if (packet_length_samples_ > 0) { buffer_level_packets = - ((1 << 8) * buffer_level_filter_->filtered_current_level()) / + ((1 << 8) * buffer_level_filter_.filtered_current_level()) / packet_length_samples_; } if (buffer_level_packets >= high_limit << 2) @@ -328,7 +321,9 @@ Operations DecisionLogic::FuturePacketAvailable( uint32_t target_timestamp, uint32_t available_timestamp, bool play_dtmf, - size_t generated_noise_samples) { + size_t generated_noise_samples, + size_t span_samples_in_packet_buffer, + size_t num_packets_in_packet_buffer) { // Required packet is not available, but a future packet is. // Check if we should continue with an ongoing expand because the new packet // is too far into the future. @@ -353,9 +348,8 @@ Operations DecisionLogic::FuturePacketAvailable( if (prev_mode == kModeRfc3389Cng || prev_mode == kModeCodecInternalCng) { size_t cur_size_samples = estimate_dtx_delay_ - ? cur_size_samples = packet_buffer_.GetSpanSamples( - decoder_frame_length, sample_rate_, true) - : packet_buffer_.NumPacketsInBuffer() * decoder_frame_length; + ? cur_size_samples = span_samples_in_packet_buffer + : num_packets_in_packet_buffer * decoder_frame_length; // Target level is in number of packets in Q8. const size_t target_level_samples = (delay_manager_->TargetLevel() * packet_length_samples_) >> 8; @@ -411,7 +405,7 @@ bool DecisionLogic::UnderTargetLevel() const { int buffer_level_packets = 0; if (packet_length_samples_ > 0) { buffer_level_packets = - ((1 << 8) * buffer_level_filter_->filtered_current_level()) / + ((1 << 8) * buffer_level_filter_.filtered_current_level()) / packet_length_samples_; } return buffer_level_packets <= delay_manager_->TargetLevel(); diff --git a/modules/audio_coding/neteq/decision_logic.h b/modules/audio_coding/neteq/decision_logic.h index 5a9bffb1bf..f0d7a9329f 100644 --- a/modules/audio_coding/neteq/decision_logic.h +++ b/modules/audio_coding/neteq/decision_logic.h @@ -11,103 +11,108 @@ #ifndef MODULES_AUDIO_CODING_NETEQ_DECISION_LOGIC_H_ #define MODULES_AUDIO_CODING_NETEQ_DECISION_LOGIC_H_ +#include "modules/audio_coding/neteq/buffer_level_filter.h" #include "modules/audio_coding/neteq/defines.h" +#include "modules/audio_coding/neteq/delay_manager.h" +#include "modules/audio_coding/neteq/delay_peak_detector.h" +#include "modules/audio_coding/neteq/neteq_controller.h" #include "modules/audio_coding/neteq/tick_timer.h" #include "rtc_base/constructor_magic.h" #include "rtc_base/experiments/field_trial_parser.h" namespace webrtc { -// Forward declarations. -class BufferLevelFilter; -class DecoderDatabase; -class DelayManager; -class Expand; -class PacketBuffer; -class SyncBuffer; -struct Packet; - // This is the class for the decision tree implementation. -class DecisionLogic final { +class DecisionLogic : public NetEqController { public: - // Static factory function which creates different types of objects depending - // on the |playout_mode|. - static DecisionLogic* Create(int fs_hz, - size_t output_size_samples, - bool disallow_time_stretching, - DecoderDatabase* decoder_database, - const PacketBuffer& packet_buffer, - DelayManager* delay_manager, - BufferLevelFilter* buffer_level_filter, - const TickTimer* tick_timer); - static const int kReinitAfterExpands = 100; static const int kMaxWaitForPacket = 10; // Constructor. - DecisionLogic(int fs_hz, - size_t output_size_samples, - bool disallow_time_stretching, - DecoderDatabase* decoder_database, - const PacketBuffer& packet_buffer, - DelayManager* delay_manager, - BufferLevelFilter* buffer_level_filter, - const TickTimer* tick_timer); + DecisionLogic(NetEqController::Config config); - ~DecisionLogic(); + ~DecisionLogic() override; // Resets object to a clean state. - void Reset(); + void Reset() override; // Resets parts of the state. Typically done when switching codecs. - void SoftReset(); + void SoftReset() override; // Sets the sample rate and the output block size. - void SetSampleRate(int fs_hz, size_t output_size_samples); + void SetSampleRate(int fs_hz, size_t output_size_samples) override; - // Returns the operation that should be done next. |sync_buffer| and |expand| - // are provided for reference. |decoder_frame_length| is the number of samples + // Given info about the latest received packet, and current jitter buffer + // status, returns the operation. |target_timestamp| and |expand_mutefactor| + // are provided for reference. |last_packet_samples| is the number of samples // obtained from the last decoded frame. If there is a packet available, it - // should be supplied in |next_packet|; otherwise it should be NULL. The mode + // should be supplied in |packet|; otherwise it should be NULL. The mode // resulting from the last call to NetEqImpl::GetAudio is supplied in - // |prev_mode|. If there is a DTMF event to play, |play_dtmf| should be set to + // |last_mode|. If there is a DTMF event to play, |play_dtmf| should be set to // true. The output variable |reset_decoder| will be set to true if a reset is // required; otherwise it is left unchanged (i.e., it can remain true if it - // was true before the call). This method end with calling - // GetDecisionSpecialized to get the actual return value. - Operations GetDecision(const SyncBuffer& sync_buffer, - const Expand& expand, - size_t decoder_frame_length, - const Packet* next_packet, - Modes prev_mode, - bool play_dtmf, - size_t generated_noise_samples, - bool* reset_decoder); + // was true before the call). + Operations GetDecision(const NetEqStatus& status, + bool* reset_decoder) override; // These methods test the |cng_state_| for different conditions. - bool CngRfc3389On() const { return cng_state_ == kCngRfc3389On; } - bool CngOff() const { return cng_state_ == kCngOff; } + bool CngRfc3389On() const override { return cng_state_ == kCngRfc3389On; } + bool CngOff() const override { return cng_state_ == kCngOff; } // Resets the |cng_state_| to kCngOff. - void SetCngOff() { cng_state_ = kCngOff; } + void SetCngOff() override { cng_state_ = kCngOff; } // Reports back to DecisionLogic whether the decision to do expand remains or // not. Note that this is necessary, since an expand decision can be changed // to kNormal in NetEqImpl::GetDecision if there is still enough data in the // sync buffer. - void ExpandDecision(Operations operation); + void ExpandDecision(Operations operation) override; // Adds |value| to |sample_memory_|. - void AddSampleMemory(int32_t value) { sample_memory_ += value; } + void AddSampleMemory(int32_t value) override { sample_memory_ += value; } + + int TargetLevelMs() override { + return ((delay_manager_->TargetLevel() * packet_length_samples_) >> 8) / + rtc::CheckedDivExact(sample_rate_, 1000); + } + + absl::optional PacketArrived(bool last_cng_or_dtmf, + size_t packet_length_samples, + bool should_update_stats, + uint16_t main_sequence_number, + uint32_t main_timestamp, + int fs_hz) override; + + void RegisterEmptyPacket() override { delay_manager_->RegisterEmptyPacket(); } + + bool SetMaximumDelay(int delay_ms) override { + return delay_manager_->SetMaximumDelay(delay_ms); + } + bool SetMinimumDelay(int delay_ms) override { + return delay_manager_->SetMinimumDelay(delay_ms); + } + bool SetBaseMinimumDelay(int delay_ms) override { + return delay_manager_->SetBaseMinimumDelay(delay_ms); + } + int GetBaseMinimumDelay() const override { + return delay_manager_->GetBaseMinimumDelay(); + } + bool PeakFound() const override { return delay_manager_->PeakFound(); } + + virtual int GetFilteredBufferLevel() const override { + return buffer_level_filter_.filtered_current_level(); + } // Accessors and mutators. - void set_sample_memory(int32_t value) { sample_memory_ = value; } - size_t noise_fast_forward() const { return noise_fast_forward_; } - size_t packet_length_samples() const { return packet_length_samples_; } - void set_packet_length_samples(size_t value) { + void set_sample_memory(int32_t value) override { sample_memory_ = value; } + size_t noise_fast_forward() const override { return noise_fast_forward_; } + size_t packet_length_samples() const override { + return packet_length_samples_; + } + void set_packet_length_samples(size_t value) override { packet_length_samples_ = value; } - void set_prev_time_scale(bool value) { prev_time_scale_ = value; } + void set_prev_time_scale(bool value) override { prev_time_scale_ = value; } private: // The value 5 sets maximum time-stretch rate to about 100 ms/s. @@ -140,7 +145,9 @@ class DecisionLogic final { uint32_t target_timestamp, uint32_t available_timestamp, bool play_dtmf, - size_t generated_noise_samples); + size_t generated_noise_samples, + size_t span_samples_in_packet_buffer, + size_t num_packets_in_packet_buffer); // Checks if enough time has elapsed since the last successful timescale // operation was done (i.e., accelerate or preemptive expand). @@ -163,23 +170,22 @@ class DecisionLogic final { // Checks if num_consecutive_expands_ >= kMaxWaitForPacket. bool MaxWaitForPacket() const; - DecoderDatabase* decoder_database_; - const PacketBuffer& packet_buffer_; - DelayManager* delay_manager_; - BufferLevelFilter* buffer_level_filter_; + DelayPeakDetector delay_peak_detector_; + std::unique_ptr delay_manager_; + BufferLevelFilter buffer_level_filter_; const TickTimer* tick_timer_; int sample_rate_; size_t output_size_samples_; - CngState cng_state_; // Remember if comfort noise is interrupted by other - // event (e.g., DTMF). + CngState cng_state_ = kCngOff; // Remember if comfort noise is interrupted by + // other event (e.g., DTMF). size_t noise_fast_forward_ = 0; - size_t packet_length_samples_; - int sample_memory_; - bool prev_time_scale_; + size_t packet_length_samples_ = 0; + int sample_memory_ = 0; + bool prev_time_scale_ = false; bool disallow_time_stretching_; std::unique_ptr timescale_countdown_; - int num_consecutive_expands_; - int time_stretched_cn_samples_; + int num_consecutive_expands_ = 0; + int time_stretched_cn_samples_ = 0; FieldTrialParameter estimate_dtx_delay_; FieldTrialParameter time_stretch_cn_; FieldTrialConstrained target_level_window_ms_; diff --git a/modules/audio_coding/neteq/decision_logic_unittest.cc b/modules/audio_coding/neteq/decision_logic_unittest.cc index 82f86c0353..9ba3b9d380 100644 --- a/modules/audio_coding/neteq/decision_logic_unittest.cc +++ b/modules/audio_coding/neteq/decision_logic_unittest.cc @@ -16,6 +16,7 @@ #include "modules/audio_coding/neteq/decoder_database.h" #include "modules/audio_coding/neteq/delay_manager.h" #include "modules/audio_coding/neteq/delay_peak_detector.h" +#include "modules/audio_coding/neteq/neteq_controller.h" #include "modules/audio_coding/neteq/packet_buffer.h" #include "modules/audio_coding/neteq/statistics_calculator.h" #include "modules/audio_coding/neteq/tick_timer.h" @@ -32,14 +33,15 @@ TEST(DecisionLogic, CreateAndDestroy) { TickTimer tick_timer; StatisticsCalculator stats; PacketBuffer packet_buffer(10, &tick_timer); - DelayPeakDetector delay_peak_detector(&tick_timer, false); - auto delay_manager = DelayManager::Create(240, 0, false, &delay_peak_detector, - &tick_timer, &stats); BufferLevelFilter buffer_level_filter; - DecisionLogic* logic = DecisionLogic::Create( - fs_hz, output_size_samples, false, &decoder_database, packet_buffer, - delay_manager.get(), &buffer_level_filter, &tick_timer); - delete logic; + NetEqController::Config config; + config.tick_timer = &tick_timer; + config.base_min_delay_ms = 0; + config.max_packets_in_buffer = 240; + config.enable_rtx_handling = false; + config.allow_time_stretching = true; + auto logic = std::make_unique(std::move(config)); + logic->SetSampleRate(fs_hz, output_size_samples); } // TODO(hlundin): Write more tests. diff --git a/modules/audio_coding/neteq/delay_manager.cc b/modules/audio_coding/neteq/delay_manager.cc index 3fda0385cd..727c522989 100644 --- a/modules/audio_coding/neteq/delay_manager.cc +++ b/modules/audio_coding/neteq/delay_manager.cc @@ -21,7 +21,6 @@ #include "modules/audio_coding/neteq/delay_peak_detector.h" #include "modules/audio_coding/neteq/histogram.h" -#include "modules/audio_coding/neteq/statistics_calculator.h" #include "modules/include/module_common_types_public.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" @@ -108,7 +107,6 @@ DelayManager::DelayManager(size_t max_packets_in_buffer, bool enable_rtx_handling, DelayPeakDetector* peak_detector, const TickTimer* tick_timer, - StatisticsCalculator* statistics, std::unique_ptr histogram) : first_packet_received_(false), max_packets_in_buffer_(max_packets_in_buffer), @@ -116,7 +114,6 @@ DelayManager::DelayManager(size_t max_packets_in_buffer, histogram_quantile_(histogram_quantile), histogram_mode_(histogram_mode), tick_timer_(tick_timer), - statistics_(statistics), base_minimum_delay_ms_(base_minimum_delay_ms), effective_minimum_delay_ms_(base_minimum_delay_ms), base_target_level_(4), // In Q0 domain. @@ -144,8 +141,7 @@ std::unique_ptr DelayManager::Create( int base_minimum_delay_ms, bool enable_rtx_handling, DelayPeakDetector* peak_detector, - const TickTimer* tick_timer, - StatisticsCalculator* statistics) { + const TickTimer* tick_timer) { const HistogramMode mode = RELATIVE_ARRIVAL_DELAY; DelayHistogramConfig config = GetDelayHistogramConfig(); const int quantile = config.quantile; @@ -153,17 +149,16 @@ std::unique_ptr DelayManager::Create( kDelayBuckets, config.forget_factor, config.start_forget_weight); return std::make_unique( max_packets_in_buffer, base_minimum_delay_ms, quantile, mode, - enable_rtx_handling, peak_detector, tick_timer, statistics, - std::move(histogram)); + enable_rtx_handling, peak_detector, tick_timer, std::move(histogram)); } DelayManager::~DelayManager() {} -int DelayManager::Update(uint16_t sequence_number, - uint32_t timestamp, - int sample_rate_hz) { +absl::optional DelayManager::Update(uint16_t sequence_number, + uint32_t timestamp, + int sample_rate_hz) { if (sample_rate_hz <= 0) { - return -1; + return absl::nullopt; } if (!first_packet_received_) { @@ -172,7 +167,7 @@ int DelayManager::Update(uint16_t sequence_number, last_seq_no_ = sequence_number; last_timestamp_ = timestamp; first_packet_received_ = true; - return 0; + return absl::nullopt; } // Try calculating packet length from current and previous timestamps. @@ -191,6 +186,7 @@ int DelayManager::Update(uint16_t sequence_number, } bool reordered = false; + absl::optional relative_delay; if (packet_len_ms > 0) { // Cannot update statistics unless |packet_len_ms| is valid. @@ -215,18 +211,16 @@ int DelayManager::Update(uint16_t sequence_number, } int iat_delay = iat_ms - packet_len_ms; - int relative_delay; if (reordered) { relative_delay = std::max(iat_delay, 0); } else { UpdateDelayHistory(iat_delay, timestamp, sample_rate_hz); relative_delay = CalculateRelativePacketArrivalDelay(); } - statistics_->RelativePacketArrivalDelay(relative_delay); switch (histogram_mode_) { case RELATIVE_ARRIVAL_DELAY: { - const int index = relative_delay / kBucketSizeMs; + const int index = relative_delay.value() / kBucketSizeMs; if (index < histogram_->NumBuckets()) { // Maximum delay to register is 2000 ms. histogram_->Add(index); @@ -250,14 +244,14 @@ int DelayManager::Update(uint16_t sequence_number, if (enable_rtx_handling_ && reordered && num_reordered_packets_ < kMaxReorderedPackets) { ++num_reordered_packets_; - return 0; + return relative_delay; } num_reordered_packets_ = 0; // Prepare for next packet arrival. packet_iat_stopwatch_ = tick_timer_->GetNewStopwatch(); last_seq_no_ = sequence_number; last_timestamp_ = timestamp; - return 0; + return relative_delay; } void DelayManager::UpdateDelayHistory(int iat_delay_ms, diff --git a/modules/audio_coding/neteq/delay_manager.h b/modules/audio_coding/neteq/delay_manager.h index a701242d4f..f1f24ac180 100644 --- a/modules/audio_coding/neteq/delay_manager.h +++ b/modules/audio_coding/neteq/delay_manager.h @@ -18,7 +18,6 @@ #include "absl/types/optional.h" #include "modules/audio_coding/neteq/histogram.h" -#include "modules/audio_coding/neteq/statistics_calculator.h" #include "modules/audio_coding/neteq/tick_timer.h" #include "rtc_base/constructor_magic.h" @@ -41,7 +40,6 @@ class DelayManager { bool enable_rtx_handling, DelayPeakDetector* peak_detector, const TickTimer* tick_timer, - StatisticsCalculator* statistics, std::unique_ptr histogram); // Create a DelayManager object. Notify the delay manager that the packet @@ -53,8 +51,7 @@ class DelayManager { int base_minimum_delay_ms, bool enable_rtx_handling, DelayPeakDetector* peak_detector, - const TickTimer* tick_timer, - StatisticsCalculator* statistics); + const TickTimer* tick_timer); virtual ~DelayManager(); @@ -62,10 +59,10 @@ class DelayManager { // |sequence_number| and |timestamp| from the RTP header. This updates the // inter-arrival time histogram and other statistics, as well as the // associated DelayPeakDetector. A new target buffer level is calculated. - // Returns 0 on success, -1 on failure (invalid sample rate). - virtual int Update(uint16_t sequence_number, - uint32_t timestamp, - int sample_rate_hz); + // Returns the relative delay if it can be calculated. + virtual absl::optional Update(uint16_t sequence_number, + uint32_t timestamp, + int sample_rate_hz); // Calculates a new target buffer level. Called from the Update() method. // Sets target_level_ (in Q8) and returns the same value. Also calculates @@ -168,7 +165,6 @@ class DelayManager { const int histogram_quantile_; const HistogramMode histogram_mode_; const TickTimer* tick_timer_; - StatisticsCalculator* statistics_; int base_minimum_delay_ms_; // Provides delay which is used by LimitTargetLevel as lower bound on target // delay. diff --git a/modules/audio_coding/neteq/delay_manager_unittest.cc b/modules/audio_coding/neteq/delay_manager_unittest.cc index c691fd5bc9..7abc3fbcfb 100644 --- a/modules/audio_coding/neteq/delay_manager_unittest.cc +++ b/modules/audio_coding/neteq/delay_manager_unittest.cc @@ -50,7 +50,7 @@ class DelayManagerTest : public ::testing::Test { virtual void TearDown(); void RecreateDelayManager(); void SetPacketAudioLength(int lengt_ms); - void InsertNextPacket(); + absl::optional InsertNextPacket(); void IncreaseTime(int inc_ms); std::unique_ptr dm_; @@ -84,11 +84,10 @@ void DelayManagerTest::RecreateDelayManager() { dm_ = std::make_unique( kMaxNumberOfPackets, kMinDelayMs, kDefaultHistogramQuantile, histogram_mode_, enable_rtx_handling_, &detector_, &tick_timer_, - &stats_, std::move(histogram)); + std::move(histogram)); } else { dm_ = DelayManager::Create(kMaxNumberOfPackets, kMinDelayMs, - enable_rtx_handling_, &detector_, &tick_timer_, - &stats_); + enable_rtx_handling_, &detector_, &tick_timer_); } } @@ -97,10 +96,11 @@ void DelayManagerTest::SetPacketAudioLength(int lengt_ms) { dm_->SetPacketAudioLength(lengt_ms); } -void DelayManagerTest::InsertNextPacket() { - EXPECT_EQ(0, dm_->Update(seq_no_, ts_, kFs)); +absl::optional DelayManagerTest::InsertNextPacket() { + auto relative_delay = dm_->Update(seq_no_, ts_, kFs); seq_no_ += 1; ts_ += kTsIncrement; + return relative_delay; } void DelayManagerTest::IncreaseTime(int inc_ms) { @@ -416,11 +416,11 @@ TEST_F(DelayManagerTest, EnableRtxHandling) { // Insert reordered packet. EXPECT_CALL(*mock_histogram_, Add(2)); - EXPECT_EQ(0, dm_->Update(seq_no_ - 3, ts_ - 3 * kFrameSizeMs, kFs)); + dm_->Update(seq_no_ - 3, ts_ - 3 * kFrameSizeMs, kFs); // Insert another reordered packet. EXPECT_CALL(*mock_histogram_, Add(1)); - EXPECT_EQ(0, dm_->Update(seq_no_ - 2, ts_ - 2 * kFrameSizeMs, kFs)); + dm_->Update(seq_no_ - 2, ts_ - 2 * kFrameSizeMs, kFs); // Insert the next packet in order and verify that the inter-arrival time is // estimated correctly. @@ -475,7 +475,7 @@ TEST_F(DelayManagerTest, EmptyPacketsNotReported) { TEST_F(DelayManagerTest, Failures) { // Wrong sample rate. - EXPECT_EQ(-1, dm_->Update(0, 0, -1)); + EXPECT_EQ(absl::nullopt, dm_->Update(0, 0, -1)); // Wrong packet size. EXPECT_EQ(-1, dm_->SetPacketAudioLength(0)); EXPECT_EQ(-1, dm_->SetPacketAudioLength(-1)); @@ -550,14 +550,14 @@ TEST_F(DelayManagerTest, RelativeArrivalDelayMode) { IncreaseTime(2 * kFrameSizeMs); EXPECT_CALL(*mock_histogram_, Add(1)); // 20ms delayed. - EXPECT_EQ(0, dm_->Update(seq_no_, ts_, kFs)); + dm_->Update(seq_no_, ts_, kFs); IncreaseTime(2 * kFrameSizeMs); EXPECT_CALL(*mock_histogram_, Add(2)); // 40ms delayed. - EXPECT_EQ(0, dm_->Update(seq_no_ + 1, ts_ + kTsIncrement, kFs)); + dm_->Update(seq_no_ + 1, ts_ + kTsIncrement, kFs); EXPECT_CALL(*mock_histogram_, Add(1)); // Reordered, 20ms delayed. - EXPECT_EQ(0, dm_->Update(seq_no_, ts_, kFs)); + dm_->Update(seq_no_, ts_, kFs); } TEST_F(DelayManagerTest, MaxDelayHistory) { @@ -579,20 +579,17 @@ TEST_F(DelayManagerTest, MaxDelayHistory) { IncreaseTime(kMaxHistoryMs + kFrameSizeMs); ts_ += kFs * kMaxHistoryMs / 1000; EXPECT_CALL(*mock_histogram_, Add(0)); // Not delayed. - EXPECT_EQ(0, dm_->Update(seq_no_, ts_, kFs)); + dm_->Update(seq_no_, ts_, kFs); } TEST_F(DelayManagerTest, RelativeArrivalDelayStatistic) { SetPacketAudioLength(kFrameSizeMs); - InsertNextPacket(); - + EXPECT_EQ(absl::nullopt, InsertNextPacket()); IncreaseTime(kFrameSizeMs); - EXPECT_CALL(stats_, RelativePacketArrivalDelay(0)); - InsertNextPacket(); - + EXPECT_EQ(0, InsertNextPacket()); IncreaseTime(2 * kFrameSizeMs); - EXPECT_CALL(stats_, RelativePacketArrivalDelay(20)); - InsertNextPacket(); + + EXPECT_EQ(20, InsertNextPacket()); } TEST_F(DelayManagerTest, DecelerationTargetLevelOffset) { diff --git a/modules/audio_coding/neteq/mock/mock_buffer_level_filter.h b/modules/audio_coding/neteq/mock/mock_buffer_level_filter.h deleted file mode 100644 index d76afa44a1..0000000000 --- a/modules/audio_coding/neteq/mock/mock_buffer_level_filter.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_BUFFER_LEVEL_FILTER_H_ -#define MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_BUFFER_LEVEL_FILTER_H_ - -#include "modules/audio_coding/neteq/buffer_level_filter.h" -#include "test/gmock.h" - -namespace webrtc { - -class MockBufferLevelFilter : public BufferLevelFilter { - public: - virtual ~MockBufferLevelFilter() { Die(); } - MOCK_METHOD0(Die, void()); - MOCK_METHOD0(Reset, void()); - MOCK_METHOD2(Update, - void(size_t buffer_size_samples, int time_stretched_samples)); - MOCK_METHOD1(SetTargetBufferLevel, void(int target_buffer_level)); - MOCK_CONST_METHOD0(filtered_current_level, int()); -}; - -} // namespace webrtc -#endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_BUFFER_LEVEL_FILTER_H_ diff --git a/modules/audio_coding/neteq/mock/mock_delay_manager.h b/modules/audio_coding/neteq/mock/mock_delay_manager.h deleted file mode 100644 index f935f7522c..0000000000 --- a/modules/audio_coding/neteq/mock/mock_delay_manager.h +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DELAY_MANAGER_H_ -#define MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DELAY_MANAGER_H_ - -#include - -#include "modules/audio_coding/neteq/delay_manager.h" -#include "modules/audio_coding/neteq/histogram.h" -#include "modules/audio_coding/neteq/statistics_calculator.h" -#include "test/gmock.h" - -namespace webrtc { - -class MockDelayManager : public DelayManager { - public: - MockDelayManager(size_t max_packets_in_buffer, - int base_min_target_delay_ms, - int histogram_quantile, - HistogramMode histogram_mode, - bool enable_rtx_handling, - DelayPeakDetector* peak_detector, - const TickTimer* tick_timer, - StatisticsCalculator* stats, - std::unique_ptr histogram) - : DelayManager(max_packets_in_buffer, - base_min_target_delay_ms, - histogram_quantile, - histogram_mode, - enable_rtx_handling, - peak_detector, - tick_timer, - stats, - std::move(histogram)) {} - virtual ~MockDelayManager() { Die(); } - MOCK_METHOD0(Die, void()); - MOCK_METHOD3(Update, - int(uint16_t sequence_number, - uint32_t timestamp, - int sample_rate_hz)); - MOCK_METHOD2(CalculateTargetLevel, int(int iat_packets, bool reordered)); - MOCK_METHOD1(SetPacketAudioLength, int(int length_ms)); - MOCK_METHOD0(Reset, void()); - MOCK_CONST_METHOD0(PeakFound, bool()); - MOCK_METHOD0(ResetPacketIatCount, void()); - MOCK_CONST_METHOD2(BufferLimits, void(int* lower_limit, int* higher_limit)); - MOCK_METHOD1(SetBaseMinimumDelay, bool(int delay_ms)); - MOCK_CONST_METHOD0(GetBaseMinimumDelay, int()); - MOCK_CONST_METHOD0(TargetLevel, int()); - MOCK_METHOD0(RegisterEmptyPacket, void()); - MOCK_CONST_METHOD0(base_target_level, int()); - MOCK_CONST_METHOD0(last_pack_cng_or_dtmf, int()); - MOCK_METHOD1(set_last_pack_cng_or_dtmf, void(int value)); -}; - -} // namespace webrtc -#endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DELAY_MANAGER_H_ diff --git a/modules/audio_coding/neteq/mock/mock_delay_peak_detector.h b/modules/audio_coding/neteq/mock/mock_delay_peak_detector.h index 82706f88a7..1613f4ca74 100644 --- a/modules/audio_coding/neteq/mock/mock_delay_peak_detector.h +++ b/modules/audio_coding/neteq/mock/mock_delay_peak_detector.h @@ -33,4 +33,5 @@ class MockDelayPeakDetector : public DelayPeakDetector { }; } // namespace webrtc + #endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DELAY_PEAK_DETECTOR_H_ diff --git a/modules/audio_coding/neteq/mock/mock_neteq_controller.h b/modules/audio_coding/neteq/mock/mock_neteq_controller.h new file mode 100644 index 0000000000..38aa3e7295 --- /dev/null +++ b/modules/audio_coding/neteq/mock/mock_neteq_controller.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_NETEQ_CONTROLLER_H_ +#define MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_NETEQ_CONTROLLER_H_ + +#include "modules/audio_coding/neteq/neteq_controller.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockNetEqController : public NetEqController { + public: + MockNetEqController() = default; + virtual ~MockNetEqController() { Die(); } + MOCK_METHOD0(Die, void()); + MOCK_METHOD0(Reset, void()); + MOCK_METHOD0(SoftReset, void()); + MOCK_METHOD2(GetDecision, + Operations(const NetEqStatus& neteq_status, + bool* reset_decoder)); + MOCK_METHOD6(Update, + void(uint16_t sequence_number, + uint32_t timestamp, + uint32_t last_played_out_timestamp, + bool new_codec, + bool cng_or_dtmf, + size_t packet_length_samples)); + MOCK_METHOD0(RegisterEmptyPacket, void()); + MOCK_METHOD2(SetSampleRate, void(int fs_hz, size_t output_size_samples)); + MOCK_METHOD1(SetMaximumDelay, bool(int delay_ms)); + MOCK_METHOD1(SetMinimumDelay, bool(int delay_ms)); + MOCK_METHOD1(SetBaseMinimumDelay, bool(int delay_ms)); + MOCK_CONST_METHOD0(GetBaseMinimumDelay, int()); + MOCK_CONST_METHOD0(CngRfc3389On, bool()); + MOCK_CONST_METHOD0(CngOff, bool()); + MOCK_METHOD0(SetCngOff, void()); + MOCK_METHOD1(ExpandDecision, void(Operations operation)); + MOCK_METHOD1(AddSampleMemory, void(int32_t value)); + MOCK_METHOD0(TargetLevelMs, int()); + MOCK_METHOD6(PacketArrived, + absl::optional(bool last_cng_or_dtmf, + size_t packet_length_samples, + bool should_update_stats, + uint16_t main_sequence_number, + uint32_t main_timestamp, + int fs_hz)); + MOCK_CONST_METHOD0(PeakFound, bool()); + MOCK_CONST_METHOD0(GetFilteredBufferLevel, int()); + MOCK_METHOD1(set_sample_memory, void(int32_t value)); + MOCK_CONST_METHOD0(noise_fast_forward, size_t()); + MOCK_CONST_METHOD0(packet_length_samples, size_t()); + MOCK_METHOD1(set_packet_length_samples, void(size_t value)); + MOCK_METHOD1(set_prev_time_scale, void(bool value)); +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_NETEQ_CONTROLLER_H_ diff --git a/modules/audio_coding/neteq/neteq_controller.h b/modules/audio_coding/neteq/neteq_controller.h new file mode 100644 index 0000000000..16e3a8e3d5 --- /dev/null +++ b/modules/audio_coding/neteq/neteq_controller.h @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_NETEQ_NETEQ_CONTROLLER_H_ +#define MODULES_AUDIO_CODING_NETEQ_NETEQ_CONTROLLER_H_ + +#include +#include + +#include +#include + +#include "absl/types/optional.h" +#include "modules/audio_coding/neteq/defines.h" +#include "modules/audio_coding/neteq/tick_timer.h" + +namespace webrtc { + +// Decides the actions that NetEq should take. This affects the behavior of the +// jitter buffer, and how it reacts to network conditions. +// This class will undergo substantial refactoring in the near future, and the +// API is expected to undergo significant changes. A target API is given below: +// +// class NetEqController { +// public: +// // Resets object to a clean state. +// void Reset(); +// // Given NetEq status, make a decision. +// Operation GetDecision(NetEqStatus neteq_status); +// // Register every packet received. +// void RegisterPacket(PacketInfo packet_info); +// // Register empty packet. +// void RegisterEmptyPacket(); +// // Register a codec switching. +// void CodecSwithed(); +// // Sets the sample rate. +// void SetSampleRate(int fs_hz); +// // Sets the packet length in samples. +// void SetPacketLengthSamples(); +// // Sets maximum delay. +// void SetMaximumDelay(int delay_ms); +// // Sets mininum delay. +// void SetMinimumDelay(int delay_ms); +// // Sets base mininum delay. +// void SetBaseMinimumDelay(int delay_ms); +// // Gets target buffer level. +// int GetTargetBufferLevelMs() const; +// // Gets filtered buffer level. +// int GetFilteredBufferLevel() const; +// // Gets base minimum delay. +// int GetBaseMinimumDelay() const; +// } + +class NetEqController { + public: + // This struct is used to create a NetEqController. + struct Config { + bool allow_time_stretching; + bool enable_rtx_handling; + int max_packets_in_buffer; + int base_min_delay_ms; + TickTimer* tick_timer; + }; + + struct PacketInfo { + uint32_t timestamp; + bool is_dtx; + bool is_cng; + }; + + struct PacketBufferInfo { + bool dtx_or_cng; + size_t num_samples; + size_t span_samples; + size_t span_samples_no_dtx; + size_t num_packets; + }; + + struct NetEqStatus { + uint32_t target_timestamp; + int16_t expand_mutefactor; + size_t last_packet_samples; + absl::optional next_packet; + Modes last_mode; + bool play_dtmf; + size_t generated_noise_samples; + PacketBufferInfo packet_buffer_info; + }; + + virtual ~NetEqController() = default; + + // Resets object to a clean state. + virtual void Reset() = 0; + + // Resets parts of the state. Typically done when switching codecs. + virtual void SoftReset() = 0; + + // Given info about the latest received packet, and current jitter buffer + // status, returns the operation. |target_timestamp| and |expand_mutefactor| + // are provided for reference. |last_packet_samples| is the number of samples + // obtained from the last decoded frame. If there is a packet available, it + // should be supplied in |packet|. The mode resulting from the last call to + // NetEqImpl::GetAudio is supplied in |last_mode|. If there is a DTMF event to + // play, |play_dtmf| should be set to true. The output variable + // |reset_decoder| will be set to true if a reset is required; otherwise it is + // left unchanged (i.e., it can remain true if it was true before the call). + virtual Operations GetDecision(const NetEqStatus& status, + bool* reset_decoder) = 0; + + // Inform NetEqController that an empty packet has arrived. + virtual void RegisterEmptyPacket() = 0; + + // Sets the sample rate and the output block size. + virtual void SetSampleRate(int fs_hz, size_t output_size_samples) = 0; + + // Sets a minimum or maximum delay in millisecond. + // Returns true if the delay bound is successfully applied, otherwise false. + virtual bool SetMaximumDelay(int delay_ms) = 0; + virtual bool SetMinimumDelay(int delay_ms) = 0; + + // Sets a base minimum delay in milliseconds for packet buffer. The effective + // minimum delay can't be lower than base minimum delay, even if a lower value + // is set using SetMinimumDelay. + // Returns true if the base minimum is successfully applied, otherwise false. + virtual bool SetBaseMinimumDelay(int delay_ms) = 0; + virtual int GetBaseMinimumDelay() const = 0; + + // These methods test the |cng_state_| for different conditions. + virtual bool CngRfc3389On() const = 0; + virtual bool CngOff() const = 0; + + // Resets the |cng_state_| to kCngOff. + virtual void SetCngOff() = 0; + + // Reports back to DecisionLogic whether the decision to do expand remains or + // not. Note that this is necessary, since an expand decision can be changed + // to kNormal in NetEqImpl::GetDecision if there is still enough data in the + // sync buffer. + virtual void ExpandDecision(Operations operation) = 0; + + // Adds |value| to |sample_memory_|. + virtual void AddSampleMemory(int32_t value) = 0; + + // Returns the target buffer level in ms. + virtual int TargetLevelMs() = 0; + + // Notify the NetEqController that a packet has arrived. Returns the relative + // arrival delay, if it can be computed. + virtual absl::optional PacketArrived(bool last_cng_or_dtmf, + size_t packet_length_samples, + bool should_update_stats, + uint16_t main_sequence_number, + uint32_t main_timestamp, + int fs_hz) = 0; + + // Returns true if a peak was found. + virtual bool PeakFound() const = 0; + + // Get the filtered buffer level in samples. + virtual int GetFilteredBufferLevel() const = 0; + + // Accessors and mutators. + virtual void set_sample_memory(int32_t value) = 0; + virtual size_t noise_fast_forward() const = 0; + virtual size_t packet_length_samples() const = 0; + virtual void set_packet_length_samples(size_t value) = 0; + virtual void set_prev_time_scale(bool value) = 0; +}; + +} // namespace webrtc +#endif // MODULES_AUDIO_CODING_NETEQ_NETEQ_CONTROLLER_H_ diff --git a/modules/audio_coding/neteq/neteq_impl.cc b/modules/audio_coding/neteq/neteq_impl.cc index 37036e34c4..7290e93f68 100644 --- a/modules/audio_coding/neteq/neteq_impl.cc +++ b/modules/audio_coding/neteq/neteq_impl.cc @@ -25,13 +25,10 @@ #include "modules/audio_coding/codecs/cng/webrtc_cng.h" #include "modules/audio_coding/neteq/accelerate.h" #include "modules/audio_coding/neteq/background_noise.h" -#include "modules/audio_coding/neteq/buffer_level_filter.h" #include "modules/audio_coding/neteq/comfort_noise.h" #include "modules/audio_coding/neteq/decision_logic.h" #include "modules/audio_coding/neteq/decoder_database.h" #include "modules/audio_coding/neteq/defines.h" -#include "modules/audio_coding/neteq/delay_manager.h" -#include "modules/audio_coding/neteq/delay_peak_detector.h" #include "modules/audio_coding/neteq/dtmf_buffer.h" #include "modules/audio_coding/neteq/dtmf_tone_generator.h" #include "modules/audio_coding/neteq/expand.h" @@ -57,6 +54,24 @@ #include "system_wrappers/include/clock.h" namespace webrtc { +namespace { + +std::unique_ptr CreateNetEqController( + int base_min_delay, + int max_packets_in_buffer, + bool enable_rtx_handling, + bool allow_time_stretching, + TickTimer* tick_timer) { + NetEqController::Config config; + config.base_min_delay_ms = base_min_delay; + config.max_packets_in_buffer = max_packets_in_buffer; + config.enable_rtx_handling = enable_rtx_handling; + config.allow_time_stretching = allow_time_stretching; + config.tick_timer = tick_timer; + return std::make_unique(std::move(config)); +} + +} // namespace NetEqImpl::Dependencies::Dependencies( const NetEq::Config& config, @@ -65,21 +80,18 @@ NetEqImpl::Dependencies::Dependencies( : clock(clock), tick_timer(new TickTimer), stats(new StatisticsCalculator), - buffer_level_filter(new BufferLevelFilter), decoder_database( new DecoderDatabase(decoder_factory, config.codec_pair_id)), - delay_peak_detector( - new DelayPeakDetector(tick_timer.get(), config.enable_rtx_handling)), - delay_manager(DelayManager::Create(config.max_packets_in_buffer, - config.min_delay_ms, - config.enable_rtx_handling, - delay_peak_detector.get(), - tick_timer.get(), - stats.get())), dtmf_buffer(new DtmfBuffer(config.sample_rate_hz)), dtmf_tone_generator(new DtmfToneGenerator), packet_buffer( new PacketBuffer(config.max_packets_in_buffer, tick_timer.get())), + neteq_controller( + CreateNetEqController(config.min_delay_ms, + config.max_packets_in_buffer, + config.enable_rtx_handling, + !config.for_test_no_time_stretching, + tick_timer.get())), red_payload_splitter(new RedPayloadSplitter), timestamp_scaler(new TimestampScaler(*decoder_database)), accelerate_factory(new AccelerateFactory), @@ -93,10 +105,7 @@ NetEqImpl::NetEqImpl(const NetEq::Config& config, bool create_components) : clock_(deps.clock), tick_timer_(std::move(deps.tick_timer)), - buffer_level_filter_(std::move(deps.buffer_level_filter)), decoder_database_(std::move(deps.decoder_database)), - delay_manager_(std::move(deps.delay_manager)), - delay_peak_detector_(std::move(deps.delay_peak_detector)), dtmf_buffer_(std::move(deps.dtmf_buffer)), dtmf_tone_generator_(std::move(deps.dtmf_tone_generator)), packet_buffer_(std::move(deps.packet_buffer)), @@ -107,6 +116,7 @@ NetEqImpl::NetEqImpl(const NetEq::Config& config, accelerate_factory_(std::move(deps.accelerate_factory)), preemptive_expand_factory_(std::move(deps.preemptive_expand_factory)), stats_(std::move(deps.stats)), + controller_(std::move(deps.neteq_controller)), last_mode_(kModeNormal), decoded_buffer_length_(kMaxFrameSize), decoded_buffer_(new int16_t[decoded_buffer_length_]), @@ -133,11 +143,12 @@ NetEqImpl::NetEqImpl(const NetEq::Config& config, << "Changing to 8000 Hz."; fs = 8000; } - delay_manager_->SetMaximumDelay(config.max_delay_ms); + controller_->SetMaximumDelay(config.max_delay_ms); fs_hz_ = fs; fs_mult_ = fs / 8000; last_output_sample_rate_hz_ = fs; output_size_samples_ = static_cast(kOutputSizeMs * 8 * fs_mult_); + controller_->SetSampleRate(fs_hz_, output_size_samples_); decoder_frame_length_ = 3 * output_size_samples_; if (create_components) { SetSampleRateAndChannels(fs, 1); // Default is 1 channel. @@ -166,7 +177,7 @@ void NetEqImpl::InsertEmptyPacket(const RTPHeader& /*rtp_header*/) { // rtp_header parameter. // https://bugs.chromium.org/p/webrtc/issues/detail?id=7611 rtc::CritScope lock(&crit_sect_); - delay_manager_->RegisterEmptyPacket(); + controller_->RegisterEmptyPacket(); } namespace { @@ -279,8 +290,8 @@ void NetEqImpl::RemoveAllPayloadTypes() { bool NetEqImpl::SetMinimumDelay(int delay_ms) { rtc::CritScope lock(&crit_sect_); if (delay_ms >= 0 && delay_ms <= 10000) { - assert(delay_manager_.get()); - return delay_manager_->SetMinimumDelay(delay_ms); + assert(controller_.get()); + return controller_->SetMinimumDelay(delay_ms); } return false; } @@ -288,8 +299,8 @@ bool NetEqImpl::SetMinimumDelay(int delay_ms) { bool NetEqImpl::SetMaximumDelay(int delay_ms) { rtc::CritScope lock(&crit_sect_); if (delay_ms >= 0 && delay_ms <= 10000) { - assert(delay_manager_.get()); - return delay_manager_->SetMaximumDelay(delay_ms); + assert(controller_.get()); + return controller_->SetMaximumDelay(delay_ms); } return false; } @@ -297,32 +308,28 @@ bool NetEqImpl::SetMaximumDelay(int delay_ms) { bool NetEqImpl::SetBaseMinimumDelayMs(int delay_ms) { rtc::CritScope lock(&crit_sect_); if (delay_ms >= 0 && delay_ms <= 10000) { - return delay_manager_->SetBaseMinimumDelay(delay_ms); + return controller_->SetBaseMinimumDelay(delay_ms); } return false; } int NetEqImpl::GetBaseMinimumDelayMs() const { rtc::CritScope lock(&crit_sect_); - return delay_manager_->GetBaseMinimumDelay(); + return controller_->GetBaseMinimumDelay(); } int NetEqImpl::TargetDelayMs() const { rtc::CritScope lock(&crit_sect_); - RTC_DCHECK(delay_manager_.get()); - // The value from TargetLevel() is in number of packets, represented in Q8. - const size_t target_delay_samples = - (delay_manager_->TargetLevel() * decoder_frame_length_) >> 8; - return static_cast(target_delay_samples) / - rtc::CheckedDivExact(fs_hz_, 1000); + RTC_DCHECK(controller_.get()); + return controller_->TargetLevelMs(); } int NetEqImpl::FilteredCurrentDelayMs() const { rtc::CritScope lock(&crit_sect_); // Sum up the filtered packet buffer level with the future length of the sync // buffer. - const int delay_samples = buffer_level_filter_->filtered_current_level() + - sync_buffer_->FutureLength(); + const int delay_samples = + controller_->GetFilteredBufferLevel() + sync_buffer_->FutureLength(); // The division below will truncate. The return value is in ms. return delay_samples / rtc::CheckedDivExact(fs_hz_, 1000); } @@ -333,12 +340,9 @@ int NetEqImpl::NetworkStatistics(NetEqNetworkStatistics* stats) { const size_t total_samples_in_buffers = packet_buffer_->NumSamplesInBuffer(decoder_frame_length_) + sync_buffer_->FutureLength(); - assert(delay_manager_.get()); - assert(decision_logic_.get()); - const int ms_per_packet = rtc::dchecked_cast( - decision_logic_->packet_length_samples() / (fs_hz_ / 1000)); - stats_->PopulateDelayManagerStats(ms_per_packet, *delay_manager_.get(), - stats); + assert(controller_.get()); + stats->preferred_buffer_size_ms = controller_->TargetLevelMs(); + stats->jitter_peaks_found = controller_->PeakFound(); stats_->GetNetworkStatistics(fs_hz_, total_samples_in_buffers, decoder_frame_length_, stats); return 0; @@ -712,38 +716,27 @@ int NetEqImpl::InsertPacketInternal(const RTPHeader& rtp_header, } } - // TODO(hlundin): Move this code to DelayManager class. const DecoderDatabase::DecoderInfo* dec_info = decoder_database_->GetDecoderInfo(main_payload_type); assert(dec_info); // Already checked that the payload type is known. - delay_manager_->LastDecodedWasCngOrDtmf(dec_info->IsComfortNoise() || - dec_info->IsDtmf()); - if (delay_manager_->last_pack_cng_or_dtmf() == 0) { - // Calculate the total speech length carried in each packet. - if (number_of_primary_packets > 0) { - const size_t packet_length_samples = - number_of_primary_packets * decoder_frame_length_; - if (packet_length_samples != decision_logic_->packet_length_samples()) { - decision_logic_->set_packet_length_samples(packet_length_samples); - delay_manager_->SetPacketAudioLength( - rtc::dchecked_cast((1000 * packet_length_samples) / fs_hz_)); - } - } - // Update statistics. - if ((enable_rtx_handling_ || (int32_t)(main_timestamp - timestamp_) >= 0) && - !new_codec_) { - // Only update statistics if incoming packet is not older than last played - // out packet or RTX handling is enabled, and if new codec flag is not - // set. - delay_manager_->Update(main_sequence_number, main_timestamp, fs_hz_); - } - } else if (delay_manager_->last_pack_cng_or_dtmf() == -1) { - // This is first "normal" packet after CNG or DTMF. - // Reset packet time counter and measure time until next packet, - // but don't update statistics. - delay_manager_->set_last_pack_cng_or_dtmf(0); - delay_manager_->ResetPacketIatCount(); + const bool last_cng_or_dtmf = + dec_info->IsComfortNoise() || dec_info->IsDtmf(); + const size_t packet_length_samples = + number_of_primary_packets * decoder_frame_length_; + // Only update statistics if incoming packet is not older than last played + // out packet or RTX handling is enabled, and if new codec flag is not + // set. + const bool should_update_stats = + (enable_rtx_handling_ || + static_cast(main_timestamp - timestamp_) >= 0) && + !new_codec_; + + auto relative_delay = controller_->PacketArrived( + last_cng_or_dtmf, packet_length_samples, should_update_stats, + main_sequence_number, main_timestamp, fs_hz_); + if (relative_delay) { + stats_->RelativePacketArrivalDelay(relative_delay.value()); } return 0; } @@ -1018,10 +1011,10 @@ int NetEqImpl::GetDecision(Operations* operation, uint64_t generated_noise_samples = generated_noise_stopwatch_ ? (generated_noise_stopwatch_->ElapsedTicks() - 1) * output_size_samples_ + - decision_logic_->noise_fast_forward() + controller_->noise_fast_forward() : 0; - if (decision_logic_->CngRfc3389On() || last_mode_ == kModeRfc3389Cng) { + if (controller_->CngRfc3389On() || last_mode_ == kModeRfc3389Cng) { // Because of timestamp peculiarities, we have to "manually" disallow using // a CNG packet with the same timestamp as the one that was last played. // This can happen when using redundancy and will cause the timing to shift. @@ -1050,7 +1043,7 @@ int NetEqImpl::GetDecision(Operations* operation, last_mode_ == kModePreemptiveExpandSuccess || last_mode_ == kModePreemptiveExpandLowEnergy) { // Subtract (samples_left + output_size_samples_) from sampleMemory. - decision_logic_->AddSampleMemory( + controller_->AddSampleMemory( -(samples_left + rtc::dchecked_cast(output_size_samples_))); } @@ -1067,11 +1060,31 @@ int NetEqImpl::GetDecision(Operations* operation, generated_noise_samples = generated_noise_stopwatch_ ? generated_noise_stopwatch_->ElapsedTicks() * output_size_samples_ + - decision_logic_->noise_fast_forward() + controller_->noise_fast_forward() : 0; - *operation = decision_logic_->GetDecision( - *sync_buffer_, *expand_, decoder_frame_length_, packet, last_mode_, - *play_dtmf, generated_noise_samples, &reset_decoder_); + NetEqController::NetEqStatus status; + status.packet_buffer_info.dtx_or_cng = + packet_buffer_->ContainsDtxOrCngPacket(decoder_database_.get()); + status.packet_buffer_info.num_samples = + packet_buffer_->NumSamplesInBuffer(decoder_frame_length_); + status.packet_buffer_info.span_samples = packet_buffer_->GetSpanSamples( + decoder_frame_length_, last_output_sample_rate_hz_, true); + status.packet_buffer_info.span_samples_no_dtx = + packet_buffer_->GetSpanSamples(decoder_frame_length_, + last_output_sample_rate_hz_, false); + status.packet_buffer_info.num_packets = packet_buffer_->NumPacketsInBuffer(); + status.target_timestamp = sync_buffer_->end_timestamp(); + status.expand_mutefactor = expand_->MuteFactor(0); + status.last_packet_samples = decoder_frame_length_; + status.last_mode = last_mode_; + status.play_dtmf = *play_dtmf; + status.generated_noise_samples = generated_noise_samples; + if (packet) { + status.next_packet = { + packet->timestamp, packet->frame && packet->frame->IsDtxPacket(), + decoder_database_->IsComfortNoise(packet->payload_type)}; + } + *operation = controller_->GetDecision(status, &reset_decoder_); // Disallow time stretching if this packet is DTX, because such a decision may // be based on earlier buffer level estimate, as we do not update buffer level @@ -1097,7 +1110,7 @@ int NetEqImpl::GetDecision(Operations* operation, return 0; } - decision_logic_->ExpandDecision(*operation); + controller_->ExpandDecision(*operation); // Check conditions for reset. if (new_codec_ || *operation == kUndefined) { @@ -1125,9 +1138,7 @@ int NetEqImpl::GetDecision(Operations* operation, sync_buffer_->IncreaseEndTimestamp(timestamp_ - end_timestamp); end_timestamp = timestamp_; new_codec_ = false; - decision_logic_->SoftReset(); - buffer_level_filter_->Reset(); - delay_manager_->Reset(); + controller_->SoftReset(); stats_->ResetMcu(); } @@ -1153,7 +1164,7 @@ int NetEqImpl::GetDecision(Operations* operation, generated_noise_stopwatch_ ? generated_noise_stopwatch_->ElapsedTicks() * output_size_samples_ + - decision_logic_->noise_fast_forward() + controller_->noise_fast_forward() : 0; if (generated_noise_samples > 0 && last_mode_ != kModeDtmf) { // Make a jump in timestamp due to the recently played comfort noise. @@ -1169,8 +1180,8 @@ int NetEqImpl::GetDecision(Operations* operation, // In order to do an accelerate we need at least 30 ms of audio data. if (samples_left >= static_cast(samples_30_ms)) { // Already have enough data, so we do not need to extract any more. - decision_logic_->set_sample_memory(samples_left); - decision_logic_->set_prev_time_scale(true); + controller_->set_sample_memory(samples_left); + controller_->set_prev_time_scale(true); return 0; } else if (samples_left >= static_cast(samples_10_ms) && decoder_frame_length_ >= samples_30_ms) { @@ -1201,8 +1212,8 @@ int NetEqImpl::GetDecision(Operations* operation, // Already have enough data, so we do not need to extract any more. // Or, avoid decoding more data as it might overflow the playout buffer. // Still try preemptive expand, though. - decision_logic_->set_sample_memory(samples_left); - decision_logic_->set_prev_time_scale(true); + controller_->set_sample_memory(samples_left); + controller_->set_prev_time_scale(true); return 0; } if (samples_left < static_cast(samples_20_ms) && @@ -1228,7 +1239,7 @@ int NetEqImpl::GetDecision(Operations* operation, int extracted_samples = 0; if (packet) { sync_buffer_->IncreaseEndTimestamp(packet->timestamp - end_timestamp); - if (decision_logic_->CngOff()) { + if (controller_->CngOff()) { // Adjustment of timestamp only corresponds to an actual packet loss // if comfort noise is not played. If comfort noise was just played, // this adjustment of timestamp is only done to get back in sync with the @@ -1238,7 +1249,7 @@ int NetEqImpl::GetDecision(Operations* operation, if (*operation != kRfc3389Cng) { // We are about to decode and use a non-CNG packet. - decision_logic_->SetCngOff(); + controller_->SetCngOff(); } extracted_samples = ExtractPackets(required_samples, packet_list); @@ -1249,8 +1260,8 @@ int NetEqImpl::GetDecision(Operations* operation, if (*operation == kAccelerate || *operation == kFastAccelerate || *operation == kPreemptiveExpand) { - decision_logic_->set_sample_memory(samples_left + extracted_samples); - decision_logic_->set_prev_time_scale(true); + controller_->set_sample_memory(samples_left + extracted_samples); + controller_->set_prev_time_scale(true); } if (*operation == kAccelerate || *operation == kFastAccelerate) { @@ -2058,13 +2069,8 @@ void NetEqImpl::SetSampleRateAndChannels(int fs_hz, size_t channels) { decoded_buffer_length_ = kMaxFrameSize * channels; decoded_buffer_.reset(new int16_t[decoded_buffer_length_]); } - - // Create DecisionLogic if it is not created yet, then communicate new sample - // rate and output size to DecisionLogic object. - if (!decision_logic_.get()) { - CreateDecisionLogic(); - } - decision_logic_->SetSampleRate(fs_hz_, output_size_samples_); + RTC_CHECK(controller_) << "Unexpectedly found no NetEqController"; + controller_->SetSampleRate(fs_hz_, output_size_samples_); } NetEqImpl::OutputType NetEqImpl::LastOutputType() { @@ -2085,11 +2091,4 @@ NetEqImpl::OutputType NetEqImpl::LastOutputType() { return OutputType::kNormalSpeech; } } - -void NetEqImpl::CreateDecisionLogic() { - decision_logic_.reset(DecisionLogic::Create( - fs_hz_, output_size_samples_, no_time_stretching_, - decoder_database_.get(), *packet_buffer_.get(), delay_manager_.get(), - buffer_level_filter_.get(), tick_timer_.get())); -} } // namespace webrtc diff --git a/modules/audio_coding/neteq/neteq_impl.h b/modules/audio_coding/neteq/neteq_impl.h index 842869f9fc..aa7eba144f 100644 --- a/modules/audio_coding/neteq/neteq_impl.h +++ b/modules/audio_coding/neteq/neteq_impl.h @@ -24,6 +24,7 @@ #include "modules/audio_coding/neteq/defines.h" // Modes, Operations #include "modules/audio_coding/neteq/expand_uma_logger.h" #include "modules/audio_coding/neteq/include/neteq.h" +#include "modules/audio_coding/neteq/neteq_controller.h" #include "modules/audio_coding/neteq/packet.h" #include "modules/audio_coding/neteq/random_vector.h" #include "modules/audio_coding/neteq/statistics_calculator.h" @@ -37,13 +38,9 @@ namespace webrtc { // Forward declarations. class Accelerate; class BackgroundNoise; -class BufferLevelFilter; class Clock; class ComfortNoise; -class DecisionLogic; class DecoderDatabase; -class DelayManager; -class DelayPeakDetector; class DtmfBuffer; class DtmfToneGenerator; class Expand; @@ -108,13 +105,11 @@ class NetEqImpl : public webrtc::NetEq { Clock* const clock; std::unique_ptr tick_timer; std::unique_ptr stats; - std::unique_ptr buffer_level_filter; std::unique_ptr decoder_database; - std::unique_ptr delay_peak_detector; - std::unique_ptr delay_manager; std::unique_ptr dtmf_buffer; std::unique_ptr dtmf_tone_generator; std::unique_ptr packet_buffer; + std::unique_ptr neteq_controller; std::unique_ptr red_payload_splitter; std::unique_ptr timestamp_scaler; std::unique_ptr accelerate_factory; @@ -338,20 +333,12 @@ class NetEqImpl : public webrtc::NetEq { virtual void UpdatePlcComponents(int fs_hz, size_t channels) RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); - // Creates DecisionLogic object with the mode given by |playout_mode_|. - virtual void CreateDecisionLogic() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); - Clock* const clock_; rtc::CriticalSection crit_sect_; const std::unique_ptr tick_timer_ RTC_GUARDED_BY(crit_sect_); - const std::unique_ptr buffer_level_filter_ - RTC_GUARDED_BY(crit_sect_); const std::unique_ptr decoder_database_ RTC_GUARDED_BY(crit_sect_); - const std::unique_ptr delay_manager_ RTC_GUARDED_BY(crit_sect_); - const std::unique_ptr delay_peak_detector_ - RTC_GUARDED_BY(crit_sect_); const std::unique_ptr dtmf_buffer_ RTC_GUARDED_BY(crit_sect_); const std::unique_ptr dtmf_tone_generator_ RTC_GUARDED_BY(crit_sect_); @@ -370,7 +357,7 @@ class NetEqImpl : public webrtc::NetEq { const std::unique_ptr stats_ RTC_GUARDED_BY(crit_sect_); std::unique_ptr background_noise_ RTC_GUARDED_BY(crit_sect_); - std::unique_ptr decision_logic_ RTC_GUARDED_BY(crit_sect_); + std::unique_ptr controller_ RTC_GUARDED_BY(crit_sect_); std::unique_ptr algorithm_buffer_ RTC_GUARDED_BY(crit_sect_); std::unique_ptr sync_buffer_ RTC_GUARDED_BY(crit_sect_); diff --git a/modules/audio_coding/neteq/neteq_impl_unittest.cc b/modules/audio_coding/neteq/neteq_impl_unittest.cc index 88629050ce..4a47a4d71d 100644 --- a/modules/audio_coding/neteq/neteq_impl_unittest.cc +++ b/modules/audio_coding/neteq/neteq_impl_unittest.cc @@ -16,17 +16,17 @@ #include "api/audio_codecs/builtin_audio_decoder_factory.h" #include "modules/audio_coding/neteq/accelerate.h" +#include "modules/audio_coding/neteq/decision_logic.h" #include "modules/audio_coding/neteq/expand.h" #include "modules/audio_coding/neteq/histogram.h" #include "modules/audio_coding/neteq/include/neteq.h" -#include "modules/audio_coding/neteq/mock/mock_buffer_level_filter.h" #include "modules/audio_coding/neteq/mock/mock_decoder_database.h" -#include "modules/audio_coding/neteq/mock/mock_delay_manager.h" -#include "modules/audio_coding/neteq/mock/mock_delay_peak_detector.h" #include "modules/audio_coding/neteq/mock/mock_dtmf_buffer.h" #include "modules/audio_coding/neteq/mock/mock_dtmf_tone_generator.h" +#include "modules/audio_coding/neteq/mock/mock_neteq_controller.h" #include "modules/audio_coding/neteq/mock/mock_packet_buffer.h" #include "modules/audio_coding/neteq/mock/mock_red_payload_splitter.h" +#include "modules/audio_coding/neteq/neteq_controller.h" #include "modules/audio_coding/neteq/preemptive_expand.h" #include "modules/audio_coding/neteq/statistics_calculator.h" #include "modules/audio_coding/neteq/sync_buffer.h" @@ -78,13 +78,6 @@ class NetEqImplTest : public ::testing::Test { // Get a local pointer to NetEq's TickTimer object. tick_timer_ = deps.tick_timer.get(); - if (use_mock_buffer_level_filter_) { - std::unique_ptr mock(new MockBufferLevelFilter); - mock_buffer_level_filter_ = mock.get(); - deps.buffer_level_filter = std::move(mock); - } - buffer_level_filter_ = deps.buffer_level_filter.get(); - if (use_mock_decoder_database_) { std::unique_ptr mock(new MockDecoderDatabase); mock_decoder_database_ = mock.get(); @@ -94,26 +87,6 @@ class NetEqImplTest : public ::testing::Test { } decoder_database_ = deps.decoder_database.get(); - if (use_mock_delay_peak_detector_) { - std::unique_ptr mock( - new MockDelayPeakDetector(tick_timer_, config_.enable_rtx_handling)); - mock_delay_peak_detector_ = mock.get(); - EXPECT_CALL(*mock_delay_peak_detector_, Reset()).Times(1); - deps.delay_peak_detector = std::move(mock); - } - delay_peak_detector_ = deps.delay_peak_detector.get(); - - if (use_mock_delay_manager_) { - std::unique_ptr mock(new MockDelayManager( - config_.max_packets_in_buffer, config_.min_delay_ms, 1020054733, - DelayManager::HistogramMode::INTER_ARRIVAL_TIME, - config_.enable_rtx_handling, delay_peak_detector_, tick_timer_, - deps.stats.get(), std::make_unique(50, 32745))); - mock_delay_manager_ = mock.get(); - deps.delay_manager = std::move(mock); - } - delay_manager_ = deps.delay_manager.get(); - if (use_mock_dtmf_buffer_) { std::unique_ptr mock( new MockDtmfBuffer(config_.sample_rate_hz)); @@ -137,6 +110,23 @@ class NetEqImplTest : public ::testing::Test { } packet_buffer_ = deps.packet_buffer.get(); + if (use_mock_neteq_controller_) { + std::unique_ptr mock(new MockNetEqController()); + mock_neteq_controller_ = mock.get(); + deps.neteq_controller = std::move(mock); + } else { + deps.stats = std::make_unique(); + NetEqController::Config controller_config; + controller_config.tick_timer = tick_timer_; + controller_config.base_min_delay_ms = config_.min_delay_ms; + controller_config.enable_rtx_handling = config_.enable_rtx_handling; + controller_config.allow_time_stretching = true; + controller_config.max_packets_in_buffer = config_.max_packets_in_buffer; + deps.neteq_controller = + std::make_unique(std::move(controller_config)); + } + neteq_controller_ = deps.neteq_controller.get(); + if (use_mock_payload_splitter_) { std::unique_ptr mock(new MockRedPayloadSplitter); mock_payload_splitter_ = mock.get(); @@ -155,10 +145,8 @@ class NetEqImplTest : public ::testing::Test { void UseNoMocks() { ASSERT_TRUE(neteq_ == NULL) << "Must call UseNoMocks before CreateInstance"; - use_mock_buffer_level_filter_ = false; use_mock_decoder_database_ = false; - use_mock_delay_peak_detector_ = false; - use_mock_delay_manager_ = false; + use_mock_neteq_controller_ = false; use_mock_dtmf_buffer_ = false; use_mock_dtmf_tone_generator_ = false; use_mock_packet_buffer_ = false; @@ -166,17 +154,11 @@ class NetEqImplTest : public ::testing::Test { } virtual ~NetEqImplTest() { - if (use_mock_buffer_level_filter_) { - EXPECT_CALL(*mock_buffer_level_filter_, Die()).Times(1); - } if (use_mock_decoder_database_) { EXPECT_CALL(*mock_decoder_database_, Die()).Times(1); } - if (use_mock_delay_manager_) { - EXPECT_CALL(*mock_delay_manager_, Die()).Times(1); - } - if (use_mock_delay_peak_detector_) { - EXPECT_CALL(*mock_delay_peak_detector_, Die()).Times(1); + if (use_mock_neteq_controller_) { + EXPECT_CALL(*mock_neteq_controller_, Die()).Times(1); } if (use_mock_dtmf_buffer_) { EXPECT_CALL(*mock_dtmf_buffer_, Die()).Times(1); @@ -242,18 +224,12 @@ class NetEqImplTest : public ::testing::Test { NetEq::Config config_; SimulatedClock clock_; TickTimer* tick_timer_ = nullptr; - MockBufferLevelFilter* mock_buffer_level_filter_ = nullptr; - BufferLevelFilter* buffer_level_filter_ = nullptr; - bool use_mock_buffer_level_filter_ = true; MockDecoderDatabase* mock_decoder_database_ = nullptr; DecoderDatabase* decoder_database_ = nullptr; bool use_mock_decoder_database_ = true; - MockDelayPeakDetector* mock_delay_peak_detector_ = nullptr; - DelayPeakDetector* delay_peak_detector_ = nullptr; - bool use_mock_delay_peak_detector_ = true; - MockDelayManager* mock_delay_manager_ = nullptr; - DelayManager* delay_manager_ = nullptr; - bool use_mock_delay_manager_ = true; + MockNetEqController* mock_neteq_controller_ = nullptr; + NetEqController* neteq_controller_ = nullptr; + bool use_mock_neteq_controller_ = true; MockDtmfBuffer* mock_dtmf_buffer_ = nullptr; DtmfBuffer* dtmf_buffer_ = nullptr; bool use_mock_dtmf_buffer_ = true; @@ -367,16 +343,20 @@ TEST_F(NetEqImplTest, InsertPacket) { // All expectations within this block must be called in this specific order. InSequence sequence; // Dummy variable. // Expectations when the first packet is inserted. - EXPECT_CALL(*mock_delay_manager_, last_pack_cng_or_dtmf()) - .Times(2) - .WillRepeatedly(Return(-1)); - EXPECT_CALL(*mock_delay_manager_, set_last_pack_cng_or_dtmf(0)).Times(1); - EXPECT_CALL(*mock_delay_manager_, ResetPacketIatCount()).Times(1); - // Expectations when the second packet is inserted. Slightly different. - EXPECT_CALL(*mock_delay_manager_, last_pack_cng_or_dtmf()) - .WillOnce(Return(0)); - EXPECT_CALL(*mock_delay_manager_, SetPacketAudioLength(30)) - .WillOnce(Return(0)); + EXPECT_CALL(*mock_neteq_controller_, + PacketArrived(/*last_cng_or_dtmf*/ false, + /*packet_length_samples*/ _, + /*should_update_stats*/ _, + /*main_sequence_number*/ kFirstSequenceNumber, + /*main_timestamp*/ kFirstTimestamp, + /*fs_hz*/ 8000)); + EXPECT_CALL(*mock_neteq_controller_, + PacketArrived(/*last_cng_or_dtmf*/ false, + /*packet_length_samples*/ _, + /*should_update_stats*/ _, + /*main_sequence_number*/ kFirstSequenceNumber + 1, + /*main_timestamp*/ kFirstTimestamp + 160, + /*fs_hz*/ 8000)); } // Insert first packet. @@ -1340,10 +1320,10 @@ TEST_F(NetEqImplTest, TickTimerIncrement) { TEST_F(NetEqImplTest, SetBaseMinimumDelay) { UseNoMocks(); - use_mock_delay_manager_ = true; + use_mock_neteq_controller_ = true; CreateInstance(); - EXPECT_CALL(*mock_delay_manager_, SetBaseMinimumDelay(_)) + EXPECT_CALL(*mock_neteq_controller_, SetBaseMinimumDelay(_)) .WillOnce(Return(true)) .WillOnce(Return(false)); @@ -1355,12 +1335,12 @@ TEST_F(NetEqImplTest, SetBaseMinimumDelay) { TEST_F(NetEqImplTest, GetBaseMinimumDelayMs) { UseNoMocks(); - use_mock_delay_manager_ = true; + use_mock_neteq_controller_ = true; CreateInstance(); const int delay_ms = 200; - EXPECT_CALL(*mock_delay_manager_, GetBaseMinimumDelay()) + EXPECT_CALL(*mock_neteq_controller_, GetBaseMinimumDelay()) .WillOnce(Return(delay_ms)); EXPECT_EQ(delay_ms, neteq_->GetBaseMinimumDelayMs()); @@ -1368,20 +1348,17 @@ TEST_F(NetEqImplTest, GetBaseMinimumDelayMs) { TEST_F(NetEqImplTest, TargetDelayMs) { UseNoMocks(); - use_mock_delay_manager_ = true; + use_mock_neteq_controller_ = true; CreateInstance(); - // Let the dummy target delay be 17 packets. - constexpr int kTargetLevelPacketsQ8 = 17 << 8; - EXPECT_CALL(*mock_delay_manager_, TargetLevel()) - .WillOnce(Return(kTargetLevelPacketsQ8)); - // Default packet size before any packet has been decoded is 30 ms, so we are - // expecting 17 * 30 = 510 ms target delay. - EXPECT_EQ(17 * 30, neteq_->TargetDelayMs()); + constexpr int kTargetLevelMs = 510; + EXPECT_CALL(*mock_neteq_controller_, TargetLevelMs()) + .WillOnce(Return(kTargetLevelMs)); + EXPECT_EQ(510, neteq_->TargetDelayMs()); } TEST_F(NetEqImplTest, InsertEmptyPacket) { UseNoMocks(); - use_mock_delay_manager_ = true; + use_mock_neteq_controller_ = true; CreateInstance(); RTPHeader rtp_header; @@ -1390,18 +1367,18 @@ TEST_F(NetEqImplTest, InsertEmptyPacket) { rtp_header.timestamp = 0x12345678; rtp_header.ssrc = 0x87654321; - EXPECT_CALL(*mock_delay_manager_, RegisterEmptyPacket()); + EXPECT_CALL(*mock_neteq_controller_, RegisterEmptyPacket()); neteq_->InsertEmptyPacket(rtp_header); } TEST_F(NetEqImplTest, EnableRtxHandling) { UseNoMocks(); - use_mock_delay_manager_ = true; + use_mock_neteq_controller_ = true; config_.enable_rtx_handling = true; CreateInstance(); - EXPECT_CALL(*mock_delay_manager_, BufferLimits(_, _)) + EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _)) .Times(1) - .WillOnce(DoAll(SetArgPointee<0>(0), SetArgPointee<1>(0))); + .WillOnce(Return(kNormal)); const int kPayloadLengthSamples = 80; const size_t kPayloadLengthBytes = 2 * kPayloadLengthSamples; // PCM 16-bit. @@ -1423,8 +1400,15 @@ TEST_F(NetEqImplTest, EnableRtxHandling) { // Insert second packet that was sent before the first packet. rtp_header.sequenceNumber -= 1; rtp_header.timestamp -= kPayloadLengthSamples; - EXPECT_CALL(*mock_delay_manager_, - Update(rtp_header.sequenceNumber, rtp_header.timestamp, _)); + EXPECT_CALL(*mock_neteq_controller_, + PacketArrived( + /*last_cng_or_dtmf*/ _, + /*packet_length_samples*/ kPayloadLengthSamples, + /*should_update_stats*/ _, + /*main_sequence_number*/ rtp_header.sequenceNumber, + /*main_timestamp*/ rtp_header.timestamp, + /*fs_hz*/ 8000)); + EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload)); } @@ -1474,7 +1458,7 @@ class NetEqImplTest120ms : public NetEqImplTest { void CreateInstanceWithDelayManagerMock() { UseNoMocks(); - use_mock_delay_manager_ = true; + use_mock_neteq_controller_ = true; CreateInstance(decoder_factory_); EXPECT_TRUE(neteq_->RegisterPayloadType( kPayloadType, SdpAudioFormat("opus", 48000, 2, {{"stereo", "1"}}))); @@ -1551,16 +1535,19 @@ TEST_F(NetEqImplTest120ms, Merge) { Register120msCodec(AudioDecoder::kSpeech); CreateInstanceWithDelayManagerMock(); + EXPECT_CALL(*mock_neteq_controller_, CngOff()).WillRepeatedly(Return(true)); InsertPacket(first_timestamp()); GetFirstPacket(); bool muted; + EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _)) + .WillOnce(Return(kExpand)); EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); InsertPacket(first_timestamp() + 2 * timestamp_diff_between_packets()); - // Delay manager reports a target level which should cause a Merge. - EXPECT_CALL(*mock_delay_manager_, TargetLevel()).WillOnce(Return(-10)); + EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _)) + .WillOnce(Return(kMerge)); EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); EXPECT_EQ(kMerge, neteq_->last_operation_for_test()); @@ -1586,10 +1573,9 @@ TEST_F(NetEqImplTest120ms, FastAccelerate) { GetFirstPacket(); InsertPacket(first_timestamp() + timestamp_diff_between_packets()); - // Delay manager report buffer limit which should cause a FastAccelerate. - EXPECT_CALL(*mock_delay_manager_, BufferLimits(_, _)) + EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _)) .Times(1) - .WillOnce(DoAll(SetArgPointee<0>(0), SetArgPointee<1>(0))); + .WillOnce(Return(kFastAccelerate)); bool muted; EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); @@ -1605,10 +1591,9 @@ TEST_F(NetEqImplTest120ms, PreemptiveExpand) { InsertPacket(first_timestamp() + timestamp_diff_between_packets()); - // Delay manager report buffer limit which should cause a PreemptiveExpand. - EXPECT_CALL(*mock_delay_manager_, BufferLimits(_, _)) + EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _)) .Times(1) - .WillOnce(DoAll(SetArgPointee<0>(100), SetArgPointee<1>(100))); + .WillOnce(Return(kPreemptiveExpand)); bool muted; EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); @@ -1624,10 +1609,9 @@ TEST_F(NetEqImplTest120ms, Accelerate) { InsertPacket(first_timestamp() + timestamp_diff_between_packets()); - // Delay manager report buffer limit which should cause a Accelerate. - EXPECT_CALL(*mock_delay_manager_, BufferLimits(_, _)) + EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _)) .Times(1) - .WillOnce(DoAll(SetArgPointee<0>(1), SetArgPointee<1>(2))); + .WillOnce(Return(kAccelerate)); bool muted; EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted)); diff --git a/modules/audio_coding/neteq/statistics_calculator.cc b/modules/audio_coding/neteq/statistics_calculator.cc index 76f661937c..343ae8a8f8 100644 --- a/modules/audio_coding/neteq/statistics_calculator.cc +++ b/modules/audio_coding/neteq/statistics_calculator.cc @@ -382,16 +382,6 @@ void StatisticsCalculator::GetNetworkStatistics(int fs_hz, Reset(); } -void StatisticsCalculator::PopulateDelayManagerStats( - int ms_per_packet, - const DelayManager& delay_manager, - NetEqNetworkStatistics* stats) { - RTC_DCHECK(stats); - stats->preferred_buffer_size_ms = - (delay_manager.TargetLevel() >> 8) * ms_per_packet; - stats->jitter_peaks_found = delay_manager.PeakFound(); -} - NetEqLifetimeStatistics StatisticsCalculator::GetLifetimeStatistics() const { return lifetime_stats_; } diff --git a/modules/audio_coding/neteq/statistics_calculator.h b/modules/audio_coding/neteq/statistics_calculator.h index a56d0f5a98..a438811c24 100644 --- a/modules/audio_coding/neteq/statistics_calculator.h +++ b/modules/audio_coding/neteq/statistics_calculator.h @@ -116,14 +116,6 @@ class StatisticsCalculator { size_t samples_per_packet, NetEqNetworkStatistics* stats); - // Populates |preferred_buffer_size_ms|, |jitter_peaks_found| and - // |clockdrift_ppm| in |stats|. This is a convenience method, and does not - // strictly have to be in the StatisticsCalculator class, but it makes sense - // since all other stats fields are populated by that class. - static void PopulateDelayManagerStats(int ms_per_packet, - const DelayManager& delay_manager, - NetEqNetworkStatistics* stats); - // Returns a copy of this class's lifetime statistics. These statistics are // never reset. NetEqLifetimeStatistics GetLifetimeStatistics() const;