zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Introduce injectable NetEqController interface.

Browse Source 
This interface is implemented by the DecisionLogic class, which now contains the DelayManager and DelayPeakDetector.

Bug: webrtc:11005
Change-Id: I4fb69fa359e60831cf153e41f101d5b623749380
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/155176
Reviewed-by: Minyue Li <minyue@webrtc.org>
Reviewed-by: Jakob Ivarsson <jakobi@webrtc.org>
Commit-Queue: Ivo Creusen <ivoc@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#29613}
This commit is contained in:
Ivo Creusen
2019-10-24 15:20:39 +02:00
committed by Commit Bot
parent 16cec3be2c
commit 53a31f7db8
17 changed files with 629 additions and 540 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
modules/audio_coding/BUILD.gn
View File

@ -969,6 +969,7 @@ rtc_library("neteq") {
"neteq/nack_tracker.cc",
"neteq/nack_tracker.h",
"neteq/neteq.cc",
"neteq/neteq_controller.h",
"neteq/neteq_impl.cc",
"neteq/neteq_impl.h",
"neteq/normal.cc",
@ -1955,14 +1956,13 @@ if (rtc_include_tests) {
"neteq/expand_unittest.cc",
"neteq/histogram_unittest.cc",
"neteq/merge_unittest.cc",
"neteq/mock/mock_buffer_level_filter.h",
"neteq/mock/mock_decoder_database.h",
"neteq/mock/mock_delay_manager.h",
"neteq/mock/mock_delay_peak_detector.h",
"neteq/mock/mock_dtmf_buffer.h",
"neteq/mock/mock_dtmf_tone_generator.h",
"neteq/mock/mock_expand.h",
"neteq/mock/mock_histogram.h",
"neteq/mock/mock_neteq_controller.h",
"neteq/mock/mock_packet_buffer.h",
"neteq/mock/mock_red_payload_splitter.h",
"neteq/mock/mock_statistics_calculator.h",

186
modules/audio_coding/neteq/decision_logic.cc
View File

@ -16,12 +16,7 @@
#include <string>
#include "absl/types/optional.h"
#include "modules/audio_coding/neteq/buffer_level_filter.h"
#include "modules/audio_coding/neteq/decoder_database.h"
#include "modules/audio_coding/neteq/delay_manager.h"
#include "modules/audio_coding/neteq/expand.h"
#include "modules/audio_coding/neteq/packet_buffer.h"
#include "modules/audio_coding/neteq/sync_buffer.h"
#include "rtc_base/checks.h"
#include "rtc_base/experiments/field_trial_parser.h"
#include "rtc_base/logging.h"
@ -37,48 +32,23 @@ constexpr int kDefaultTargetLevelWindowMs = 100;
namespace webrtc {
DecisionLogic* DecisionLogic::Create(int fs_hz,
size_t output_size_samples,
bool disallow_time_stretching,
DecoderDatabase* decoder_database,
const PacketBuffer& packet_buffer,
DelayManager* delay_manager,
BufferLevelFilter* buffer_level_filter,
const TickTimer* tick_timer) {
return new DecisionLogic(fs_hz, output_size_samples, disallow_time_stretching,
decoder_database, packet_buffer, delay_manager,
buffer_level_filter, tick_timer);
}
DecisionLogic::DecisionLogic(int fs_hz,
size_t output_size_samples,
bool disallow_time_stretching,
DecoderDatabase* decoder_database,
const PacketBuffer& packet_buffer,
DelayManager* delay_manager,
BufferLevelFilter* buffer_level_filter,
const TickTimer* tick_timer)
: decoder_database_(decoder_database),
packet_buffer_(packet_buffer),
delay_manager_(delay_manager),
buffer_level_filter_(buffer_level_filter),
tick_timer_(tick_timer),
cng_state_(kCngOff),
packet_length_samples_(0),
sample_memory_(0),
prev_time_scale_(false),
disallow_time_stretching_(disallow_time_stretching),
DecisionLogic::DecisionLogic(NetEqController::Config config)
: delay_peak_detector_(config.tick_timer, config.enable_rtx_handling),
delay_manager_(DelayManager::Create(config.max_packets_in_buffer,
config.base_min_delay_ms,
config.enable_rtx_handling,
&delay_peak_detector_,
config.tick_timer)),
tick_timer_(config.tick_timer),
disallow_time_stretching_(!config.allow_time_stretching),
timescale_countdown_(
tick_timer_->GetNewCountdown(kMinTimescaleInterval + 1)),
num_consecutive_expands_(0),
time_stretched_cn_samples_(0),
estimate_dtx_delay_("estimate_dtx_delay", false),
time_stretch_cn_("time_stretch_cn", false),
target_level_window_ms_("target_level_window",
kDefaultTargetLevelWindowMs,
0,
absl::nullopt) {
SetSampleRate(fs_hz, output_size_samples);
const std::string field_trial_name =
field_trial::FindFullName("WebRTC-Audio-NetEqDecisionLogicSettings");
ParseFieldTrial(
@ -110,6 +80,8 @@ void DecisionLogic::SoftReset() {
timescale_countdown_ =
tick_timer_->GetNewCountdown(kMinTimescaleInterval + 1);
time_stretched_cn_samples_ = 0;
delay_manager_->Reset();
buffer_level_filter_.Reset();
}
void DecisionLogic::SetSampleRate(int fs_hz, size_t output_size_samples) {
@ -119,69 +91,54 @@ void DecisionLogic::SetSampleRate(int fs_hz, size_t output_size_samples) {
output_size_samples_ = output_size_samples;
}
Operations DecisionLogic::GetDecision(const SyncBuffer& sync_buffer,
const Expand& expand,
size_t decoder_frame_length,
const Packet* next_packet,
Modes prev_mode,
bool play_dtmf,
size_t generated_noise_samples,
Operations DecisionLogic::GetDecision(const NetEqStatus& status,
bool* reset_decoder) {
// If last mode was CNG (or Expand, since this could be covering up for
// a lost CNG packet), remember that CNG is on. This is needed if comfort
// noise is interrupted by DTMF.
if (prev_mode == kModeRfc3389Cng) {
if (status.last_mode == kModeRfc3389Cng) {
cng_state_ = kCngRfc3389On;
} else if (prev_mode == kModeCodecInternalCng) {
} else if (status.last_mode == kModeCodecInternalCng) {
cng_state_ = kCngInternalOn;
}
size_t cur_size_samples =
estimate_dtx_delay_
? packet_buffer_.GetSpanSamples(decoder_frame_length, sample_rate_,
true)
: packet_buffer_.NumSamplesInBuffer(decoder_frame_length);
size_t cur_size_samples = estimate_dtx_delay_
? status.packet_buffer_info.span_samples
: status.packet_buffer_info.num_samples;
prev_time_scale_ =
prev_time_scale_ && (prev_mode == kModeAccelerateSuccess ||
prev_mode == kModeAccelerateLowEnergy ||
prev_mode == kModePreemptiveExpandSuccess ||
prev_mode == kModePreemptiveExpandLowEnergy);
prev_time_scale_ && (status.last_mode == kModeAccelerateSuccess ||
status.last_mode == kModeAccelerateLowEnergy ||
status.last_mode == kModePreemptiveExpandSuccess ||
status.last_mode == kModePreemptiveExpandLowEnergy);
// Do not update buffer history if currently playing CNG since it will bias
// the filtered buffer level.
if (prev_mode != kModeRfc3389Cng && prev_mode != kModeCodecInternalCng &&
!(next_packet && next_packet->frame &&
next_packet->frame->IsDtxPacket() && !estimate_dtx_delay_)) {
if (status.last_mode != kModeRfc3389Cng &&
status.last_mode != kModeCodecInternalCng &&
!(status.next_packet && status.next_packet->is_dtx &&
!estimate_dtx_delay_)) {
FilterBufferLevel(cur_size_samples);
}
// Guard for errors, to avoid getting stuck in error mode.
if (prev_mode == kModeError) {
if (!next_packet) {
if (status.last_mode == kModeError) {
if (!status.next_packet) {
return kExpand;
} else {
return kUndefined; // Use kUndefined to flag for a reset.
}
}
uint32_t target_timestamp = sync_buffer.end_timestamp();
uint32_t available_timestamp = 0;
bool is_cng_packet = false;
if (next_packet) {
available_timestamp = next_packet->timestamp;
is_cng_packet =
decoder_database_->IsComfortNoise(next_packet->payload_type);
}
if (is_cng_packet) {
return CngOperation(prev_mode, target_timestamp, available_timestamp,
generated_noise_samples);
if (status.next_packet && status.next_packet->is_cng) {
return CngOperation(status.last_mode, status.target_timestamp,
status.next_packet->timestamp,
status.generated_noise_samples);
}
// Handle the case with no packet at all available (except maybe DTMF).
if (!next_packet) {
return NoPacket(play_dtmf);
if (!status.next_packet) {
return NoPacket(status.play_dtmf);
}
// If the expand period was very long, reset NetEQ since it is likely that the
@ -198,26 +155,30 @@ Operations DecisionLogic::GetDecision(const SyncBuffer& sync_buffer,
// if the mute factor is low enough (otherwise the expansion was short enough
// to not be noticable).
// Note that the MuteFactor is in Q14, so a value of 16384 corresponds to 1.
size_t current_span = packet_buffer_.GetSpanSamples(
decoder_frame_length, sample_rate_, estimate_dtx_delay_);
if ((prev_mode == kModeExpand || prev_mode == kModeCodecPlc) &&
expand.MuteFactor(0) < 16384 / 2 &&
const size_t current_span =
estimate_dtx_delay_ ? status.packet_buffer_info.span_samples
: status.packet_buffer_info.span_samples_no_dtx;
if ((status.last_mode == kModeExpand || status.last_mode == kModeCodecPlc) &&
status.expand_mutefactor < 16384 / 2 &&
current_span<static_cast<size_t>(delay_manager_->TargetLevel() *
packet_length_samples_ *
kPostponeDecodingLevel / 100)>> 8 &&
!packet_buffer_.ContainsDtxOrCngPacket(decoder_database_)) {
!status.packet_buffer_info.dtx_or_cng) {
return kExpand;
}
const uint32_t five_seconds_samples = static_cast<uint32_t>(5 * sample_rate_);
// Check if the required packet is available.
if (target_timestamp == available_timestamp) {
return ExpectedPacketAvailable(prev_mode, play_dtmf);
} else if (!PacketBuffer::IsObsoleteTimestamp(
available_timestamp, target_timestamp, five_seconds_samples)) {
return FuturePacketAvailable(decoder_frame_length, prev_mode,
target_timestamp, available_timestamp,
play_dtmf, generated_noise_samples);
if (status.target_timestamp == status.next_packet->timestamp) {
return ExpectedPacketAvailable(status.last_mode, status.play_dtmf);
} else if (!PacketBuffer::IsObsoleteTimestamp(status.next_packet->timestamp,
status.target_timestamp,
five_seconds_samples)) {
return FuturePacketAvailable(
status.last_packet_samples, status.last_mode, status.target_timestamp,
status.next_packet->timestamp, status.play_dtmf,
status.generated_noise_samples, status.packet_buffer_info.span_samples,
status.packet_buffer_info.num_packets);
} else {
// This implies that available_timestamp < target_timestamp, which can
// happen when a new stream or codec is received. Signal for a reset.
@ -233,8 +194,40 @@ void DecisionLogic::ExpandDecision(Operations operation) {
}
}
absl::optional<int> DecisionLogic::PacketArrived(bool last_cng_or_dtmf,
size_t packet_length_samples,
bool should_update_stats,
uint16_t main_sequence_number,
uint32_t main_timestamp,
int fs_hz) {
delay_manager_->LastDecodedWasCngOrDtmf(last_cng_or_dtmf);
absl::optional<int> relative_delay;
if (delay_manager_->last_pack_cng_or_dtmf() == 0) {
// Calculate the total speech length carried in each packet.
if (packet_length_samples > 0 &&
packet_length_samples != packet_length_samples_) {
packet_length_samples_ = packet_length_samples;
delay_manager_->SetPacketAudioLength(
rtc::dchecked_cast<int>((1000 * packet_length_samples) / fs_hz));
}
// Update statistics.
if (should_update_stats) {
relative_delay =
delay_manager_->Update(main_sequence_number, main_timestamp, fs_hz);
}
} else if (delay_manager_->last_pack_cng_or_dtmf() == -1) {
// This is first "normal" packet after CNG or DTMF.
// Reset packet time counter and measure time until next packet,
// but don't update statistics.
delay_manager_->set_last_pack_cng_or_dtmf(0);
delay_manager_->ResetPacketIatCount();
}
return relative_delay;
}
void DecisionLogic::FilterBufferLevel(size_t buffer_size_samples) {
buffer_level_filter_->SetTargetBufferLevel(
buffer_level_filter_.SetTargetBufferLevel(
delay_manager_->base_target_level());
int time_stretched_samples = time_stretched_cn_samples_;
@ -243,7 +236,7 @@ void DecisionLogic::FilterBufferLevel(size_t buffer_size_samples) {
timescale_countdown_ = tick_timer_->GetNewCountdown(kMinTimescaleInterval);
}
buffer_level_filter_->Update(buffer_size_samples, time_stretched_samples);
buffer_level_filter_.Update(buffer_size_samples, time_stretched_samples);
prev_time_scale_ = false;
time_stretched_cn_samples_ = 0;
}
@ -307,7 +300,7 @@ Operations DecisionLogic::ExpectedPacketAvailable(Modes prev_mode,
int buffer_level_packets = 0;
if (packet_length_samples_ > 0) {
buffer_level_packets =
((1 << 8) * buffer_level_filter_->filtered_current_level()) /
((1 << 8) * buffer_level_filter_.filtered_current_level()) /
packet_length_samples_;
}
if (buffer_level_packets >= high_limit << 2)
@ -328,7 +321,9 @@ Operations DecisionLogic::FuturePacketAvailable(
uint32_t target_timestamp,
uint32_t available_timestamp,
bool play_dtmf,
size_t generated_noise_samples) {
size_t generated_noise_samples,
size_t span_samples_in_packet_buffer,
size_t num_packets_in_packet_buffer) {
// Required packet is not available, but a future packet is.
// Check if we should continue with an ongoing expand because the new packet
// is too far into the future.
@ -353,9 +348,8 @@ Operations DecisionLogic::FuturePacketAvailable(
if (prev_mode == kModeRfc3389Cng || prev_mode == kModeCodecInternalCng) {
size_t cur_size_samples =
estimate_dtx_delay_
? cur_size_samples = packet_buffer_.GetSpanSamples(
decoder_frame_length, sample_rate_, true)
: packet_buffer_.NumPacketsInBuffer() * decoder_frame_length;
? cur_size_samples = span_samples_in_packet_buffer
: num_packets_in_packet_buffer * decoder_frame_length;
// Target level is in number of packets in Q8.
const size_t target_level_samples =
(delay_manager_->TargetLevel() * packet_length_samples_) >> 8;
@ -411,7 +405,7 @@ bool DecisionLogic::UnderTargetLevel() const {
int buffer_level_packets = 0;
if (packet_length_samples_ > 0) {
buffer_level_packets =
((1 << 8) * buffer_level_filter_->filtered_current_level()) /
((1 << 8) * buffer_level_filter_.filtered_current_level()) /
packet_length_samples_;
}
return buffer_level_packets <= delay_manager_->TargetLevel();

144
modules/audio_coding/neteq/decision_logic.h
View File

@ -11,103 +11,108 @@
#ifndef MODULES_AUDIO_CODING_NETEQ_DECISION_LOGIC_H_
#define MODULES_AUDIO_CODING_NETEQ_DECISION_LOGIC_H_
#include "modules/audio_coding/neteq/buffer_level_filter.h"
#include "modules/audio_coding/neteq/defines.h"
#include "modules/audio_coding/neteq/delay_manager.h"
#include "modules/audio_coding/neteq/delay_peak_detector.h"
#include "modules/audio_coding/neteq/neteq_controller.h"
#include "modules/audio_coding/neteq/tick_timer.h"
#include "rtc_base/constructor_magic.h"
#include "rtc_base/experiments/field_trial_parser.h"
namespace webrtc {
// Forward declarations.
class BufferLevelFilter;
class DecoderDatabase;
class DelayManager;
class Expand;
class PacketBuffer;
class SyncBuffer;
struct Packet;
// This is the class for the decision tree implementation.
class DecisionLogic final {
class DecisionLogic : public NetEqController {
public:
// Static factory function which creates different types of objects depending
// on the |playout_mode|.
static DecisionLogic* Create(int fs_hz,
size_t output_size_samples,
bool disallow_time_stretching,
DecoderDatabase* decoder_database,
const PacketBuffer& packet_buffer,
DelayManager* delay_manager,
BufferLevelFilter* buffer_level_filter,
const TickTimer* tick_timer);
static const int kReinitAfterExpands = 100;
static const int kMaxWaitForPacket = 10;
// Constructor.
DecisionLogic(int fs_hz,
size_t output_size_samples,
bool disallow_time_stretching,
DecoderDatabase* decoder_database,
const PacketBuffer& packet_buffer,
DelayManager* delay_manager,
BufferLevelFilter* buffer_level_filter,
const TickTimer* tick_timer);
DecisionLogic(NetEqController::Config config);
~DecisionLogic();
~DecisionLogic() override;
// Resets object to a clean state.
void Reset();
void Reset() override;
// Resets parts of the state. Typically done when switching codecs.
void SoftReset();
void SoftReset() override;
// Sets the sample rate and the output block size.
void SetSampleRate(int fs_hz, size_t output_size_samples);
void SetSampleRate(int fs_hz, size_t output_size_samples) override;
// Returns the operation that should be done next. |sync_buffer| and |expand|
// are provided for reference. |decoder_frame_length| is the number of samples
// Given info about the latest received packet, and current jitter buffer
// status, returns the operation. |target_timestamp| and |expand_mutefactor|
// are provided for reference. |last_packet_samples| is the number of samples
// obtained from the last decoded frame. If there is a packet available, it
// should be supplied in |next_packet|; otherwise it should be NULL. The mode
// should be supplied in |packet|; otherwise it should be NULL. The mode
// resulting from the last call to NetEqImpl::GetAudio is supplied in
// |prev_mode|. If there is a DTMF event to play, |play_dtmf| should be set to
// |last_mode|. If there is a DTMF event to play, |play_dtmf| should be set to
// true. The output variable |reset_decoder| will be set to true if a reset is
// required; otherwise it is left unchanged (i.e., it can remain true if it
// was true before the call). This method end with calling
// GetDecisionSpecialized to get the actual return value.
Operations GetDecision(const SyncBuffer& sync_buffer,
const Expand& expand,
size_t decoder_frame_length,
const Packet* next_packet,
Modes prev_mode,
bool play_dtmf,
size_t generated_noise_samples,
bool* reset_decoder);
// was true before the call).
Operations GetDecision(const NetEqStatus& status,
bool* reset_decoder) override;
// These methods test the |cng_state_| for different conditions.
bool CngRfc3389On() const { return cng_state_ == kCngRfc3389On; }
bool CngOff() const { return cng_state_ == kCngOff; }
bool CngRfc3389On() const override { return cng_state_ == kCngRfc3389On; }
bool CngOff() const override { return cng_state_ == kCngOff; }
// Resets the |cng_state_| to kCngOff.
void SetCngOff() { cng_state_ = kCngOff; }
void SetCngOff() override { cng_state_ = kCngOff; }
// Reports back to DecisionLogic whether the decision to do expand remains or
// not. Note that this is necessary, since an expand decision can be changed
// to kNormal in NetEqImpl::GetDecision if there is still enough data in the
// sync buffer.
void ExpandDecision(Operations operation);
void ExpandDecision(Operations operation) override;
// Adds |value| to |sample_memory_|.
void AddSampleMemory(int32_t value) { sample_memory_ += value; }
void AddSampleMemory(int32_t value) override { sample_memory_ += value; }
int TargetLevelMs() override {
return ((delay_manager_->TargetLevel() * packet_length_samples_) >> 8) /
rtc::CheckedDivExact(sample_rate_, 1000);
}
absl::optional<int> PacketArrived(bool last_cng_or_dtmf,
size_t packet_length_samples,
bool should_update_stats,
uint16_t main_sequence_number,
uint32_t main_timestamp,
int fs_hz) override;
void RegisterEmptyPacket() override { delay_manager_->RegisterEmptyPacket(); }
bool SetMaximumDelay(int delay_ms) override {
return delay_manager_->SetMaximumDelay(delay_ms);
}
bool SetMinimumDelay(int delay_ms) override {
return delay_manager_->SetMinimumDelay(delay_ms);
}
bool SetBaseMinimumDelay(int delay_ms) override {
return delay_manager_->SetBaseMinimumDelay(delay_ms);
}
int GetBaseMinimumDelay() const override {
return delay_manager_->GetBaseMinimumDelay();
}
bool PeakFound() const override { return delay_manager_->PeakFound(); }
virtual int GetFilteredBufferLevel() const override {
return buffer_level_filter_.filtered_current_level();
}
// Accessors and mutators.
void set_sample_memory(int32_t value) { sample_memory_ = value; }
size_t noise_fast_forward() const { return noise_fast_forward_; }
size_t packet_length_samples() const { return packet_length_samples_; }
void set_packet_length_samples(size_t value) {
void set_sample_memory(int32_t value) override { sample_memory_ = value; }
size_t noise_fast_forward() const override { return noise_fast_forward_; }
size_t packet_length_samples() const override {
return packet_length_samples_;
}
void set_packet_length_samples(size_t value) override {
packet_length_samples_ = value;
}
void set_prev_time_scale(bool value) { prev_time_scale_ = value; }
void set_prev_time_scale(bool value) override { prev_time_scale_ = value; }
private:
// The value 5 sets maximum time-stretch rate to about 100 ms/s.
@ -140,7 +145,9 @@ class DecisionLogic final {
uint32_t target_timestamp,
uint32_t available_timestamp,
bool play_dtmf,
size_t generated_noise_samples);
size_t generated_noise_samples,
size_t span_samples_in_packet_buffer,
size_t num_packets_in_packet_buffer);
// Checks if enough time has elapsed since the last successful timescale
// operation was done (i.e., accelerate or preemptive expand).
@ -163,23 +170,22 @@ class DecisionLogic final {
// Checks if num_consecutive_expands_ >= kMaxWaitForPacket.
bool MaxWaitForPacket() const;
DecoderDatabase* decoder_database_;
const PacketBuffer& packet_buffer_;
DelayManager* delay_manager_;
BufferLevelFilter* buffer_level_filter_;
DelayPeakDetector delay_peak_detector_;
std::unique_ptr<DelayManager> delay_manager_;
BufferLevelFilter buffer_level_filter_;
const TickTimer* tick_timer_;
int sample_rate_;
size_t output_size_samples_;
CngState cng_state_; // Remember if comfort noise is interrupted by other
// event (e.g., DTMF).
CngState cng_state_ = kCngOff; // Remember if comfort noise is interrupted by
// other event (e.g., DTMF).
size_t noise_fast_forward_ = 0;
size_t packet_length_samples_;
int sample_memory_;
bool prev_time_scale_;
size_t packet_length_samples_ = 0;
int sample_memory_ = 0;
bool prev_time_scale_ = false;
bool disallow_time_stretching_;
std::unique_ptr<TickTimer::Countdown> timescale_countdown_;
int num_consecutive_expands_;
int time_stretched_cn_samples_;
int num_consecutive_expands_ = 0;
int time_stretched_cn_samples_ = 0;
FieldTrialParameter<bool> estimate_dtx_delay_;
FieldTrialParameter<bool> time_stretch_cn_;
FieldTrialConstrained<int> target_level_window_ms_;

16
modules/audio_coding/neteq/decision_logic_unittest.cc
View File

@ -16,6 +16,7 @@
#include "modules/audio_coding/neteq/decoder_database.h"
#include "modules/audio_coding/neteq/delay_manager.h"
#include "modules/audio_coding/neteq/delay_peak_detector.h"
#include "modules/audio_coding/neteq/neteq_controller.h"
#include "modules/audio_coding/neteq/packet_buffer.h"
#include "modules/audio_coding/neteq/statistics_calculator.h"
#include "modules/audio_coding/neteq/tick_timer.h"
@ -32,14 +33,15 @@ TEST(DecisionLogic, CreateAndDestroy) {
TickTimer tick_timer;
StatisticsCalculator stats;
PacketBuffer packet_buffer(10, &tick_timer);
DelayPeakDetector delay_peak_detector(&tick_timer, false);
auto delay_manager = DelayManager::Create(240, 0, false, &delay_peak_detector,
&tick_timer, &stats);
BufferLevelFilter buffer_level_filter;
DecisionLogic* logic = DecisionLogic::Create(
fs_hz, output_size_samples, false, &decoder_database, packet_buffer,
delay_manager.get(), &buffer_level_filter, &tick_timer);
delete logic;
NetEqController::Config config;
config.tick_timer = &tick_timer;
config.base_min_delay_ms = 0;
config.max_packets_in_buffer = 240;
config.enable_rtx_handling = false;
config.allow_time_stretching = true;
auto logic = std::make_unique<DecisionLogic>(std::move(config));
logic->SetSampleRate(fs_hz, output_size_samples);
}
// TODO(hlundin): Write more tests.

28
modules/audio_coding/neteq/delay_manager.cc
View File

@ -21,7 +21,6 @@
#include "modules/audio_coding/neteq/delay_peak_detector.h"
#include "modules/audio_coding/neteq/histogram.h"
#include "modules/audio_coding/neteq/statistics_calculator.h"
#include "modules/include/module_common_types_public.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
@ -108,7 +107,6 @@ DelayManager::DelayManager(size_t max_packets_in_buffer,
bool enable_rtx_handling,
DelayPeakDetector* peak_detector,
const TickTimer* tick_timer,
StatisticsCalculator* statistics,
std::unique_ptr<Histogram> histogram)
: first_packet_received_(false),
max_packets_in_buffer_(max_packets_in_buffer),
@ -116,7 +114,6 @@ DelayManager::DelayManager(size_t max_packets_in_buffer,
histogram_quantile_(histogram_quantile),
histogram_mode_(histogram_mode),
tick_timer_(tick_timer),
statistics_(statistics),
base_minimum_delay_ms_(base_minimum_delay_ms),
effective_minimum_delay_ms_(base_minimum_delay_ms),
base_target_level_(4), // In Q0 domain.
@ -144,8 +141,7 @@ std::unique_ptr<DelayManager> DelayManager::Create(
int base_minimum_delay_ms,
bool enable_rtx_handling,
DelayPeakDetector* peak_detector,
const TickTimer* tick_timer,
StatisticsCalculator* statistics) {
const TickTimer* tick_timer) {
const HistogramMode mode = RELATIVE_ARRIVAL_DELAY;
DelayHistogramConfig config = GetDelayHistogramConfig();
const int quantile = config.quantile;
@ -153,17 +149,16 @@ std::unique_ptr<DelayManager> DelayManager::Create(
kDelayBuckets, config.forget_factor, config.start_forget_weight);
return std::make_unique<DelayManager>(
max_packets_in_buffer, base_minimum_delay_ms, quantile, mode,
enable_rtx_handling, peak_detector, tick_timer, statistics,
std::move(histogram));
enable_rtx_handling, peak_detector, tick_timer, std::move(histogram));
}
DelayManager::~DelayManager() {}
int DelayManager::Update(uint16_t sequence_number,
uint32_t timestamp,
int sample_rate_hz) {
absl::optional<int> DelayManager::Update(uint16_t sequence_number,
uint32_t timestamp,
int sample_rate_hz) {
if (sample_rate_hz <= 0) {
return -1;
return absl::nullopt;
}
if (!first_packet_received_) {
@ -172,7 +167,7 @@ int DelayManager::Update(uint16_t sequence_number,
last_seq_no_ = sequence_number;
last_timestamp_ = timestamp;
first_packet_received_ = true;
return 0;
return absl::nullopt;
}
// Try calculating packet length from current and previous timestamps.
@ -191,6 +186,7 @@ int DelayManager::Update(uint16_t sequence_number,
}
bool reordered = false;
absl::optional<int> relative_delay;
if (packet_len_ms > 0) {
// Cannot update statistics unless |packet_len_ms| is valid.
@ -215,18 +211,16 @@ int DelayManager::Update(uint16_t sequence_number,
}
int iat_delay = iat_ms - packet_len_ms;
int relative_delay;
if (reordered) {
relative_delay = std::max(iat_delay, 0);
} else {
UpdateDelayHistory(iat_delay, timestamp, sample_rate_hz);
relative_delay = CalculateRelativePacketArrivalDelay();
}
statistics_->RelativePacketArrivalDelay(relative_delay);
switch (histogram_mode_) {
case RELATIVE_ARRIVAL_DELAY: {
const int index = relative_delay / kBucketSizeMs;
const int index = relative_delay.value() / kBucketSizeMs;
if (index < histogram_->NumBuckets()) {
// Maximum delay to register is 2000 ms.
histogram_->Add(index);
@ -250,14 +244,14 @@ int DelayManager::Update(uint16_t sequence_number,
if (enable_rtx_handling_ && reordered &&
num_reordered_packets_ < kMaxReorderedPackets) {
++num_reordered_packets_;
return 0;
return relative_delay;
}
num_reordered_packets_ = 0;
// Prepare for next packet arrival.
packet_iat_stopwatch_ = tick_timer_->GetNewStopwatch();
last_seq_no_ = sequence_number;
last_timestamp_ = timestamp;
return 0;
return relative_delay;
}
void DelayManager::UpdateDelayHistory(int iat_delay_ms,

14
modules/audio_coding/neteq/delay_manager.h
View File

@ -18,7 +18,6 @@
#include "absl/types/optional.h"
#include "modules/audio_coding/neteq/histogram.h"
#include "modules/audio_coding/neteq/statistics_calculator.h"
#include "modules/audio_coding/neteq/tick_timer.h"
#include "rtc_base/constructor_magic.h"
@ -41,7 +40,6 @@ class DelayManager {
bool enable_rtx_handling,
DelayPeakDetector* peak_detector,
const TickTimer* tick_timer,
StatisticsCalculator* statistics,
std::unique_ptr<Histogram> histogram);
// Create a DelayManager object. Notify the delay manager that the packet
@ -53,8 +51,7 @@ class DelayManager {
int base_minimum_delay_ms,
bool enable_rtx_handling,
DelayPeakDetector* peak_detector,
const TickTimer* tick_timer,
StatisticsCalculator* statistics);
const TickTimer* tick_timer);
virtual ~DelayManager();
@ -62,10 +59,10 @@ class DelayManager {
// |sequence_number| and |timestamp| from the RTP header. This updates the
// inter-arrival time histogram and other statistics, as well as the
// associated DelayPeakDetector. A new target buffer level is calculated.
// Returns 0 on success, -1 on failure (invalid sample rate).
virtual int Update(uint16_t sequence_number,
uint32_t timestamp,
int sample_rate_hz);
// Returns the relative delay if it can be calculated.
virtual absl::optional<int> Update(uint16_t sequence_number,
uint32_t timestamp,
int sample_rate_hz);
// Calculates a new target buffer level. Called from the Update() method.
// Sets target_level_ (in Q8) and returns the same value. Also calculates
@ -168,7 +165,6 @@ class DelayManager {
const int histogram_quantile_;
const HistogramMode histogram_mode_;
const TickTimer* tick_timer_;
StatisticsCalculator* statistics_;
int base_minimum_delay_ms_;
// Provides delay which is used by LimitTargetLevel as lower bound on target
// delay.

37
modules/audio_coding/neteq/delay_manager_unittest.cc
View File

@ -50,7 +50,7 @@ class DelayManagerTest : public ::testing::Test {
virtual void TearDown();
void RecreateDelayManager();
void SetPacketAudioLength(int lengt_ms);
void InsertNextPacket();
absl::optional<int> InsertNextPacket();
void IncreaseTime(int inc_ms);
std::unique_ptr<DelayManager> dm_;
@ -84,11 +84,10 @@ void DelayManagerTest::RecreateDelayManager() {
dm_ = std::make_unique<DelayManager>(
kMaxNumberOfPackets, kMinDelayMs, kDefaultHistogramQuantile,
histogram_mode_, enable_rtx_handling_, &detector_, &tick_timer_,
&stats_, std::move(histogram));
std::move(histogram));
} else {
dm_ = DelayManager::Create(kMaxNumberOfPackets, kMinDelayMs,
enable_rtx_handling_, &detector_, &tick_timer_,
&stats_);
enable_rtx_handling_, &detector_, &tick_timer_);
}
}
@ -97,10 +96,11 @@ void DelayManagerTest::SetPacketAudioLength(int lengt_ms) {
dm_->SetPacketAudioLength(lengt_ms);
}
void DelayManagerTest::InsertNextPacket() {
EXPECT_EQ(0, dm_->Update(seq_no_, ts_, kFs));
absl::optional<int> DelayManagerTest::InsertNextPacket() {
auto relative_delay = dm_->Update(seq_no_, ts_, kFs);
seq_no_ += 1;
ts_ += kTsIncrement;
return relative_delay;
}
void DelayManagerTest::IncreaseTime(int inc_ms) {
@ -416,11 +416,11 @@ TEST_F(DelayManagerTest, EnableRtxHandling) {
// Insert reordered packet.
EXPECT_CALL(*mock_histogram_, Add(2));
EXPECT_EQ(0, dm_->Update(seq_no_ - 3, ts_ - 3 * kFrameSizeMs, kFs));
dm_->Update(seq_no_ - 3, ts_ - 3 * kFrameSizeMs, kFs);
// Insert another reordered packet.
EXPECT_CALL(*mock_histogram_, Add(1));
EXPECT_EQ(0, dm_->Update(seq_no_ - 2, ts_ - 2 * kFrameSizeMs, kFs));
dm_->Update(seq_no_ - 2, ts_ - 2 * kFrameSizeMs, kFs);
// Insert the next packet in order and verify that the inter-arrival time is
// estimated correctly.
@ -475,7 +475,7 @@ TEST_F(DelayManagerTest, EmptyPacketsNotReported) {
TEST_F(DelayManagerTest, Failures) {
// Wrong sample rate.
EXPECT_EQ(-1, dm_->Update(0, 0, -1));
EXPECT_EQ(absl::nullopt, dm_->Update(0, 0, -1));
// Wrong packet size.
EXPECT_EQ(-1, dm_->SetPacketAudioLength(0));
EXPECT_EQ(-1, dm_->SetPacketAudioLength(-1));
@ -550,14 +550,14 @@ TEST_F(DelayManagerTest, RelativeArrivalDelayMode) {
IncreaseTime(2 * kFrameSizeMs);
EXPECT_CALL(*mock_histogram_, Add(1)); // 20ms delayed.
EXPECT_EQ(0, dm_->Update(seq_no_, ts_, kFs));
dm_->Update(seq_no_, ts_, kFs);
IncreaseTime(2 * kFrameSizeMs);
EXPECT_CALL(*mock_histogram_, Add(2)); // 40ms delayed.
EXPECT_EQ(0, dm_->Update(seq_no_ + 1, ts_ + kTsIncrement, kFs));
dm_->Update(seq_no_ + 1, ts_ + kTsIncrement, kFs);
EXPECT_CALL(*mock_histogram_, Add(1)); // Reordered, 20ms delayed.
EXPECT_EQ(0, dm_->Update(seq_no_, ts_, kFs));
dm_->Update(seq_no_, ts_, kFs);
}
TEST_F(DelayManagerTest, MaxDelayHistory) {
@ -579,20 +579,17 @@ TEST_F(DelayManagerTest, MaxDelayHistory) {
IncreaseTime(kMaxHistoryMs + kFrameSizeMs);
ts_ += kFs * kMaxHistoryMs / 1000;
EXPECT_CALL(*mock_histogram_, Add(0)); // Not delayed.
EXPECT_EQ(0, dm_->Update(seq_no_, ts_, kFs));
dm_->Update(seq_no_, ts_, kFs);
}
TEST_F(DelayManagerTest, RelativeArrivalDelayStatistic) {
SetPacketAudioLength(kFrameSizeMs);
InsertNextPacket();
EXPECT_EQ(absl::nullopt, InsertNextPacket());
IncreaseTime(kFrameSizeMs);
EXPECT_CALL(stats_, RelativePacketArrivalDelay(0));
InsertNextPacket();
EXPECT_EQ(0, InsertNextPacket());
IncreaseTime(2 * kFrameSizeMs);
EXPECT_CALL(stats_, RelativePacketArrivalDelay(20));
InsertNextPacket();
EXPECT_EQ(20, InsertNextPacket());
}
TEST_F(DelayManagerTest, DecelerationTargetLevelOffset) {

31
modules/audio_coding/neteq/mock/mock_buffer_level_filter.h
View File

@ -1,31 +0,0 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_BUFFER_LEVEL_FILTER_H_
#define MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_BUFFER_LEVEL_FILTER_H_
#include "modules/audio_coding/neteq/buffer_level_filter.h"
#include "test/gmock.h"
namespace webrtc {
class MockBufferLevelFilter : public BufferLevelFilter {
public:
virtual ~MockBufferLevelFilter() { Die(); }
MOCK_METHOD0(Die, void());
MOCK_METHOD0(Reset, void());
MOCK_METHOD2(Update,
void(size_t buffer_size_samples, int time_stretched_samples));
MOCK_METHOD1(SetTargetBufferLevel, void(int target_buffer_level));
MOCK_CONST_METHOD0(filtered_current_level, int());
};
} // namespace webrtc
#endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_BUFFER_LEVEL_FILTER_H_

65
modules/audio_coding/neteq/mock/mock_delay_manager.h
View File

@ -1,65 +0,0 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DELAY_MANAGER_H_
#define MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DELAY_MANAGER_H_
#include <algorithm>
#include "modules/audio_coding/neteq/delay_manager.h"
#include "modules/audio_coding/neteq/histogram.h"
#include "modules/audio_coding/neteq/statistics_calculator.h"
#include "test/gmock.h"
namespace webrtc {
class MockDelayManager : public DelayManager {
public:
MockDelayManager(size_t max_packets_in_buffer,
int base_min_target_delay_ms,
int histogram_quantile,
HistogramMode histogram_mode,
bool enable_rtx_handling,
DelayPeakDetector* peak_detector,
const TickTimer* tick_timer,
StatisticsCalculator* stats,
std::unique_ptr<Histogram> histogram)
: DelayManager(max_packets_in_buffer,
base_min_target_delay_ms,
histogram_quantile,
histogram_mode,
enable_rtx_handling,
peak_detector,
tick_timer,
stats,
std::move(histogram)) {}
virtual ~MockDelayManager() { Die(); }
MOCK_METHOD0(Die, void());
MOCK_METHOD3(Update,
int(uint16_t sequence_number,
uint32_t timestamp,
int sample_rate_hz));
MOCK_METHOD2(CalculateTargetLevel, int(int iat_packets, bool reordered));
MOCK_METHOD1(SetPacketAudioLength, int(int length_ms));
MOCK_METHOD0(Reset, void());
MOCK_CONST_METHOD0(PeakFound, bool());
MOCK_METHOD0(ResetPacketIatCount, void());
MOCK_CONST_METHOD2(BufferLimits, void(int* lower_limit, int* higher_limit));
MOCK_METHOD1(SetBaseMinimumDelay, bool(int delay_ms));
MOCK_CONST_METHOD0(GetBaseMinimumDelay, int());
MOCK_CONST_METHOD0(TargetLevel, int());
MOCK_METHOD0(RegisterEmptyPacket, void());
MOCK_CONST_METHOD0(base_target_level, int());
MOCK_CONST_METHOD0(last_pack_cng_or_dtmf, int());
MOCK_METHOD1(set_last_pack_cng_or_dtmf, void(int value));
};
} // namespace webrtc
#endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DELAY_MANAGER_H_

1
modules/audio_coding/neteq/mock/mock_delay_peak_detector.h
View File

@ -33,4 +33,5 @@ class MockDelayPeakDetector : public DelayPeakDetector {
};
} // namespace webrtc
#endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_DELAY_PEAK_DETECTOR_H_

65
modules/audio_coding/neteq/mock/mock_neteq_controller.h Normal file
View File

@ -0,0 +1,65 @@
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_NETEQ_CONTROLLER_H_
#define MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_NETEQ_CONTROLLER_H_
#include "modules/audio_coding/neteq/neteq_controller.h"
#include "test/gmock.h"
namespace webrtc {
class MockNetEqController : public NetEqController {
public:
MockNetEqController() = default;
virtual ~MockNetEqController() { Die(); }
MOCK_METHOD0(Die, void());
MOCK_METHOD0(Reset, void());
MOCK_METHOD0(SoftReset, void());
MOCK_METHOD2(GetDecision,
Operations(const NetEqStatus& neteq_status,
bool* reset_decoder));
MOCK_METHOD6(Update,
void(uint16_t sequence_number,
uint32_t timestamp,
uint32_t last_played_out_timestamp,
bool new_codec,
bool cng_or_dtmf,
size_t packet_length_samples));
MOCK_METHOD0(RegisterEmptyPacket, void());
MOCK_METHOD2(SetSampleRate, void(int fs_hz, size_t output_size_samples));
MOCK_METHOD1(SetMaximumDelay, bool(int delay_ms));
MOCK_METHOD1(SetMinimumDelay, bool(int delay_ms));
MOCK_METHOD1(SetBaseMinimumDelay, bool(int delay_ms));
MOCK_CONST_METHOD0(GetBaseMinimumDelay, int());
MOCK_CONST_METHOD0(CngRfc3389On, bool());
MOCK_CONST_METHOD0(CngOff, bool());
MOCK_METHOD0(SetCngOff, void());
MOCK_METHOD1(ExpandDecision, void(Operations operation));
MOCK_METHOD1(AddSampleMemory, void(int32_t value));
MOCK_METHOD0(TargetLevelMs, int());
MOCK_METHOD6(PacketArrived,
absl::optional<int>(bool last_cng_or_dtmf,
size_t packet_length_samples,
bool should_update_stats,
uint16_t main_sequence_number,
uint32_t main_timestamp,
int fs_hz));
MOCK_CONST_METHOD0(PeakFound, bool());
MOCK_CONST_METHOD0(GetFilteredBufferLevel, int());
MOCK_METHOD1(set_sample_memory, void(int32_t value));
MOCK_CONST_METHOD0(noise_fast_forward, size_t());
MOCK_CONST_METHOD0(packet_length_samples, size_t());
MOCK_METHOD1(set_packet_length_samples, void(size_t value));
MOCK_METHOD1(set_prev_time_scale, void(bool value));
};
} // namespace webrtc
#endif // MODULES_AUDIO_CODING_NETEQ_MOCK_MOCK_NETEQ_CONTROLLER_H_

178
modules/audio_coding/neteq/neteq_controller.h Normal file
View File

@ -0,0 +1,178 @@
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_AUDIO_CODING_NETEQ_NETEQ_CONTROLLER_H_
#define MODULES_AUDIO_CODING_NETEQ_NETEQ_CONTROLLER_H_
#include <cstddef>
#include <cstdint>
#include <functional>
#include <memory>
#include "absl/types/optional.h"
#include "modules/audio_coding/neteq/defines.h"
#include "modules/audio_coding/neteq/tick_timer.h"
namespace webrtc {
// Decides the actions that NetEq should take. This affects the behavior of the
// jitter buffer, and how it reacts to network conditions.
// This class will undergo substantial refactoring in the near future, and the
// API is expected to undergo significant changes. A target API is given below:
//
// class NetEqController {
// public:
// // Resets object to a clean state.
// void Reset();
// // Given NetEq status, make a decision.
// Operation GetDecision(NetEqStatus neteq_status);
// // Register every packet received.
// void RegisterPacket(PacketInfo packet_info);
// // Register empty packet.
// void RegisterEmptyPacket();
// // Register a codec switching.
// void CodecSwithed();
// // Sets the sample rate.
// void SetSampleRate(int fs_hz);
// // Sets the packet length in samples.
// void SetPacketLengthSamples();
// // Sets maximum delay.
// void SetMaximumDelay(int delay_ms);
// // Sets mininum delay.
// void SetMinimumDelay(int delay_ms);
// // Sets base mininum delay.
// void SetBaseMinimumDelay(int delay_ms);
// // Gets target buffer level.
// int GetTargetBufferLevelMs() const;
// // Gets filtered buffer level.
// int GetFilteredBufferLevel() const;
// // Gets base minimum delay.
// int GetBaseMinimumDelay() const;
// }
class NetEqController {
public:
// This struct is used to create a NetEqController.
struct Config {
bool allow_time_stretching;
bool enable_rtx_handling;
int max_packets_in_buffer;
int base_min_delay_ms;
TickTimer* tick_timer;
};
struct PacketInfo {
uint32_t timestamp;
bool is_dtx;
bool is_cng;
};
struct PacketBufferInfo {
bool dtx_or_cng;
size_t num_samples;
size_t span_samples;
size_t span_samples_no_dtx;
size_t num_packets;
};
struct NetEqStatus {
uint32_t target_timestamp;
int16_t expand_mutefactor;
size_t last_packet_samples;
absl::optional<PacketInfo> next_packet;
Modes last_mode;
bool play_dtmf;
size_t generated_noise_samples;
PacketBufferInfo packet_buffer_info;
};
virtual ~NetEqController() = default;
// Resets object to a clean state.
virtual void Reset() = 0;
// Resets parts of the state. Typically done when switching codecs.
virtual void SoftReset() = 0;
// Given info about the latest received packet, and current jitter buffer
// status, returns the operation. |target_timestamp| and |expand_mutefactor|
// are provided for reference. |last_packet_samples| is the number of samples
// obtained from the last decoded frame. If there is a packet available, it
// should be supplied in |packet|. The mode resulting from the last call to
// NetEqImpl::GetAudio is supplied in |last_mode|. If there is a DTMF event to
// play, |play_dtmf| should be set to true. The output variable
// |reset_decoder| will be set to true if a reset is required; otherwise it is
// left unchanged (i.e., it can remain true if it was true before the call).
virtual Operations GetDecision(const NetEqStatus& status,
bool* reset_decoder) = 0;
// Inform NetEqController that an empty packet has arrived.
virtual void RegisterEmptyPacket() = 0;
// Sets the sample rate and the output block size.
virtual void SetSampleRate(int fs_hz, size_t output_size_samples) = 0;
// Sets a minimum or maximum delay in millisecond.
// Returns true if the delay bound is successfully applied, otherwise false.
virtual bool SetMaximumDelay(int delay_ms) = 0;
virtual bool SetMinimumDelay(int delay_ms) = 0;
// Sets a base minimum delay in milliseconds for packet buffer. The effective
// minimum delay can't be lower than base minimum delay, even if a lower value
// is set using SetMinimumDelay.
// Returns true if the base minimum is successfully applied, otherwise false.
virtual bool SetBaseMinimumDelay(int delay_ms) = 0;
virtual int GetBaseMinimumDelay() const = 0;
// These methods test the |cng_state_| for different conditions.
virtual bool CngRfc3389On() const = 0;
virtual bool CngOff() const = 0;
// Resets the |cng_state_| to kCngOff.
virtual void SetCngOff() = 0;
// Reports back to DecisionLogic whether the decision to do expand remains or
// not. Note that this is necessary, since an expand decision can be changed
// to kNormal in NetEqImpl::GetDecision if there is still enough data in the
// sync buffer.
virtual void ExpandDecision(Operations operation) = 0;
// Adds |value| to |sample_memory_|.
virtual void AddSampleMemory(int32_t value) = 0;
// Returns the target buffer level in ms.
virtual int TargetLevelMs() = 0;
// Notify the NetEqController that a packet has arrived. Returns the relative
// arrival delay, if it can be computed.
virtual absl::optional<int> PacketArrived(bool last_cng_or_dtmf,
size_t packet_length_samples,
bool should_update_stats,
uint16_t main_sequence_number,
uint32_t main_timestamp,
int fs_hz) = 0;
// Returns true if a peak was found.
virtual bool PeakFound() const = 0;
// Get the filtered buffer level in samples.
virtual int GetFilteredBufferLevel() const = 0;
// Accessors and mutators.
virtual void set_sample_memory(int32_t value) = 0;
virtual size_t noise_fast_forward() const = 0;
virtual size_t packet_length_samples() const = 0;
virtual void set_packet_length_samples(size_t value) = 0;
virtual void set_prev_time_scale(bool value) = 0;
};
} // namespace webrtc
#endif // MODULES_AUDIO_CODING_NETEQ_NETEQ_CONTROLLER_H_

197
modules/audio_coding/neteq/neteq_impl.cc
View File

@ -25,13 +25,10 @@
#include "modules/audio_coding/codecs/cng/webrtc_cng.h"
#include "modules/audio_coding/neteq/accelerate.h"
#include "modules/audio_coding/neteq/background_noise.h"
#include "modules/audio_coding/neteq/buffer_level_filter.h"
#include "modules/audio_coding/neteq/comfort_noise.h"
#include "modules/audio_coding/neteq/decision_logic.h"
#include "modules/audio_coding/neteq/decoder_database.h"
#include "modules/audio_coding/neteq/defines.h"
#include "modules/audio_coding/neteq/delay_manager.h"
#include "modules/audio_coding/neteq/delay_peak_detector.h"
#include "modules/audio_coding/neteq/dtmf_buffer.h"
#include "modules/audio_coding/neteq/dtmf_tone_generator.h"
#include "modules/audio_coding/neteq/expand.h"
@ -57,6 +54,24 @@
#include "system_wrappers/include/clock.h"
namespace webrtc {
namespace {
std::unique_ptr<NetEqController> CreateNetEqController(
int base_min_delay,
int max_packets_in_buffer,
bool enable_rtx_handling,
bool allow_time_stretching,
TickTimer* tick_timer) {
NetEqController::Config config;
config.base_min_delay_ms = base_min_delay;
config.max_packets_in_buffer = max_packets_in_buffer;
config.enable_rtx_handling = enable_rtx_handling;
config.allow_time_stretching = allow_time_stretching;
config.tick_timer = tick_timer;
return std::make_unique<DecisionLogic>(std::move(config));
}
} // namespace
NetEqImpl::Dependencies::Dependencies(
const NetEq::Config& config,
@ -65,21 +80,18 @@ NetEqImpl::Dependencies::Dependencies(
: clock(clock),
tick_timer(new TickTimer),
stats(new StatisticsCalculator),
buffer_level_filter(new BufferLevelFilter),
decoder_database(
new DecoderDatabase(decoder_factory, config.codec_pair_id)),
delay_peak_detector(
new DelayPeakDetector(tick_timer.get(), config.enable_rtx_handling)),
delay_manager(DelayManager::Create(config.max_packets_in_buffer,
config.min_delay_ms,
config.enable_rtx_handling,
delay_peak_detector.get(),
tick_timer.get(),
stats.get())),
dtmf_buffer(new DtmfBuffer(config.sample_rate_hz)),
dtmf_tone_generator(new DtmfToneGenerator),
packet_buffer(
new PacketBuffer(config.max_packets_in_buffer, tick_timer.get())),
neteq_controller(
CreateNetEqController(config.min_delay_ms,
config.max_packets_in_buffer,
config.enable_rtx_handling,
!config.for_test_no_time_stretching,
tick_timer.get())),
red_payload_splitter(new RedPayloadSplitter),
timestamp_scaler(new TimestampScaler(*decoder_database)),
accelerate_factory(new AccelerateFactory),
@ -93,10 +105,7 @@ NetEqImpl::NetEqImpl(const NetEq::Config& config,
bool create_components)
: clock_(deps.clock),
tick_timer_(std::move(deps.tick_timer)),
buffer_level_filter_(std::move(deps.buffer_level_filter)),
decoder_database_(std::move(deps.decoder_database)),
delay_manager_(std::move(deps.delay_manager)),
delay_peak_detector_(std::move(deps.delay_peak_detector)),
dtmf_buffer_(std::move(deps.dtmf_buffer)),
dtmf_tone_generator_(std::move(deps.dtmf_tone_generator)),
packet_buffer_(std::move(deps.packet_buffer)),
@ -107,6 +116,7 @@ NetEqImpl::NetEqImpl(const NetEq::Config& config,
accelerate_factory_(std::move(deps.accelerate_factory)),
preemptive_expand_factory_(std::move(deps.preemptive_expand_factory)),
stats_(std::move(deps.stats)),
controller_(std::move(deps.neteq_controller)),
last_mode_(kModeNormal),
decoded_buffer_length_(kMaxFrameSize),
decoded_buffer_(new int16_t[decoded_buffer_length_]),
@ -133,11 +143,12 @@ NetEqImpl::NetEqImpl(const NetEq::Config& config,
<< "Changing to 8000 Hz.";
fs = 8000;
}
delay_manager_->SetMaximumDelay(config.max_delay_ms);
controller_->SetMaximumDelay(config.max_delay_ms);
fs_hz_ = fs;
fs_mult_ = fs / 8000;
last_output_sample_rate_hz_ = fs;
output_size_samples_ = static_cast<size_t>(kOutputSizeMs * 8 * fs_mult_);
controller_->SetSampleRate(fs_hz_, output_size_samples_);
decoder_frame_length_ = 3 * output_size_samples_;
if (create_components) {
SetSampleRateAndChannels(fs, 1); // Default is 1 channel.
@ -166,7 +177,7 @@ void NetEqImpl::InsertEmptyPacket(const RTPHeader& /*rtp_header*/) {
// rtp_header parameter.
// https://bugs.chromium.org/p/webrtc/issues/detail?id=7611
rtc::CritScope lock(&crit_sect_);
delay_manager_->RegisterEmptyPacket();
controller_->RegisterEmptyPacket();
}
namespace {
@ -279,8 +290,8 @@ void NetEqImpl::RemoveAllPayloadTypes() {
bool NetEqImpl::SetMinimumDelay(int delay_ms) {
rtc::CritScope lock(&crit_sect_);
if (delay_ms >= 0 && delay_ms <= 10000) {
assert(delay_manager_.get());
return delay_manager_->SetMinimumDelay(delay_ms);
assert(controller_.get());
return controller_->SetMinimumDelay(delay_ms);
}
return false;
}
@ -288,8 +299,8 @@ bool NetEqImpl::SetMinimumDelay(int delay_ms) {
bool NetEqImpl::SetMaximumDelay(int delay_ms) {
rtc::CritScope lock(&crit_sect_);
if (delay_ms >= 0 && delay_ms <= 10000) {
assert(delay_manager_.get());
return delay_manager_->SetMaximumDelay(delay_ms);
assert(controller_.get());
return controller_->SetMaximumDelay(delay_ms);
}
return false;
}
@ -297,32 +308,28 @@ bool NetEqImpl::SetMaximumDelay(int delay_ms) {
bool NetEqImpl::SetBaseMinimumDelayMs(int delay_ms) {
rtc::CritScope lock(&crit_sect_);
if (delay_ms >= 0 && delay_ms <= 10000) {
return delay_manager_->SetBaseMinimumDelay(delay_ms);
return controller_->SetBaseMinimumDelay(delay_ms);
}
return false;
}
int NetEqImpl::GetBaseMinimumDelayMs() const {
rtc::CritScope lock(&crit_sect_);
return delay_manager_->GetBaseMinimumDelay();
return controller_->GetBaseMinimumDelay();
}
int NetEqImpl::TargetDelayMs() const {
rtc::CritScope lock(&crit_sect_);
RTC_DCHECK(delay_manager_.get());
// The value from TargetLevel() is in number of packets, represented in Q8.
const size_t target_delay_samples =
(delay_manager_->TargetLevel() * decoder_frame_length_) >> 8;
return static_cast<int>(target_delay_samples) /
rtc::CheckedDivExact(fs_hz_, 1000);
RTC_DCHECK(controller_.get());
return controller_->TargetLevelMs();
}
int NetEqImpl::FilteredCurrentDelayMs() const {
rtc::CritScope lock(&crit_sect_);
// Sum up the filtered packet buffer level with the future length of the sync
// buffer.
const int delay_samples = buffer_level_filter_->filtered_current_level() +
sync_buffer_->FutureLength();
const int delay_samples =
controller_->GetFilteredBufferLevel() + sync_buffer_->FutureLength();
// The division below will truncate. The return value is in ms.
return delay_samples / rtc::CheckedDivExact(fs_hz_, 1000);
}
@ -333,12 +340,9 @@ int NetEqImpl::NetworkStatistics(NetEqNetworkStatistics* stats) {
const size_t total_samples_in_buffers =
packet_buffer_->NumSamplesInBuffer(decoder_frame_length_) +
sync_buffer_->FutureLength();
assert(delay_manager_.get());
assert(decision_logic_.get());
const int ms_per_packet = rtc::dchecked_cast<int>(
decision_logic_->packet_length_samples() / (fs_hz_ / 1000));
stats_->PopulateDelayManagerStats(ms_per_packet, *delay_manager_.get(),
stats);
assert(controller_.get());
stats->preferred_buffer_size_ms = controller_->TargetLevelMs();
stats->jitter_peaks_found = controller_->PeakFound();
stats_->GetNetworkStatistics(fs_hz_, total_samples_in_buffers,
decoder_frame_length_, stats);
return 0;
@ -712,38 +716,27 @@ int NetEqImpl::InsertPacketInternal(const RTPHeader& rtp_header,
}
}
// TODO(hlundin): Move this code to DelayManager class.
const DecoderDatabase::DecoderInfo* dec_info =
decoder_database_->GetDecoderInfo(main_payload_type);
assert(dec_info); // Already checked that the payload type is known.
delay_manager_->LastDecodedWasCngOrDtmf(dec_info->IsComfortNoise() ||
dec_info->IsDtmf());
if (delay_manager_->last_pack_cng_or_dtmf() == 0) {
// Calculate the total speech length carried in each packet.
if (number_of_primary_packets > 0) {
const size_t packet_length_samples =
number_of_primary_packets * decoder_frame_length_;
if (packet_length_samples != decision_logic_->packet_length_samples()) {
decision_logic_->set_packet_length_samples(packet_length_samples);
delay_manager_->SetPacketAudioLength(
rtc::dchecked_cast<int>((1000 * packet_length_samples) / fs_hz_));
}
}
// Update statistics.
if ((enable_rtx_handling_ || (int32_t)(main_timestamp - timestamp_) >= 0) &&
!new_codec_) {
// Only update statistics if incoming packet is not older than last played
// out packet or RTX handling is enabled, and if new codec flag is not
// set.
delay_manager_->Update(main_sequence_number, main_timestamp, fs_hz_);
}
} else if (delay_manager_->last_pack_cng_or_dtmf() == -1) {
// This is first "normal" packet after CNG or DTMF.
// Reset packet time counter and measure time until next packet,
// but don't update statistics.
delay_manager_->set_last_pack_cng_or_dtmf(0);
delay_manager_->ResetPacketIatCount();
const bool last_cng_or_dtmf =
dec_info->IsComfortNoise() || dec_info->IsDtmf();
const size_t packet_length_samples =
number_of_primary_packets * decoder_frame_length_;
// Only update statistics if incoming packet is not older than last played
// out packet or RTX handling is enabled, and if new codec flag is not
// set.
const bool should_update_stats =
(enable_rtx_handling_ ||
static_cast<int32_t>(main_timestamp - timestamp_) >= 0) &&
!new_codec_;
auto relative_delay = controller_->PacketArrived(
last_cng_or_dtmf, packet_length_samples, should_update_stats,
main_sequence_number, main_timestamp, fs_hz_);
if (relative_delay) {
stats_->RelativePacketArrivalDelay(relative_delay.value());
}
return 0;
}
@ -1018,10 +1011,10 @@ int NetEqImpl::GetDecision(Operations* operation,
uint64_t generated_noise_samples =
generated_noise_stopwatch_ ? (generated_noise_stopwatch_->ElapsedTicks() -
1) * output_size_samples_ +
decision_logic_->noise_fast_forward()
controller_->noise_fast_forward()
: 0;
if (decision_logic_->CngRfc3389On() || last_mode_ == kModeRfc3389Cng) {
if (controller_->CngRfc3389On() || last_mode_ == kModeRfc3389Cng) {
// Because of timestamp peculiarities, we have to "manually" disallow using
// a CNG packet with the same timestamp as the one that was last played.
// This can happen when using redundancy and will cause the timing to shift.
@ -1050,7 +1043,7 @@ int NetEqImpl::GetDecision(Operations* operation,
last_mode_ == kModePreemptiveExpandSuccess ||
last_mode_ == kModePreemptiveExpandLowEnergy) {
// Subtract (samples_left + output_size_samples_) from sampleMemory.
decision_logic_->AddSampleMemory(
controller_->AddSampleMemory(
-(samples_left + rtc::dchecked_cast<int>(output_size_samples_)));
}
@ -1067,11 +1060,31 @@ int NetEqImpl::GetDecision(Operations* operation,
generated_noise_samples =
generated_noise_stopwatch_
? generated_noise_stopwatch_->ElapsedTicks() * output_size_samples_ +
decision_logic_->noise_fast_forward()
controller_->noise_fast_forward()
: 0;
*operation = decision_logic_->GetDecision(
*sync_buffer_, *expand_, decoder_frame_length_, packet, last_mode_,
*play_dtmf, generated_noise_samples, &reset_decoder_);
NetEqController::NetEqStatus status;
status.packet_buffer_info.dtx_or_cng =
packet_buffer_->ContainsDtxOrCngPacket(decoder_database_.get());
status.packet_buffer_info.num_samples =
packet_buffer_->NumSamplesInBuffer(decoder_frame_length_);
status.packet_buffer_info.span_samples = packet_buffer_->GetSpanSamples(
decoder_frame_length_, last_output_sample_rate_hz_, true);
status.packet_buffer_info.span_samples_no_dtx =
packet_buffer_->GetSpanSamples(decoder_frame_length_,
last_output_sample_rate_hz_, false);
status.packet_buffer_info.num_packets = packet_buffer_->NumPacketsInBuffer();
status.target_timestamp = sync_buffer_->end_timestamp();
status.expand_mutefactor = expand_->MuteFactor(0);
status.last_packet_samples = decoder_frame_length_;
status.last_mode = last_mode_;
status.play_dtmf = *play_dtmf;
status.generated_noise_samples = generated_noise_samples;
if (packet) {
status.next_packet = {
packet->timestamp, packet->frame && packet->frame->IsDtxPacket(),
decoder_database_->IsComfortNoise(packet->payload_type)};
}
*operation = controller_->GetDecision(status, &reset_decoder_);
// Disallow time stretching if this packet is DTX, because such a decision may
// be based on earlier buffer level estimate, as we do not update buffer level
@ -1097,7 +1110,7 @@ int NetEqImpl::GetDecision(Operations* operation,
return 0;
}
decision_logic_->ExpandDecision(*operation);
controller_->ExpandDecision(*operation);
// Check conditions for reset.
if (new_codec_ || *operation == kUndefined) {
@ -1125,9 +1138,7 @@ int NetEqImpl::GetDecision(Operations* operation,
sync_buffer_->IncreaseEndTimestamp(timestamp_ - end_timestamp);
end_timestamp = timestamp_;
new_codec_ = false;
decision_logic_->SoftReset();
buffer_level_filter_->Reset();
delay_manager_->Reset();
controller_->SoftReset();
stats_->ResetMcu();
}
@ -1153,7 +1164,7 @@ int NetEqImpl::GetDecision(Operations* operation,
generated_noise_stopwatch_
? generated_noise_stopwatch_->ElapsedTicks() *
output_size_samples_ +
decision_logic_->noise_fast_forward()
controller_->noise_fast_forward()
: 0;
if (generated_noise_samples > 0 && last_mode_ != kModeDtmf) {
// Make a jump in timestamp due to the recently played comfort noise.
@ -1169,8 +1180,8 @@ int NetEqImpl::GetDecision(Operations* operation,
// In order to do an accelerate we need at least 30 ms of audio data.
if (samples_left >= static_cast<int>(samples_30_ms)) {
// Already have enough data, so we do not need to extract any more.
decision_logic_->set_sample_memory(samples_left);
decision_logic_->set_prev_time_scale(true);
controller_->set_sample_memory(samples_left);
controller_->set_prev_time_scale(true);
return 0;
} else if (samples_left >= static_cast<int>(samples_10_ms) &&
decoder_frame_length_ >= samples_30_ms) {
@ -1201,8 +1212,8 @@ int NetEqImpl::GetDecision(Operations* operation,
// Already have enough data, so we do not need to extract any more.
// Or, avoid decoding more data as it might overflow the playout buffer.
// Still try preemptive expand, though.
decision_logic_->set_sample_memory(samples_left);
decision_logic_->set_prev_time_scale(true);
controller_->set_sample_memory(samples_left);
controller_->set_prev_time_scale(true);
return 0;
}
if (samples_left < static_cast<int>(samples_20_ms) &&
@ -1228,7 +1239,7 @@ int NetEqImpl::GetDecision(Operations* operation,
int extracted_samples = 0;
if (packet) {
sync_buffer_->IncreaseEndTimestamp(packet->timestamp - end_timestamp);
if (decision_logic_->CngOff()) {
if (controller_->CngOff()) {
// Adjustment of timestamp only corresponds to an actual packet loss
// if comfort noise is not played. If comfort noise was just played,
// this adjustment of timestamp is only done to get back in sync with the
@ -1238,7 +1249,7 @@ int NetEqImpl::GetDecision(Operations* operation,
if (*operation != kRfc3389Cng) {
// We are about to decode and use a non-CNG packet.
decision_logic_->SetCngOff();
controller_->SetCngOff();
}
extracted_samples = ExtractPackets(required_samples, packet_list);
@ -1249,8 +1260,8 @@ int NetEqImpl::GetDecision(Operations* operation,
if (*operation == kAccelerate || *operation == kFastAccelerate ||
*operation == kPreemptiveExpand) {
decision_logic_->set_sample_memory(samples_left + extracted_samples);
decision_logic_->set_prev_time_scale(true);
controller_->set_sample_memory(samples_left + extracted_samples);
controller_->set_prev_time_scale(true);
}
if (*operation == kAccelerate || *operation == kFastAccelerate) {
@ -2058,13 +2069,8 @@ void NetEqImpl::SetSampleRateAndChannels(int fs_hz, size_t channels) {
decoded_buffer_length_ = kMaxFrameSize * channels;
decoded_buffer_.reset(new int16_t[decoded_buffer_length_]);
}
// Create DecisionLogic if it is not created yet, then communicate new sample
// rate and output size to DecisionLogic object.
if (!decision_logic_.get()) {
CreateDecisionLogic();
}
decision_logic_->SetSampleRate(fs_hz_, output_size_samples_);
RTC_CHECK(controller_) << "Unexpectedly found no NetEqController";
controller_->SetSampleRate(fs_hz_, output_size_samples_);
}
NetEqImpl::OutputType NetEqImpl::LastOutputType() {
@ -2085,11 +2091,4 @@ NetEqImpl::OutputType NetEqImpl::LastOutputType() {
return OutputType::kNormalSpeech;
}
}
void NetEqImpl::CreateDecisionLogic() {
decision_logic_.reset(DecisionLogic::Create(
fs_hz_, output_size_samples_, no_time_stretching_,
decoder_database_.get(), *packet_buffer_.get(), delay_manager_.get(),
buffer_level_filter_.get(), tick_timer_.get()));
}
} // namespace webrtc

19
modules/audio_coding/neteq/neteq_impl.h
View File

@ -24,6 +24,7 @@
#include "modules/audio_coding/neteq/defines.h" // Modes, Operations
#include "modules/audio_coding/neteq/expand_uma_logger.h"
#include "modules/audio_coding/neteq/include/neteq.h"
#include "modules/audio_coding/neteq/neteq_controller.h"
#include "modules/audio_coding/neteq/packet.h"
#include "modules/audio_coding/neteq/random_vector.h"
#include "modules/audio_coding/neteq/statistics_calculator.h"
@ -37,13 +38,9 @@ namespace webrtc {
// Forward declarations.
class Accelerate;
class BackgroundNoise;
class BufferLevelFilter;
class Clock;
class ComfortNoise;
class DecisionLogic;
class DecoderDatabase;
class DelayManager;
class DelayPeakDetector;
class DtmfBuffer;
class DtmfToneGenerator;
class Expand;
@ -108,13 +105,11 @@ class NetEqImpl : public webrtc::NetEq {
Clock* const clock;
std::unique_ptr<TickTimer> tick_timer;
std::unique_ptr<StatisticsCalculator> stats;
std::unique_ptr<BufferLevelFilter> buffer_level_filter;
std::unique_ptr<DecoderDatabase> decoder_database;
std::unique_ptr<DelayPeakDetector> delay_peak_detector;
std::unique_ptr<DelayManager> delay_manager;
std::unique_ptr<DtmfBuffer> dtmf_buffer;
std::unique_ptr<DtmfToneGenerator> dtmf_tone_generator;
std::unique_ptr<PacketBuffer> packet_buffer;
std::unique_ptr<NetEqController> neteq_controller;
std::unique_ptr<RedPayloadSplitter> red_payload_splitter;
std::unique_ptr<TimestampScaler> timestamp_scaler;
std::unique_ptr<AccelerateFactory> accelerate_factory;
@ -338,20 +333,12 @@ class NetEqImpl : public webrtc::NetEq {
virtual void UpdatePlcComponents(int fs_hz, size_t channels)
RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
// Creates DecisionLogic object with the mode given by |playout_mode_|.
virtual void CreateDecisionLogic() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
Clock* const clock_;
rtc::CriticalSection crit_sect_;
const std::unique_ptr<TickTimer> tick_timer_ RTC_GUARDED_BY(crit_sect_);
const std::unique_ptr<BufferLevelFilter> buffer_level_filter_
RTC_GUARDED_BY(crit_sect_);
const std::unique_ptr<DecoderDatabase> decoder_database_
RTC_GUARDED_BY(crit_sect_);
const std::unique_ptr<DelayManager> delay_manager_ RTC_GUARDED_BY(crit_sect_);
const std::unique_ptr<DelayPeakDetector> delay_peak_detector_
RTC_GUARDED_BY(crit_sect_);
const std::unique_ptr<DtmfBuffer> dtmf_buffer_ RTC_GUARDED_BY(crit_sect_);
const std::unique_ptr<DtmfToneGenerator> dtmf_tone_generator_
RTC_GUARDED_BY(crit_sect_);
@ -370,7 +357,7 @@ class NetEqImpl : public webrtc::NetEq {
const std::unique_ptr<StatisticsCalculator> stats_ RTC_GUARDED_BY(crit_sect_);
std::unique_ptr<BackgroundNoise> background_noise_ RTC_GUARDED_BY(crit_sect_);
std::unique_ptr<DecisionLogic> decision_logic_ RTC_GUARDED_BY(crit_sect_);
std::unique_ptr<NetEqController> controller_ RTC_GUARDED_BY(crit_sect_);
std::unique_ptr<AudioMultiVector> algorithm_buffer_
RTC_GUARDED_BY(crit_sect_);
std::unique_ptr<SyncBuffer> sync_buffer_ RTC_GUARDED_BY(crit_sect_);

166
modules/audio_coding/neteq/neteq_impl_unittest.cc
View File

@ -16,17 +16,17 @@
#include "api/audio_codecs/builtin_audio_decoder_factory.h"
#include "modules/audio_coding/neteq/accelerate.h"
#include "modules/audio_coding/neteq/decision_logic.h"
#include "modules/audio_coding/neteq/expand.h"
#include "modules/audio_coding/neteq/histogram.h"
#include "modules/audio_coding/neteq/include/neteq.h"
#include "modules/audio_coding/neteq/mock/mock_buffer_level_filter.h"
#include "modules/audio_coding/neteq/mock/mock_decoder_database.h"
#include "modules/audio_coding/neteq/mock/mock_delay_manager.h"
#include "modules/audio_coding/neteq/mock/mock_delay_peak_detector.h"
#include "modules/audio_coding/neteq/mock/mock_dtmf_buffer.h"
#include "modules/audio_coding/neteq/mock/mock_dtmf_tone_generator.h"
#include "modules/audio_coding/neteq/mock/mock_neteq_controller.h"
#include "modules/audio_coding/neteq/mock/mock_packet_buffer.h"
#include "modules/audio_coding/neteq/mock/mock_red_payload_splitter.h"
#include "modules/audio_coding/neteq/neteq_controller.h"
#include "modules/audio_coding/neteq/preemptive_expand.h"
#include "modules/audio_coding/neteq/statistics_calculator.h"
#include "modules/audio_coding/neteq/sync_buffer.h"
@ -78,13 +78,6 @@ class NetEqImplTest : public ::testing::Test {
// Get a local pointer to NetEq's TickTimer object.
tick_timer_ = deps.tick_timer.get();
if (use_mock_buffer_level_filter_) {
std::unique_ptr<MockBufferLevelFilter> mock(new MockBufferLevelFilter);
mock_buffer_level_filter_ = mock.get();
deps.buffer_level_filter = std::move(mock);
}
buffer_level_filter_ = deps.buffer_level_filter.get();
if (use_mock_decoder_database_) {
std::unique_ptr<MockDecoderDatabase> mock(new MockDecoderDatabase);
mock_decoder_database_ = mock.get();
@ -94,26 +87,6 @@ class NetEqImplTest : public ::testing::Test {
}
decoder_database_ = deps.decoder_database.get();
if (use_mock_delay_peak_detector_) {
std::unique_ptr<MockDelayPeakDetector> mock(
new MockDelayPeakDetector(tick_timer_, config_.enable_rtx_handling));
mock_delay_peak_detector_ = mock.get();
EXPECT_CALL(*mock_delay_peak_detector_, Reset()).Times(1);
deps.delay_peak_detector = std::move(mock);
}
delay_peak_detector_ = deps.delay_peak_detector.get();
if (use_mock_delay_manager_) {
std::unique_ptr<MockDelayManager> mock(new MockDelayManager(
config_.max_packets_in_buffer, config_.min_delay_ms, 1020054733,
DelayManager::HistogramMode::INTER_ARRIVAL_TIME,
config_.enable_rtx_handling, delay_peak_detector_, tick_timer_,
deps.stats.get(), std::make_unique<Histogram>(50, 32745)));
mock_delay_manager_ = mock.get();
deps.delay_manager = std::move(mock);
}
delay_manager_ = deps.delay_manager.get();
if (use_mock_dtmf_buffer_) {
std::unique_ptr<MockDtmfBuffer> mock(
new MockDtmfBuffer(config_.sample_rate_hz));
@ -137,6 +110,23 @@ class NetEqImplTest : public ::testing::Test {
}
packet_buffer_ = deps.packet_buffer.get();
if (use_mock_neteq_controller_) {
std::unique_ptr<MockNetEqController> mock(new MockNetEqController());
mock_neteq_controller_ = mock.get();
deps.neteq_controller = std::move(mock);
} else {
deps.stats = std::make_unique<StatisticsCalculator>();
NetEqController::Config controller_config;
controller_config.tick_timer = tick_timer_;
controller_config.base_min_delay_ms = config_.min_delay_ms;
controller_config.enable_rtx_handling = config_.enable_rtx_handling;
controller_config.allow_time_stretching = true;
controller_config.max_packets_in_buffer = config_.max_packets_in_buffer;
deps.neteq_controller =
std::make_unique<DecisionLogic>(std::move(controller_config));
}
neteq_controller_ = deps.neteq_controller.get();
if (use_mock_payload_splitter_) {
std::unique_ptr<MockRedPayloadSplitter> mock(new MockRedPayloadSplitter);
mock_payload_splitter_ = mock.get();
@ -155,10 +145,8 @@ class NetEqImplTest : public ::testing::Test {
void UseNoMocks() {
ASSERT_TRUE(neteq_ == NULL) << "Must call UseNoMocks before CreateInstance";
use_mock_buffer_level_filter_ = false;
use_mock_decoder_database_ = false;
use_mock_delay_peak_detector_ = false;
use_mock_delay_manager_ = false;
use_mock_neteq_controller_ = false;
use_mock_dtmf_buffer_ = false;
use_mock_dtmf_tone_generator_ = false;
use_mock_packet_buffer_ = false;
@ -166,17 +154,11 @@ class NetEqImplTest : public ::testing::Test {
}
virtual ~NetEqImplTest() {
if (use_mock_buffer_level_filter_) {
EXPECT_CALL(*mock_buffer_level_filter_, Die()).Times(1);
}
if (use_mock_decoder_database_) {
EXPECT_CALL(*mock_decoder_database_, Die()).Times(1);
}
if (use_mock_delay_manager_) {
EXPECT_CALL(*mock_delay_manager_, Die()).Times(1);
}
if (use_mock_delay_peak_detector_) {
EXPECT_CALL(*mock_delay_peak_detector_, Die()).Times(1);
if (use_mock_neteq_controller_) {
EXPECT_CALL(*mock_neteq_controller_, Die()).Times(1);
}
if (use_mock_dtmf_buffer_) {
EXPECT_CALL(*mock_dtmf_buffer_, Die()).Times(1);
@ -242,18 +224,12 @@ class NetEqImplTest : public ::testing::Test {
NetEq::Config config_;
SimulatedClock clock_;
TickTimer* tick_timer_ = nullptr;
MockBufferLevelFilter* mock_buffer_level_filter_ = nullptr;
BufferLevelFilter* buffer_level_filter_ = nullptr;
bool use_mock_buffer_level_filter_ = true;
MockDecoderDatabase* mock_decoder_database_ = nullptr;
DecoderDatabase* decoder_database_ = nullptr;
bool use_mock_decoder_database_ = true;
MockDelayPeakDetector* mock_delay_peak_detector_ = nullptr;
DelayPeakDetector* delay_peak_detector_ = nullptr;
bool use_mock_delay_peak_detector_ = true;
MockDelayManager* mock_delay_manager_ = nullptr;
DelayManager* delay_manager_ = nullptr;
bool use_mock_delay_manager_ = true;
MockNetEqController* mock_neteq_controller_ = nullptr;
NetEqController* neteq_controller_ = nullptr;
bool use_mock_neteq_controller_ = true;
MockDtmfBuffer* mock_dtmf_buffer_ = nullptr;
DtmfBuffer* dtmf_buffer_ = nullptr;
bool use_mock_dtmf_buffer_ = true;
@ -367,16 +343,20 @@ TEST_F(NetEqImplTest, InsertPacket) {
// All expectations within this block must be called in this specific order.
InSequence sequence; // Dummy variable.
// Expectations when the first packet is inserted.
EXPECT_CALL(*mock_delay_manager_, last_pack_cng_or_dtmf())
.Times(2)
.WillRepeatedly(Return(-1));
EXPECT_CALL(*mock_delay_manager_, set_last_pack_cng_or_dtmf(0)).Times(1);
EXPECT_CALL(*mock_delay_manager_, ResetPacketIatCount()).Times(1);
// Expectations when the second packet is inserted. Slightly different.
EXPECT_CALL(*mock_delay_manager_, last_pack_cng_or_dtmf())
.WillOnce(Return(0));
EXPECT_CALL(*mock_delay_manager_, SetPacketAudioLength(30))
.WillOnce(Return(0));
EXPECT_CALL(*mock_neteq_controller_,
PacketArrived(/*last_cng_or_dtmf*/ false,
/*packet_length_samples*/ _,
/*should_update_stats*/ _,
/*main_sequence_number*/ kFirstSequenceNumber,
/*main_timestamp*/ kFirstTimestamp,
/*fs_hz*/ 8000));
EXPECT_CALL(*mock_neteq_controller_,
PacketArrived(/*last_cng_or_dtmf*/ false,
/*packet_length_samples*/ _,
/*should_update_stats*/ _,
/*main_sequence_number*/ kFirstSequenceNumber + 1,
/*main_timestamp*/ kFirstTimestamp + 160,
/*fs_hz*/ 8000));
}
// Insert first packet.
@ -1340,10 +1320,10 @@ TEST_F(NetEqImplTest, TickTimerIncrement) {
TEST_F(NetEqImplTest, SetBaseMinimumDelay) {
UseNoMocks();
use_mock_delay_manager_ = true;
use_mock_neteq_controller_ = true;
CreateInstance();
EXPECT_CALL(*mock_delay_manager_, SetBaseMinimumDelay(_))
EXPECT_CALL(*mock_neteq_controller_, SetBaseMinimumDelay(_))
.WillOnce(Return(true))
.WillOnce(Return(false));
@ -1355,12 +1335,12 @@ TEST_F(NetEqImplTest, SetBaseMinimumDelay) {
TEST_F(NetEqImplTest, GetBaseMinimumDelayMs) {
UseNoMocks();
use_mock_delay_manager_ = true;
use_mock_neteq_controller_ = true;
CreateInstance();
const int delay_ms = 200;
EXPECT_CALL(*mock_delay_manager_, GetBaseMinimumDelay())
EXPECT_CALL(*mock_neteq_controller_, GetBaseMinimumDelay())
.WillOnce(Return(delay_ms));
EXPECT_EQ(delay_ms, neteq_->GetBaseMinimumDelayMs());
@ -1368,20 +1348,17 @@ TEST_F(NetEqImplTest, GetBaseMinimumDelayMs) {
TEST_F(NetEqImplTest, TargetDelayMs) {
UseNoMocks();
use_mock_delay_manager_ = true;
use_mock_neteq_controller_ = true;
CreateInstance();
// Let the dummy target delay be 17 packets.
constexpr int kTargetLevelPacketsQ8 = 17 << 8;
EXPECT_CALL(*mock_delay_manager_, TargetLevel())
.WillOnce(Return(kTargetLevelPacketsQ8));
// Default packet size before any packet has been decoded is 30 ms, so we are
// expecting 17 * 30 = 510 ms target delay.
EXPECT_EQ(17 * 30, neteq_->TargetDelayMs());
constexpr int kTargetLevelMs = 510;
EXPECT_CALL(*mock_neteq_controller_, TargetLevelMs())
.WillOnce(Return(kTargetLevelMs));
EXPECT_EQ(510, neteq_->TargetDelayMs());
}
TEST_F(NetEqImplTest, InsertEmptyPacket) {
UseNoMocks();
use_mock_delay_manager_ = true;
use_mock_neteq_controller_ = true;
CreateInstance();
RTPHeader rtp_header;
@ -1390,18 +1367,18 @@ TEST_F(NetEqImplTest, InsertEmptyPacket) {
rtp_header.timestamp = 0x12345678;
rtp_header.ssrc = 0x87654321;
EXPECT_CALL(*mock_delay_manager_, RegisterEmptyPacket());
EXPECT_CALL(*mock_neteq_controller_, RegisterEmptyPacket());
neteq_->InsertEmptyPacket(rtp_header);
}
TEST_F(NetEqImplTest, EnableRtxHandling) {
UseNoMocks();
use_mock_delay_manager_ = true;
use_mock_neteq_controller_ = true;
config_.enable_rtx_handling = true;
CreateInstance();
EXPECT_CALL(*mock_delay_manager_, BufferLimits(_, _))
EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _))
.Times(1)
.WillOnce(DoAll(SetArgPointee<0>(0), SetArgPointee<1>(0)));
.WillOnce(Return(kNormal));
const int kPayloadLengthSamples = 80;
const size_t kPayloadLengthBytes = 2 * kPayloadLengthSamples; // PCM 16-bit.
@ -1423,8 +1400,15 @@ TEST_F(NetEqImplTest, EnableRtxHandling) {
// Insert second packet that was sent before the first packet.
rtp_header.sequenceNumber -= 1;
rtp_header.timestamp -= kPayloadLengthSamples;
EXPECT_CALL(*mock_delay_manager_,
Update(rtp_header.sequenceNumber, rtp_header.timestamp, _));
EXPECT_CALL(*mock_neteq_controller_,
PacketArrived(
/*last_cng_or_dtmf*/ _,
/*packet_length_samples*/ kPayloadLengthSamples,
/*should_update_stats*/ _,
/*main_sequence_number*/ rtp_header.sequenceNumber,
/*main_timestamp*/ rtp_header.timestamp,
/*fs_hz*/ 8000));
EXPECT_EQ(NetEq::kOK, neteq_->InsertPacket(rtp_header, payload));
}
@ -1474,7 +1458,7 @@ class NetEqImplTest120ms : public NetEqImplTest {
void CreateInstanceWithDelayManagerMock() {
UseNoMocks();
use_mock_delay_manager_ = true;
use_mock_neteq_controller_ = true;
CreateInstance(decoder_factory_);
EXPECT_TRUE(neteq_->RegisterPayloadType(
kPayloadType, SdpAudioFormat("opus", 48000, 2, {{"stereo", "1"}})));
@ -1551,16 +1535,19 @@ TEST_F(NetEqImplTest120ms, Merge) {
Register120msCodec(AudioDecoder::kSpeech);
CreateInstanceWithDelayManagerMock();
EXPECT_CALL(*mock_neteq_controller_, CngOff()).WillRepeatedly(Return(true));
InsertPacket(first_timestamp());
GetFirstPacket();
bool muted;
EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _))
.WillOnce(Return(kExpand));
EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted));
InsertPacket(first_timestamp() + 2 * timestamp_diff_between_packets());
// Delay manager reports a target level which should cause a Merge.
EXPECT_CALL(*mock_delay_manager_, TargetLevel()).WillOnce(Return(-10));
EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _))
.WillOnce(Return(kMerge));
EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted));
EXPECT_EQ(kMerge, neteq_->last_operation_for_test());
@ -1586,10 +1573,9 @@ TEST_F(NetEqImplTest120ms, FastAccelerate) {
GetFirstPacket();
InsertPacket(first_timestamp() + timestamp_diff_between_packets());
// Delay manager report buffer limit which should cause a FastAccelerate.
EXPECT_CALL(*mock_delay_manager_, BufferLimits(_, _))
EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _))
.Times(1)
.WillOnce(DoAll(SetArgPointee<0>(0), SetArgPointee<1>(0)));
.WillOnce(Return(kFastAccelerate));
bool muted;
EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted));
@ -1605,10 +1591,9 @@ TEST_F(NetEqImplTest120ms, PreemptiveExpand) {
InsertPacket(first_timestamp() + timestamp_diff_between_packets());
// Delay manager report buffer limit which should cause a PreemptiveExpand.
EXPECT_CALL(*mock_delay_manager_, BufferLimits(_, _))
EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _))
.Times(1)
.WillOnce(DoAll(SetArgPointee<0>(100), SetArgPointee<1>(100)));
.WillOnce(Return(kPreemptiveExpand));
bool muted;
EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted));
@ -1624,10 +1609,9 @@ TEST_F(NetEqImplTest120ms, Accelerate) {
InsertPacket(first_timestamp() + timestamp_diff_between_packets());
// Delay manager report buffer limit which should cause a Accelerate.
EXPECT_CALL(*mock_delay_manager_, BufferLimits(_, _))
EXPECT_CALL(*mock_neteq_controller_, GetDecision(_, _))
.Times(1)
.WillOnce(DoAll(SetArgPointee<0>(1), SetArgPointee<1>(2)));
.WillOnce(Return(kAccelerate));
bool muted;
EXPECT_EQ(NetEq::kOK, neteq_->GetAudio(&output_, &muted));

10
modules/audio_coding/neteq/statistics_calculator.cc
View File

@ -382,16 +382,6 @@ void StatisticsCalculator::GetNetworkStatistics(int fs_hz,
Reset();
}
void StatisticsCalculator::PopulateDelayManagerStats(
int ms_per_packet,
const DelayManager& delay_manager,
NetEqNetworkStatistics* stats) {
RTC_DCHECK(stats);
stats->preferred_buffer_size_ms =
(delay_manager.TargetLevel() >> 8) * ms_per_packet;
stats->jitter_peaks_found = delay_manager.PeakFound();
}
NetEqLifetimeStatistics StatisticsCalculator::GetLifetimeStatistics() const {
return lifetime_stats_;
}

8
modules/audio_coding/neteq/statistics_calculator.h
View File

@ -116,14 +116,6 @@ class StatisticsCalculator {
size_t samples_per_packet,
NetEqNetworkStatistics* stats);
// Populates |preferred_buffer_size_ms|, |jitter_peaks_found| and
// |clockdrift_ppm| in |stats|. This is a convenience method, and does not
// strictly have to be in the StatisticsCalculator class, but it makes sense
// since all other stats fields are populated by that class.
static void PopulateDelayManagerStats(int ms_per_packet,
const DelayManager& delay_manager,
NetEqNetworkStatistics* stats);
// Returns a copy of this class's lifetime statistics. These statistics are
// never reset.
NetEqLifetimeStatistics GetLifetimeStatistics() const;