zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add plumbing of RtpPacketInfos to each AudioFrame as input for SourceTracker.

Browse Source 
This change adds the plumbing of RtpPacketInfo from ChannelReceive::OnRtpPacket() to ChannelReceive::GetAudioFrameWithInfo() for audio. It is a step towards replacing the non-spec compliant ContributingSources that updates itself at packet-receive time, with the spec-compliant SourceTracker that will update itself at frame-delivery-to-track time.

Bug: webrtc:10668
Change-Id: I03385d6865bbc7bfbef7634f88de820a934f787a
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/139890
Reviewed-by: Stefan Holmer <stefan@webrtc.org>
Reviewed-by: Minyue Li <minyue@webrtc.org>
Commit-Queue: Chen Xing <chxg@google.com>
Cr-Commit-Position: refs/heads/master@{#28434}
This commit is contained in:
Chen Xing
2019-07-01 17:16:32 +02:00
committed by Commit Bot
parent 62eb89d221
commit 3e8ef940fe
23 changed files with 195 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
modules/audio_coding/neteq/neteq_impl.cc
View File

@ -15,6 +15,7 @@
#include <cstdint>
#include <cstring>
#include <list>
#include <map>
#include <utility>
#include <vector>
@ -52,13 +53,16 @@
#include "rtc_base/sanitizer.h"
#include "rtc_base/strings/audio_format_to_string.h"
#include "rtc_base/trace_event.h"
#include "system_wrappers/include/clock.h"
namespace webrtc {
NetEqImpl::Dependencies::Dependencies(
const NetEq::Config& config,
Clock* clock,
const rtc::scoped_refptr<AudioDecoderFactory>& decoder_factory)
: tick_timer(new TickTimer),
: clock(clock),
tick_timer(new TickTimer),
stats(new StatisticsCalculator),
buffer_level_filter(new BufferLevelFilter),
decoder_database(
@ -86,7 +90,8 @@ NetEqImpl::Dependencies::~Dependencies() = default;
NetEqImpl::NetEqImpl(const NetEq::Config& config,
Dependencies&& deps,
bool create_components)
: tick_timer_(std::move(deps.tick_timer)),
: clock_(deps.clock),
tick_timer_(std::move(deps.tick_timer)),
buffer_level_filter_(std::move(deps.buffer_level_filter)),
decoder_database_(std::move(deps.decoder_database)),
delay_manager_(std::move(deps.delay_manager)),
@ -468,17 +473,20 @@ int NetEqImpl::InsertPacketInternal(const RTPHeader& rtp_header,
RTC_LOG_F(LS_ERROR) << "payload is empty";
return kInvalidPointer;
}
int64_t receive_time_ms = clock_->TimeInMilliseconds();
stats_->ReceivedPacket();
PacketList packet_list;
// Insert packet in a packet list.
packet_list.push_back([&rtp_header, &payload] {
packet_list.push_back([&rtp_header, &payload, &receive_time_ms] {
// Convert to Packet.
Packet packet;
packet.payload_type = rtp_header.payloadType;
packet.sequence_number = rtp_header.sequenceNumber;
packet.timestamp = rtp_header.timestamp;
packet.payload.SetData(payload.data(), payload.size());
packet.packet_info = RtpPacketInfo(rtp_header, receive_time_ms);
// Waiting time will be set upon inserting the packet in the buffer.
RTC_DCHECK(!packet.waiting_time);
return packet;
@ -611,6 +619,7 @@ int NetEqImpl::InsertPacketInternal(const RTPHeader& rtp_header,
const auto sequence_number = packet.sequence_number;
const auto payload_type = packet.payload_type;
const Packet::Priority original_priority = packet.priority;
const auto& packet_info = packet.packet_info;
auto packet_from_result = [&](AudioDecoder::ParseResult& result) {
Packet new_packet;
new_packet.sequence_number = sequence_number;
@ -618,6 +627,7 @@ int NetEqImpl::InsertPacketInternal(const RTPHeader& rtp_header,
new_packet.timestamp = result.timestamp;
new_packet.priority.codec_level = result.priority;
new_packet.priority.red_level = original_priority.red_level;
new_packet.packet_info = packet_info;
new_packet.frame = std::move(result.frame);
return new_packet;
};
@ -879,7 +889,16 @@ int NetEqImpl::GetAudioInternal(AudioFrame* audio_frame,
comfort_noise_->Reset();
}
// Copy from |algorithm_buffer| to |sync_buffer_|.
// We treat it as if all packets referenced to by |last_decoded_packet_infos_|
// were mashed together when creating the samples in |algorithm_buffer_|.
RtpPacketInfos packet_infos(std::move(last_decoded_packet_infos_));
last_decoded_packet_infos_.clear();
// Copy samples from |algorithm_buffer_| to |sync_buffer_|.
//
// TODO(bugs.webrtc.org/10757):
// We would in the future also like to pass |packet_infos| so that we can do
// sample-perfect tracking of that information across |sync_buffer_|.
sync_buffer_->PushBack(*algorithm_buffer_);
// Extract data from |sync_buffer_| to |output|.
@ -897,6 +916,13 @@ int NetEqImpl::GetAudioInternal(AudioFrame* audio_frame,
sync_buffer_->GetNextAudioInterleaved(num_output_samples_per_channel,
audio_frame);
audio_frame->sample_rate_hz_ = fs_hz_;
// TODO(bugs.webrtc.org/10757):
// We don't have the ability to properly track individual packets once their
// audio samples have entered |sync_buffer_|. So for now, treat it as if
// |packet_infos| from packets decoded by the current |GetAudioInternal()|
// call were all consumed assembling the current audio frame and the current
// audio frame only.
audio_frame->packet_infos_ = std::move(packet_infos);
if (sync_buffer_->FutureLength() < expand_->overlap_length()) {
// The sync buffer should always contain |overlap_length| samples, but now
// too many samples have been extracted. Reinstall the |overlap_length|
@ -1392,6 +1418,7 @@ int NetEqImpl::DecodeLoop(PacketList* packet_list,
int* decoded_length,
AudioDecoder::SpeechType* speech_type) {
RTC_DCHECK(last_decoded_timestamps_.empty());
RTC_DCHECK(last_decoded_packet_infos_.empty());
// Do decoding.
while (!packet_list->empty() && !decoder_database_->IsComfortNoise(
@ -1409,6 +1436,8 @@ int NetEqImpl::DecodeLoop(PacketList* packet_list,
rtc::ArrayView<int16_t>(&decoded_buffer_[*decoded_length],
decoded_buffer_length_ - *decoded_length));
last_decoded_timestamps_.push_back(packet_list->front().timestamp);
last_decoded_packet_infos_.push_back(
std::move(packet_list->front().packet_info));
packet_list->pop_front();
if (opt_result) {
const auto& result = *opt_result;
@ -1424,6 +1453,7 @@ int NetEqImpl::DecodeLoop(PacketList* packet_list,
// TODO(ossu): What to put here?
RTC_LOG(LS_WARNING) << "Decode error";
*decoded_length = -1;
last_decoded_packet_infos_.clear();
packet_list->clear();
break;
}