zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Aligning time in audio jitter buffer plot to other plots in rtc event log visualizer.

Browse Source 
Bug: webrtc:9147
Change-Id: I4ddb3e93ea04a11a68e097ecad731d6d9d6842a9
Reviewed-on: https://webrtc-review.googlesource.com/75322
Reviewed-by: Björn Terelius <terelius@webrtc.org>
Reviewed-by: Henrik Lundin <henrik.lundin@webrtc.org>
Commit-Queue: Minyue Li <minyue@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#23712}
This commit is contained in:
Minyue Li
2018-06-21 11:47:14 +02:00
committed by Commit Bot
parent 1ec04f19c6
commit 45fc6dfaaa
4 changed files with 133 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files

208
modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc
View File

@ -23,6 +23,13 @@
namespace webrtc {
namespace test {
namespace {
std::string kArrivalDelayX = "arrival_delay_x";
std::string kArrivalDelayY = "arrival_delay_y";
std::string kTargetDelayX = "target_delay_x";
std::string kTargetDelayY = "target_delay_y";
std::string kPlayoutDelayX = "playout_delay_x";
std::string kPlayoutDelayY = "playout_delay_y";
// Helper function for NetEqDelayAnalyzer::CreateGraphs. Returns the
// interpolated value of a function at the point x. Vector x_vec contains the
// sample points, and y_vec contains the function values at these points. The
@ -54,6 +61,26 @@ double LinearInterpolate(double x,
}
return y;
}
void PrintDelays(const NetEqDelayAnalyzer::Delays& delays,
int64_t ref_time_ms,
const std::string& var_name_x,
const std::string& var_name_y,
std::ofstream& output,
const std::string& terminator = "") {
output << var_name_x << " = [ ";
for (const std::pair<int64_t, float>& delay : delays) {
output << (delay.first - ref_time_ms) / 1000.f << ", ";
}
output << "]" << terminator << std::endl;
output << var_name_y << " = [ ";
for (const std::pair<int64_t, float>& delay : delays) {
output << delay.second << ", ";
}
output << "]" << terminator << std::endl;
}
} // namespace
void NetEqDelayAnalyzer::AfterInsertPacket(
@ -97,12 +124,10 @@ void NetEqDelayAnalyzer::AfterGetAudio(int64_t time_now_ms,
++get_audio_count_;
}
void NetEqDelayAnalyzer::CreateGraphs(
std::vector<float>* send_time_s,
std::vector<float>* arrival_delay_ms,
std::vector<float>* corrected_arrival_delay_ms,
std::vector<absl::optional<float>>* playout_delay_ms,
std::vector<absl::optional<float>>* target_delay_ms) const {
void NetEqDelayAnalyzer::CreateGraphs(Delays* arrival_delay_ms,
Delays* corrected_arrival_delay_ms,
Delays* playout_delay_ms,
Delays* target_delay_ms) const {
if (get_audio_time_ms_.empty()) {
return;
}
@ -123,111 +148,76 @@ void NetEqDelayAnalyzer::CreateGraphs(
// calculates the base offset.
for (auto& d : data_) {
rtp_timestamps_ms.push_back(
unwrapper.Unwrap(d.first) /
static_cast<double>(unwrapper.Unwrap(d.first)) /
rtc::CheckedDivExact(last_sample_rate_hz_, 1000));
offset =
std::min(offset, d.second.arrival_time_ms - rtp_timestamps_ms.back());
}
// Calculate send times in seconds for each packet. This is the (unwrapped)
// RTP timestamp in ms divided by 1000.
send_time_s->resize(rtp_timestamps_ms.size());
std::transform(rtp_timestamps_ms.begin(), rtp_timestamps_ms.end(),
send_time_s->begin(), [rtp_timestamps_ms](double x) {
return (x - rtp_timestamps_ms[0]) / 1000.f;
});
RTC_DCHECK_EQ(send_time_s->size(), rtp_timestamps_ms.size());
// This loop traverses the data again and populates the graph vectors. The
// reason to have two loops and traverse twice is that the offset cannot be
// known until the first traversal is done. Meanwhile, the final offset must
// be known already at the start of this second loop.
auto data_it = data_.cbegin();
for (size_t i = 0; i < send_time_s->size(); ++i, ++data_it) {
RTC_DCHECK(data_it != data_.end());
const double offset_send_time_ms = rtp_timestamps_ms[i] + offset;
const auto& timing = data_it->second;
corrected_arrival_delay_ms->push_back(
size_t i = 0;
for (const auto& data : data_) {
const double offset_send_time_ms = rtp_timestamps_ms[i++] + offset;
const auto& timing = data.second;
corrected_arrival_delay_ms->push_back(std::make_pair(
timing.arrival_time_ms,
LinearInterpolate(timing.arrival_time_ms, get_audio_time_ms_,
nominal_get_audio_time_ms) -
offset_send_time_ms);
arrival_delay_ms->push_back(timing.arrival_time_ms - offset_send_time_ms);
offset_send_time_ms));
arrival_delay_ms->push_back(std::make_pair(
timing.arrival_time_ms, timing.arrival_time_ms - offset_send_time_ms));
if (timing.decode_get_audio_count) {
// This packet was decoded.
RTC_DCHECK(timing.sync_delay_ms);
const float playout_ms = *timing.decode_get_audio_count * 10 +
get_audio_time_ms_[0] + *timing.sync_delay_ms -
offset_send_time_ms;
playout_delay_ms->push_back(playout_ms);
const int64_t get_audio_time =
*timing.decode_get_audio_count * 10 + get_audio_time_ms_[0];
const float playout_ms =
get_audio_time + *timing.sync_delay_ms - offset_send_time_ms;
playout_delay_ms->push_back(std::make_pair(get_audio_time, playout_ms));
RTC_DCHECK(timing.target_delay_ms);
RTC_DCHECK(timing.current_delay_ms);
const float target =
playout_ms - *timing.current_delay_ms + *timing.target_delay_ms;
target_delay_ms->push_back(target);
} else {
// This packet was never decoded. Mark target and playout delays as empty.
playout_delay_ms->push_back(absl::nullopt);
target_delay_ms->push_back(absl::nullopt);
target_delay_ms->push_back(std::make_pair(get_audio_time, target));
}
}
RTC_DCHECK(data_it == data_.end());
RTC_DCHECK_EQ(send_time_s->size(), corrected_arrival_delay_ms->size());
RTC_DCHECK_EQ(send_time_s->size(), playout_delay_ms->size());
RTC_DCHECK_EQ(send_time_s->size(), target_delay_ms->size());
}
void NetEqDelayAnalyzer::CreateMatlabScript(
const std::string& script_name) const {
std::vector<float> send_time_s;
std::vector<float> arrival_delay_ms;
std::vector<float> corrected_arrival_delay_ms;
std::vector<absl::optional<float>> playout_delay_ms;
std::vector<absl::optional<float>> target_delay_ms;
CreateGraphs(&send_time_s, &arrival_delay_ms, &corrected_arrival_delay_ms,
Delays arrival_delay_ms;
Delays corrected_arrival_delay_ms;
Delays playout_delay_ms;
Delays target_delay_ms;
CreateGraphs(&arrival_delay_ms, &corrected_arrival_delay_ms,
&playout_delay_ms, &target_delay_ms);
// Maybe better to find the actually smallest timestamp, to surely avoid
// x-axis starting from negative.
const int64_t ref_time_ms = arrival_delay_ms.front().first;
// Create an output file stream to Matlab script file.
std::ofstream output(script_name);
// The iterator is used to batch-output comma-separated values from vectors.
std::ostream_iterator<float> output_iterator(output, ",");
output << "send_time_s = [ ";
std::copy(send_time_s.begin(), send_time_s.end(), output_iterator);
output << "];" << std::endl;
PrintDelays(corrected_arrival_delay_ms, ref_time_ms, kArrivalDelayX,
kArrivalDelayY, output, ";");
output << "arrival_delay_ms = [ ";
std::copy(arrival_delay_ms.begin(), arrival_delay_ms.end(), output_iterator);
output << "];" << std::endl;
// PrintDelays(corrected_arrival_delay_x, kCorrectedArrivalDelayX,
// kCorrectedArrivalDelayY, output);
output << "corrected_arrival_delay_ms = [ ";
std::copy(corrected_arrival_delay_ms.begin(),
corrected_arrival_delay_ms.end(), output_iterator);
output << "];" << std::endl;
PrintDelays(playout_delay_ms, ref_time_ms, kPlayoutDelayX, kPlayoutDelayY,
output, ";");
output << "playout_delay_ms = [ ";
for (const auto& v : playout_delay_ms) {
if (!v) {
output << "nan, ";
} else {
output << *v << ", ";
}
}
output << "];" << std::endl;
PrintDelays(target_delay_ms, ref_time_ms, kTargetDelayX, kTargetDelayY,
output, ";");
output << "target_delay_ms = [ ";
for (const auto& v : target_delay_ms) {
if (!v) {
output << "nan, ";
} else {
output << *v << ", ";
}
}
output << "];" << std::endl;
output << "h=plot(send_time_s, arrival_delay_ms, "
<< "send_time_s, target_delay_ms, 'g.', "
<< "send_time_s, playout_delay_ms);" << std::endl;
output << "h=plot(" << kArrivalDelayX << ", " << kArrivalDelayY << ", "
<< kTargetDelayX << ", " << kTargetDelayY << ", 'g.', "
<< kPlayoutDelayX << ", " << kPlayoutDelayY << ");" << std::endl;
output << "set(h(1),'color',0.75*[1 1 1]);" << std::endl;
output << "set(h(2),'markersize',6);" << std::endl;
output << "set(h(3),'linew',1.5);" << std::endl;
@ -235,7 +225,7 @@ void NetEqDelayAnalyzer::CreateMatlabScript(
output << "axis tight" << std::endl;
output << "ax2=axis;" << std::endl;
output << "axis([ax2(1:3) ax1(4)])" << std::endl;
output << "xlabel('send time [s]');" << std::endl;
output << "xlabel('time [s]');" << std::endl;
output << "ylabel('relative delay [ms]');" << std::endl;
if (!ssrcs_.empty()) {
auto ssrc_it = ssrcs_.cbegin();
@ -255,65 +245,45 @@ void NetEqDelayAnalyzer::CreateMatlabScript(
void NetEqDelayAnalyzer::CreatePythonScript(
const std::string& script_name) const {
std::vector<float> send_time_s;
std::vector<float> arrival_delay_ms;
std::vector<float> corrected_arrival_delay_ms;
std::vector<absl::optional<float>> playout_delay_ms;
std::vector<absl::optional<float>> target_delay_ms;
CreateGraphs(&send_time_s, &arrival_delay_ms, &corrected_arrival_delay_ms,
Delays arrival_delay_ms;
Delays corrected_arrival_delay_ms;
Delays playout_delay_ms;
Delays target_delay_ms;
CreateGraphs(&arrival_delay_ms, &corrected_arrival_delay_ms,
&playout_delay_ms, &target_delay_ms);
// Maybe better to find the actually smallest timestamp, to surely avoid
// x-axis starting from negative.
const int64_t ref_time_ms = arrival_delay_ms.front().first;
// Create an output file stream to the python script file.
std::ofstream output(script_name);
// The iterator is used to batch-output comma-separated values from vectors.
std::ostream_iterator<float> output_iterator(output, ",");
// Necessary includes
output << "import numpy as np" << std::endl;
output << "import matplotlib.pyplot as plt" << std::endl;
output << "send_time_s = [";
std::copy(send_time_s.begin(), send_time_s.end(), output_iterator);
output << "]" << std::endl;
PrintDelays(corrected_arrival_delay_ms, ref_time_ms, kArrivalDelayX,
kArrivalDelayY, output);
output << "arrival_delay_ms = [";
std::copy(arrival_delay_ms.begin(), arrival_delay_ms.end(), output_iterator);
output << "]" << std::endl;
// PrintDelays(corrected_arrival_delay_x, kCorrectedArrivalDelayX,
// kCorrectedArrivalDelayY, output);
output << "corrected_arrival_delay_ms = [";
std::copy(corrected_arrival_delay_ms.begin(),
corrected_arrival_delay_ms.end(), output_iterator);
output << "]" << std::endl;
PrintDelays(playout_delay_ms, ref_time_ms, kPlayoutDelayX, kPlayoutDelayY,
output);
output << "playout_delay_ms = [";
for (const auto& v : playout_delay_ms) {
if (!v) {
output << "float('nan'), ";
} else {
output << *v << ", ";
}
}
output << "]" << std::endl;
output << "target_delay_ms = [";
for (const auto& v : target_delay_ms) {
if (!v) {
output << "float('nan'), ";
} else {
output << *v << ", ";
}
}
output << "]" << std::endl;
PrintDelays(target_delay_ms, ref_time_ms, kTargetDelayX, kTargetDelayY,
output);
output << "if __name__ == '__main__':" << std::endl;
output << " h=plt.plot(send_time_s, arrival_delay_ms, "
<< "send_time_s, target_delay_ms, 'g.', "
<< "send_time_s, playout_delay_ms)" << std::endl;
output << " h=plt.plot(" << kArrivalDelayX << ", " << kArrivalDelayY << ", "
<< kTargetDelayX << ", " << kTargetDelayY << ", 'g.', "
<< kPlayoutDelayX << ", " << kPlayoutDelayY << ")" << std::endl;
output << " plt.setp(h[0],'color',[.75, .75, .75])" << std::endl;
output << " plt.setp(h[1],'markersize',6)" << std::endl;
output << " plt.setp(h[2],'linewidth',1.5)" << std::endl;
output << " plt.axis('tight')" << std::endl;
output << " plt.xlabel('send time [s]')" << std::endl;
output << " plt.xlabel('time [s]')" << std::endl;
output << " plt.ylabel('relative delay [ms]')" << std::endl;
if (!ssrcs_.empty()) {
auto ssrc_it = ssrcs_.cbegin();

14
modules/audio_coding/neteq/tools/neteq_delay_analyzer.h
View File

@ -37,11 +37,11 @@ class NetEqDelayAnalyzer : public test::NetEqPostInsertPacket,
bool muted,
NetEq* neteq) override;
void CreateGraphs(std::vector<float>* send_times_s,
std::vector<float>* arrival_delay_ms,
std::vector<float>* corrected_arrival_delay_ms,
std::vector<absl::optional<float>>* playout_delay_ms,
std::vector<absl::optional<float>>* target_delay_ms) const;
using Delays = std::vector<std::pair<int64_t, float>>;
void CreateGraphs(Delays* arrival_delay_ms,
Delays* corrected_arrival_delay_ms,
Delays* playout_delay_ms,
Delays* target_delay_ms) const;
// Creates a matlab script with file name script_name. When executed in
// Matlab, the script will generate graphs with the same timing information
@ -55,8 +55,8 @@ class NetEqDelayAnalyzer : public test::NetEqPostInsertPacket,
private:
struct TimingData {
explicit TimingData(double at) : arrival_time_ms(at) {}
double arrival_time_ms;
explicit TimingData(int64_t at) : arrival_time_ms(at) {}
int64_t arrival_time_ms;
absl::optional<int64_t> decode_get_audio_count;
absl::optional<int64_t> sync_delay_ms;
absl::optional<int> target_delay_ms;

75
rtc_tools/event_log_visualizer/analyzer.cc
View File

@ -1777,49 +1777,43 @@ EventLogAnalyzer::NetEqStatsGetterMap EventLogAnalyzer::SimulateNetEq(
void EventLogAnalyzer::CreateAudioJitterBufferGraph(
const NetEqStatsGetterMap& neteq_stats,
Plot* plot) const {
if (neteq_stats.size() < 1)
return;
RTC_CHECK(!neteq_stats.empty());
const uint32_t ssrc = neteq_stats.begin()->first;
std::vector<float> send_times_s;
std::vector<float> arrival_delay_ms;
std::vector<float> corrected_arrival_delay_ms;
std::vector<absl::optional<float>> playout_delay_ms;
std::vector<absl::optional<float>> target_delay_ms;
test::NetEqDelayAnalyzer::Delays arrival_delay_ms;
test::NetEqDelayAnalyzer::Delays corrected_arrival_delay_ms;
test::NetEqDelayAnalyzer::Delays playout_delay_ms;
test::NetEqDelayAnalyzer::Delays target_delay_ms;
neteq_stats.at(ssrc)->delay_analyzer()->CreateGraphs(
&send_times_s, &arrival_delay_ms, &corrected_arrival_delay_ms,
&playout_delay_ms, &target_delay_ms);
RTC_DCHECK_EQ(send_times_s.size(), arrival_delay_ms.size());
RTC_DCHECK_EQ(send_times_s.size(), corrected_arrival_delay_ms.size());
RTC_DCHECK_EQ(send_times_s.size(), playout_delay_ms.size());
RTC_DCHECK_EQ(send_times_s.size(), target_delay_ms.size());
&arrival_delay_ms, &corrected_arrival_delay_ms, &playout_delay_ms,
&target_delay_ms);
std::map<uint32_t, TimeSeries> time_series_packet_arrival;
std::map<uint32_t, TimeSeries> time_series_relative_packet_arrival;
std::map<uint32_t, TimeSeries> time_series_play_time;
std::map<uint32_t, TimeSeries> time_series_target_time;
float min_y_axis = 0.f;
float max_y_axis = 0.f;
for (size_t i = 0; i < send_times_s.size(); ++i) {
time_series_packet_arrival[ssrc].points.emplace_back(
TimeSeriesPoint(send_times_s[i], arrival_delay_ms[i]));
for (const auto& data : arrival_delay_ms) {
const float x = ToCallTimeSec(data.first * 1000); // ms to us.
const float y = data.second;
time_series_packet_arrival[ssrc].points.emplace_back(TimeSeriesPoint(x, y));
}
for (const auto& data : corrected_arrival_delay_ms) {
const float x = ToCallTimeSec(data.first * 1000); // ms to us.
const float y = data.second;
time_series_relative_packet_arrival[ssrc].points.emplace_back(
TimeSeriesPoint(send_times_s[i], corrected_arrival_delay_ms[i]));
min_y_axis = std::min(min_y_axis, corrected_arrival_delay_ms[i]);
max_y_axis = std::max(max_y_axis, corrected_arrival_delay_ms[i]);
if (playout_delay_ms[i]) {
time_series_play_time[ssrc].points.emplace_back(
TimeSeriesPoint(send_times_s[i], *playout_delay_ms[i]));
min_y_axis = std::min(min_y_axis, *playout_delay_ms[i]);
max_y_axis = std::max(max_y_axis, *playout_delay_ms[i]);
}
if (target_delay_ms[i]) {
time_series_target_time[ssrc].points.emplace_back(
TimeSeriesPoint(send_times_s[i], *target_delay_ms[i]));
min_y_axis = std::min(min_y_axis, *target_delay_ms[i]);
max_y_axis = std::max(max_y_axis, *target_delay_ms[i]);
}
TimeSeriesPoint(x, y));
}
for (const auto& data : playout_delay_ms) {
const float x = ToCallTimeSec(data.first * 1000); // ms to us.
const float y = data.second;
time_series_play_time[ssrc].points.emplace_back(TimeSeriesPoint(x, y));
}
for (const auto& data : target_delay_ms) {
const float x = ToCallTimeSec(data.first * 1000); // ms to us.
const float y = data.second;
time_series_target_time[ssrc].points.emplace_back(TimeSeriesPoint(x, y));
}
// This code is adapted for a single stream. The creation of the streams above
@ -1847,8 +1841,8 @@ void EventLogAnalyzer::CreateAudioJitterBufferGraph(
plot->SetXAxis(ToCallTimeSec(begin_time_), call_duration_s_, "Time (s)",
kLeftMargin, kRightMargin);
plot->SetYAxis(min_y_axis, max_y_axis, "Relative delay (ms)", kBottomMargin,
kTopMargin);
plot->SetSuggestedYAxis(0, 1, "Relative delay (ms)", kBottomMargin,
kTopMargin);
plot->SetTitle("NetEq timing for " + GetStreamName(kIncomingPacket, ssrc));
}
@ -1857,12 +1851,7 @@ void EventLogAnalyzer::CreateNetEqStatsGraph(
rtc::FunctionView<float(const NetEqNetworkStatistics&)> stats_extractor,
const std::string& plot_name,
Plot* plot) const {
if (neteq_stats.size() < 1)
return;
std::map<uint32_t, TimeSeries> time_series;
float min_y_axis = std::numeric_limits<float>::max();
float max_y_axis = std::numeric_limits<float>::min();
for (const auto& st : neteq_stats) {
const uint32_t ssrc = st.first;
@ -1872,8 +1861,6 @@ void EventLogAnalyzer::CreateNetEqStatsGraph(
const float time = ToCallTimeSec(stats[i].first * 1000); // ms to us.
const float value = stats_extractor(stats[i].second);
time_series[ssrc].points.emplace_back(TimeSeriesPoint(time, value));
min_y_axis = std::min(min_y_axis, value);
max_y_axis = std::max(max_y_axis, value);
}
}
@ -1885,7 +1872,7 @@ void EventLogAnalyzer::CreateNetEqStatsGraph(
plot->SetXAxis(ToCallTimeSec(begin_time_), call_duration_s_, "Time (s)",
kLeftMargin, kRightMargin);
plot->SetYAxis(min_y_axis, max_y_axis, plot_name, kBottomMargin, kTopMargin);
plot->SetSuggestedYAxis(0, 1, plot_name, kBottomMargin, kTopMargin);
plot->SetTitle(plot_name);
}

8
rtc_tools/event_log_visualizer/main.cc
View File

@ -335,8 +335,12 @@ int main(int argc, char* argv[]) {
"audio_processing/conversational_speech/EN_script2_F_sp2_B1", "wav");
}
auto neteq_stats = analyzer.SimulateNetEq(wav_path, 48000);
analyzer.CreateAudioJitterBufferGraph(neteq_stats,
collection->AppendNewPlot());
if (!neteq_stats.empty()) {
analyzer.CreateAudioJitterBufferGraph(neteq_stats,
collection->AppendNewPlot());
}
analyzer.CreateNetEqStatsGraph(
neteq_stats,
[](const webrtc::NetEqNetworkStatistics& stats) {