diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_test.cc b/webrtc/modules/remote_bitrate_estimator/test/bwe_test.cc index 52d7f38be4..72283b98c7 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/bwe_test.cc +++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_test.cc @@ -914,13 +914,9 @@ void BweTest::RunPauseResumeFlows(BandwidthEstimatorType bwe_type) { filter.choke.set_capacity_kbps(3500); RunFor(40 * 1000); // 0-40s. - senders[0].get()->Pause(); - metric_recorders[0].get()->PauseFlow(); RunFor(20 * 1000); // 40-60s. - - senders[0].get()->Resume(); - metric_recorders[0].get()->ResumeFlow(20 * 1000); + senders[0].get()->Resume(20 * 1000); RunFor(60 * 1000); // 60-120s. int64_t paused[] = {20 * 1000, 0, 0}; diff --git a/webrtc/modules/remote_bitrate_estimator/test/metric_recorder.cc b/webrtc/modules/remote_bitrate_estimator/test/metric_recorder.cc index eb068e5657..6202b4a6a3 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/metric_recorder.cc +++ b/webrtc/modules/remote_bitrate_estimator/test/metric_recorder.cc @@ -10,6 +10,8 @@ #include "webrtc/modules/remote_bitrate_estimator/test/metric_recorder.h" +#include "webrtc/modules/remote_bitrate_estimator/test/packet_sender.h" + #include namespace webrtc { @@ -57,7 +59,6 @@ MetricRecorder::MetricRecorder(const std::string algorithm_name, LinkShare* link_share) : algorithm_name_(algorithm_name), flow_id_(flow_id), - packet_sender_(packet_sender), link_share_(link_share), now_ms_(0), sum_delays_ms_(0), @@ -71,6 +72,8 @@ MetricRecorder::MetricRecorder(const std::string algorithm_name, started_computing_metrics_(false), num_packets_received_(0) { std::fill_n(sum_lp_weighted_estimate_error_, 2, 0); + if (packet_sender != nullptr) + packet_sender->set_metric_recorder(this); } void MetricRecorder::SetPlotInformation( @@ -82,6 +85,7 @@ void MetricRecorder::SetPlotInformation( plot_information_[i].prefix = prefixes[i]; } plot_information_[kThroughput].plot_interval_ms = 100; + plot_information_[kSendingEstimate].plot_interval_ms = 100; plot_information_[kDelay].plot_interval_ms = 100; plot_information_[kLoss].plot_interval_ms = 500; plot_information_[kObjective].plot_interval_ms = 1000; @@ -91,8 +95,9 @@ void MetricRecorder::SetPlotInformation( for (int i = kThroughput; i < kNumMetrics; ++i) { plot_information_[i].last_plot_ms = 0; switch (i) { - case kAvailablePerFlow: + case kSendingEstimate: case kObjective: + case kAvailablePerFlow: plot_information_[i].plot = false; break; case kLoss: @@ -143,7 +148,7 @@ void MetricRecorder::PlotLine(int windows_id, static_cast(y), algorithm_name_); } -void MetricRecorder::UpdateTime(int64_t time_ms) { +void MetricRecorder::UpdateTimeMs(int64_t time_ms) { now_ms_ = std::max(now_ms_, time_ms); } @@ -154,7 +159,11 @@ void MetricRecorder::UpdateThroughput(int64_t bitrate_kbps, plot_information_[kThroughput].Update(now_ms_, bitrate_kbps); } -void MetricRecorder::UpdateDelay(int64_t delay_ms) { +void MetricRecorder::UpdateSendingEstimateKbps(int64_t bitrate_kbps) { + plot_information_[kSendingEstimate].Update(now_ms_, bitrate_kbps); +} + +void MetricRecorder::UpdateDelayMs(int64_t delay_ms) { PushDelayMs(delay_ms, now_ms_); plot_information_[kDelay].Update(now_ms_, delay_ms); } @@ -180,9 +189,7 @@ uint32_t MetricRecorder::GetAvailablePerFlowKbps() { } uint32_t MetricRecorder::GetSendingEstimateKbps() { - if (packet_sender_ == nullptr) - return 0; - return packet_sender_->TargetBitrateKbps(); + return static_cast(plot_information_[kSendingEstimate].value); } void MetricRecorder::PushDelayMs(int64_t delay_ms, int64_t arrival_time_ms) { @@ -361,7 +368,7 @@ void MetricRecorder::PauseFlow() { } void MetricRecorder::ResumeFlow(int64_t paused_time_ms) { - UpdateTime(now_ms_ + paused_time_ms); + UpdateTimeMs(now_ms_ + paused_time_ms); PlotZero(); link_share_->ResumeFlow(flow_id_); } diff --git a/webrtc/modules/remote_bitrate_estimator/test/metric_recorder.h b/webrtc/modules/remote_bitrate_estimator/test/metric_recorder.h index 13a76cc6d1..2be13e0b0b 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/metric_recorder.h +++ b/webrtc/modules/remote_bitrate_estimator/test/metric_recorder.h @@ -11,18 +11,21 @@ #ifndef WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_TEST_METRIC_RECORDER_H_ #define WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_TEST_METRIC_RECORDER_H_ +#include #include #include #include #include "webrtc/base/common.h" #include "webrtc/test/testsupport/gtest_prod_util.h" -#include "webrtc/modules/remote_bitrate_estimator/test/packet_sender.h" namespace webrtc { namespace testing { namespace bwe { +class ChokeFilter; +class PacketSender; + class LinkShare { public: explicit LinkShare(ChokeFilter* choke_filter); @@ -79,9 +82,10 @@ class MetricRecorder { void PlotDynamics(int metric); void PlotAllDynamics(); - void UpdateTime(int64_t time_ms); + void UpdateTimeMs(int64_t time_ms); void UpdateThroughput(int64_t bitrate_kbps, size_t payload_size); - void UpdateDelay(int64_t delay_ms); + void UpdateSendingEstimateKbps(int64_t bitrate_kbps); + void UpdateDelayMs(int64_t delay_ms); void UpdateLoss(float loss_ratio); void UpdateObjective(); @@ -146,6 +150,7 @@ class MetricRecorder { enum Metrics { kThroughput = 0, + kSendingEstimate, kDelay, kLoss, kObjective, @@ -156,7 +161,6 @@ class MetricRecorder { std::string algorithm_name_; int flow_id_; - PacketSender* packet_sender_; LinkShare* link_share_; int64_t now_ms_; diff --git a/webrtc/modules/remote_bitrate_estimator/test/metric_recorder_unittest.cc b/webrtc/modules/remote_bitrate_estimator/test/metric_recorder_unittest.cc index b364dfda8d..7d4ed5fd5f 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/metric_recorder_unittest.cc +++ b/webrtc/modules/remote_bitrate_estimator/test/metric_recorder_unittest.cc @@ -10,6 +10,7 @@ #include "webrtc/modules/remote_bitrate_estimator/test/metric_recorder.h" +#include #include #include @@ -46,7 +47,7 @@ TEST_F(MetricRecorderTest, RegularPackets) { for (int i = 0; i < kNumPackets; ++i) { int64_t arrival_time_ms = kInterpacketGapMs * i + kDelayMs; - metric_recorder_.UpdateTime(arrival_time_ms); + metric_recorder_.UpdateTimeMs(arrival_time_ms); metric_recorder_.PushDelayMs(kDelayMs, arrival_time_ms); metric_recorder_.PushThroughputBytes(kPayloadSizeBytes, arrival_time_ms); } @@ -80,7 +81,7 @@ TEST_F(MetricRecorderTest, VariableDelayPackets) { for (int i = 0; i < kNumPackets; ++i) { int64_t arrival_time_ms = kInterpacketGapMs * i + delays_ms[i]; last_received_ms = std::max(last_received_ms, arrival_time_ms); - metric_recorder_.UpdateTime(arrival_time_ms); + metric_recorder_.UpdateTimeMs(arrival_time_ms); metric_recorder_.PushDelayMs(delays_ms[i], arrival_time_ms); metric_recorder_.PushThroughputBytes(kPayloadSizeBytes, arrival_time_ms); } diff --git a/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.cc b/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.cc index 7032b08456..0edfeca4a6 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.cc +++ b/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.cc @@ -44,10 +44,11 @@ PacketReceiver::PacketReceiver(PacketProcessorListener* listener, // Metric recorder plots them in separated figures, // alignment will take place with the #1 left axis. - prefixes.push_back("Throughput_kbps#1"); // Throughput. - prefixes.push_back("Delay_ms_#1"); // Delay. - prefixes.push_back("Packet_Loss_#1"); // Loss. - prefixes.push_back("Objective_function_#1"); // Objective. + prefixes.push_back("Throughput_kbps#1"); + prefixes.push_back("Sending_Estimate_kbps#1"); + prefixes.push_back("Delay_ms_#1"); + prefixes.push_back("Packet_Loss_#1"); + prefixes.push_back("Objective_function_#1"); // Plot Total/PerFlow Available capacity together with throughputs. prefixes.push_back("Throughput_kbps#1"); // Total Available. @@ -93,7 +94,7 @@ void PacketReceiver::RunFor(int64_t time_ms, Packets* in_out) { delay_stats_.Push(arrival_time_ms - send_time_ms); if (metric_recorder_ != nullptr) { - metric_recorder_->UpdateTime(arrival_time_ms); + metric_recorder_->UpdateTimeMs(arrival_time_ms); UpdateMetrics(arrival_time_ms, send_time_ms, media_packet->payload_size()); metric_recorder_->PlotAllDynamics(); @@ -119,7 +120,7 @@ void PacketReceiver::UpdateMetrics(int64_t arrival_time_ms, int64_t send_time_ms, size_t payload_size) { metric_recorder_->UpdateThroughput(bwe_receiver_->RecentKbps(), payload_size); - metric_recorder_->UpdateDelay(arrival_time_ms - send_time_ms); + metric_recorder_->UpdateDelayMs(arrival_time_ms - send_time_ms); metric_recorder_->UpdateLoss(bwe_receiver_->RecentPacketLossRatio()); metric_recorder_->UpdateObjective(); } diff --git a/webrtc/modules/remote_bitrate_estimator/test/packet_sender.cc b/webrtc/modules/remote_bitrate_estimator/test/packet_sender.cc index a7a68aa28b..85cbdef8cf 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/packet_sender.cc +++ b/webrtc/modules/remote_bitrate_estimator/test/packet_sender.cc @@ -17,11 +17,39 @@ #include "webrtc/base/checks.h" #include "webrtc/modules/interface/module_common_types.h" #include "webrtc/modules/remote_bitrate_estimator/test/bwe.h" +#include "webrtc/modules/remote_bitrate_estimator/test/metric_recorder.h" namespace webrtc { namespace testing { namespace bwe { +void PacketSender::Pause() { + running_ = false; + if (metric_recorder_ != nullptr) { + metric_recorder_->PauseFlow(); + } +} + +void PacketSender::Resume(int64_t paused_time_ms) { + running_ = true; + if (metric_recorder_ != nullptr) { + metric_recorder_->ResumeFlow(paused_time_ms); + } +} + +void PacketSender::set_metric_recorder(MetricRecorder* metric_recorder) { + metric_recorder_ = metric_recorder; +} + +void PacketSender::RecordBitrate() { + if (metric_recorder_ != nullptr) { + BWE_TEST_LOGGING_CONTEXT("Sender"); + BWE_TEST_LOGGING_CONTEXT(*flow_ids().begin()); + metric_recorder_->UpdateTimeMs(clock_.TimeInMilliseconds()); + metric_recorder_->UpdateSendingEstimateKbps(TargetBitrateKbps()); + } +} + std::list GetFeedbackPackets(Packets* in_out, int64_t end_time_ms, int flow_id) { @@ -44,7 +72,6 @@ VideoSender::VideoSender(PacketProcessorListener* listener, VideoSource* source, BandwidthEstimatorType estimator_type) : PacketSender(listener, source->flow_id()), - running_(true), source_(source), bwe_(CreateBweSender(estimator_type, source_->bits_per_second() / 1000, @@ -57,6 +84,16 @@ VideoSender::VideoSender(PacketProcessorListener* listener, VideoSender::~VideoSender() { } +void VideoSender::Pause() { + previous_sending_bitrate_ = TargetBitrateKbps(); + PacketSender::Pause(); +} + +void VideoSender::Resume(int64_t paused_time_ms) { + source_->SetBitrateBps(previous_sending_bitrate_); + PacketSender::Resume(paused_time_ms); +} + void VideoSender::RunFor(int64_t time_ms, Packets* in_out) { std::list feedbacks = GetFeedbackPackets( in_out, clock_.TimeInMilliseconds() + time_ms, source_->flow_id()); @@ -109,16 +146,7 @@ void VideoSender::OnNetworkChanged(uint32_t target_bitrate_bps, uint8_t fraction_lost, int64_t rtt) { source_->SetBitrateBps(target_bitrate_bps); -} - -void VideoSender::Pause() { - running_ = false; - previous_sending_bitrate_ = TargetBitrateKbps(); -} - -void VideoSender::Resume() { - running_ = true; - source_->SetBitrateBps(previous_sending_bitrate_); + RecordBitrate(); } uint32_t VideoSender::TargetBitrateKbps() { @@ -309,7 +337,6 @@ TcpSender::TcpSender(PacketProcessorListener* listener, last_rtt_ms_(0), total_sent_bytes_(0), send_limit_bytes_(send_limit_bytes), - running_(true), last_generated_packets_ms_(0), num_recent_sent_packets_(0), bitrate_kbps_(0) { @@ -319,18 +346,16 @@ void TcpSender::RunFor(int64_t time_ms, Packets* in_out) { if (clock_.TimeInMilliseconds() + time_ms < offset_ms_) { clock_.AdvanceTimeMilliseconds(time_ms); if (running_) { - running_ = false; + Pause(); } return; } - if (!running_) { - running_ = true; + if (!running_ && total_sent_bytes_ == 0) { + Resume(offset_ms_); } int64_t start_time_ms = clock_.TimeInMilliseconds(); - BWE_TEST_LOGGING_CONTEXT("Sender"); - BWE_TEST_LOGGING_CONTEXT(*flow_ids().begin()); std::list feedbacks = GetFeedbackPackets( in_out, clock_.TimeInMilliseconds() + time_ms, *flow_ids().begin()); @@ -426,7 +451,7 @@ Packets TcpSender::GeneratePackets(size_t num_packets) { for (size_t i = 0; i < num_packets; ++i) { if ((total_sent_bytes_ + kPacketSizeBytes) > send_limit_bytes_) { if (running_) { - running_ = false; + Pause(); } break; } @@ -454,6 +479,8 @@ void TcpSender::UpdateSendBitrateEstimate(size_t num_packets) { last_generated_packets_ms_ = clock_.TimeInMilliseconds(); num_recent_sent_packets_ = 0; } + + RecordBitrate(); } uint32_t TcpSender::TargetBitrateKbps() { diff --git a/webrtc/modules/remote_bitrate_estimator/test/packet_sender.h b/webrtc/modules/remote_bitrate_estimator/test/packet_sender.h index 8f0b3cdfcb..a6acea0395 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/packet_sender.h +++ b/webrtc/modules/remote_bitrate_estimator/test/packet_sender.h @@ -25,14 +25,18 @@ namespace webrtc { namespace testing { namespace bwe { +class MetricRecorder; + class PacketSender : public PacketProcessor { public: PacketSender(PacketProcessorListener* listener, int flow_id) : PacketProcessor(listener, flow_id, kSender), + running_(true), // For Packet::send_time_us() to be comparable with timestamps from // clock_, the clock of the PacketSender and the Source must be aligned. // We assume that both start at time 0. - clock_(0) {} + clock_(0), + metric_recorder_(nullptr) {} virtual ~PacketSender() {} // Call GiveFeedback() with the returned interval in milliseconds, provided // there is a new estimate available. @@ -44,8 +48,18 @@ class PacketSender : public PacketProcessor { virtual uint32_t TargetBitrateKbps() { return 0; } + virtual void Pause(); + virtual void Resume(int64_t paused_time_ms); + + void set_metric_recorder(MetricRecorder* metric_recorder); + virtual void RecordBitrate(); + protected: + bool running_; // Initialized by default as true. SimulatedClock clock_; + + private: + MetricRecorder* metric_recorder_; }; class VideoSender : public PacketSender, public BitrateObserver { @@ -67,15 +81,14 @@ class VideoSender : public PacketSender, public BitrateObserver { uint8_t fraction_lost, int64_t rtt) override; - void Pause(); - void Resume(); + void Pause() override; + void Resume(int64_t paused_time_ms) override; protected: void ProcessFeedbackAndGeneratePackets(int64_t time_ms, std::list* feedbacks, Packets* generated); - bool running_; VideoSource* source_; rtc::scoped_ptr bwe_; int64_t start_of_run_ms_; @@ -171,7 +184,6 @@ class TcpSender : public PacketSender { int64_t last_rtt_ms_; int total_sent_bytes_; int send_limit_bytes_; // Initialized by default as kNoLimit. - bool running_; // Initialized by default as true. int64_t last_generated_packets_ms_; size_t num_recent_sent_packets_; uint32_t bitrate_kbps_; diff --git a/webrtc/modules/remote_bitrate_estimator/test/plot_dynamics.py b/webrtc/modules/remote_bitrate_estimator/test/plot_dynamics.py index 63104d7866..a37d95479a 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/plot_dynamics.py +++ b/webrtc/modules/remote_bitrate_estimator/test/plot_dynamics.py @@ -73,27 +73,32 @@ def plotVar(v, ax, show_legend, show_x_label): ax.set_ylabel(v.getYLabel(), fontsize='large') for alg in v._samples.keys(): - i = 1 + for series in v._samples[alg].keys(): + x = [sample[0] for sample in v._samples[alg][series]] y = [sample[1] for sample in v._samples[alg][series]] x = numpy.array(x) y = numpy.array(y) - line = plt.plot(x, y, label=alg, linewidth=4.0) - colormap = {'Available1':'#AAAAAA', - 'Available2':'#AAAAAA', - 'GCC1':'#80D000', - 'GCC2':'#008000', - 'GCC3':'#00F000', - 'GCC4':'#00B000', - 'GCC5':'#70B020', - 'NADA1':'#0000AA', - 'NADA2':'#A0A0FF', - 'NADA3':'#0000FF', - 'NADA4':'#C0A0FF', - 'NADA5':'#9060B0',} - key = alg + str(i) + line = plt.plot(x, y, label=alg, linewidth=4.0) + + colormap = {'Available0':'#AAAAAA', + 'Available1':'#AAAAAA', + 'GCC0':'#80D000', + 'GCC1':'#008000', + 'GCC2':'#00F000', + 'GCC3':'#00B000', + 'GCC4':'#70B020', + 'NADA0':'#0000AA', + 'NADA1':'#A0A0FF', + 'NADA2':'#0000FF', + 'NADA3':'#C0A0FF', + 'NADA4':'#9060B0',} + + flow_id = re.search(r'(\d+(,\d+)*)', series) # One or multiple ids. + key = alg + flow_id.group(1) + if key in colormap: plt.setp(line, color=colormap[key]) elif alg == 'TCP': @@ -109,11 +114,10 @@ def plotVar(v, ax, show_legend, show_x_label): if v.getYMax() >= 0: y2 = v.getYMax() plt.axis((0, x2, 0, y2)) - i += 1 if show_legend: legend = plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.40), - shadow=True, fontsize='medium', ncol=len(v._samples)) + shadow=True, fontsize='large', ncol=len(v._samples)) if __name__ == '__main__': @@ -121,6 +125,8 @@ if __name__ == '__main__': ('Throughput_kbps', "Time (s)", "Throughput (kbps)", 1, 4000), ('Delay_ms', "Time (s)", "One-way Delay (ms)", 2, 500), ('Packet_Loss', "Time (s)", "Packet Loss Ratio", 3, 1.0), + # ('Sending_Estimate_kbps', "Time (s)", "Sending Estimate (kbps)", + # 4, 4000), ] var = [] @@ -138,7 +144,7 @@ if __name__ == '__main__': if v.getID() in line: v.addSample(line) - matplotlib.rcParams.update({'font.size': 20}) + matplotlib.rcParams.update({'font.size': 48/len(variables)}) # Plot variables. fig = plt.figure()