From 14b42c20b0e6f87ab13f50676b74d9377fb4bddf Mon Sep 17 00:00:00 2001 From: Artem Titov Date: Mon, 26 Sep 2022 13:21:14 +0200 Subject: [PATCH] Migrate call_perf_tests to new perf metrics export API Bug: b/246095034 Change-Id: I23add90d8a70ae56e3c1c8b550276344645965ee Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/276625 Reviewed-by: Danil Chapovalov Reviewed-by: Mirko Bonadei Commit-Queue: Artem Titov Cr-Commit-Position: refs/heads/main@{#38202} --- call/BUILD.gn | 4 ++- call/call_perf_tests.cc | 72 +++++++++++++++++++++++++---------------- call/rampup_tests.cc | 70 +++++++++++++++++++++------------------ call/rampup_tests.h | 7 ++-- 4 files changed, 90 insertions(+), 63 deletions(-) diff --git a/call/BUILD.gn b/call/BUILD.gn index 36f42d868f..efcb59107e 100644 --- a/call/BUILD.gn +++ b/call/BUILD.gn @@ -548,11 +548,14 @@ if (rtc_include_tests) { "../api:rtc_event_log_output_file", "../api:simulated_network_api", "../api/audio_codecs:builtin_audio_encoder_factory", + "../api/numerics", "../api/rtc_event_log", "../api/rtc_event_log:rtc_event_log_factory", "../api/task_queue", "../api/task_queue:default_task_queue_factory", "../api/task_queue:pending_task_safety_flag", + "../api/test/metrics:global_metrics_logger_and_exporter", + "../api/test/metrics:metric", "../api/video:builtin_video_bitrate_allocator_factory", "../api/video:video_bitrate_allocation", "../api/video_codecs:video_codecs_api", @@ -584,7 +587,6 @@ if (rtc_include_tests) { "../test:field_trial", "../test:fileutils", "../test:null_transport", - "../test:perf_test", "../test:test_common", "../test:test_support", "../test:video_test_common", diff --git a/call/call_perf_tests.cc b/call/call_perf_tests.cc index 7fa7ee1234..3b6db5d626 100644 --- a/call/call_perf_tests.cc +++ b/call/call_perf_tests.cc @@ -15,9 +15,12 @@ #include "absl/strings/string_view.h" #include "api/audio_codecs/builtin_audio_encoder_factory.h" +#include "api/numerics/samples_stats_counter.h" #include "api/rtc_event_log/rtc_event_log.h" #include "api/task_queue/pending_task_safety_flag.h" #include "api/task_queue/task_queue_base.h" +#include "api/test/metrics/global_metrics_logger_and_exporter.h" +#include "api/test/metrics/metric.h" #include "api/test/simulated_network.h" #include "api/video/builtin_video_bitrate_allocator_factory.h" #include "api/video/video_bitrate_allocation.h" @@ -49,7 +52,6 @@ #include "test/null_transport.h" #include "test/rtp_rtcp_observer.h" #include "test/testsupport/file_utils.h" -#include "test/testsupport/perf_test.h" #include "test/video_encoder_proxy_factory.h" #include "video/transport_adapter.h" @@ -57,9 +59,15 @@ using webrtc::test::DriftingClock; namespace webrtc { namespace { + +using ::webrtc::test::GetGlobalMetricsLogger; +using ::webrtc::test::ImprovementDirection; +using ::webrtc::test::Unit; + enum : int { // The first valid value is 1. kTransportSequenceNumberExtensionId = 1, }; + } // namespace class CallPerfTest : public test::CallTest { @@ -133,15 +141,16 @@ class VideoRtcpAndSyncObserver : public test::RtpRtcpObserver, if (std::abs(stats.sync_offset_ms) < kInSyncThresholdMs) { if (first_time_in_sync_ == -1) { first_time_in_sync_ = now_ms; - webrtc::test::PrintResult("sync_convergence_time", test_label_, - "synchronization", time_since_creation, "ms", - false); + GetGlobalMetricsLogger()->LogSingleValueMetric( + "sync_convergence_time" + test_label_, "synchronization", + time_since_creation, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter); } if (time_since_creation > kMinRunTimeMs) observation_complete_.Set(); } if (first_time_in_sync_ != -1) - sync_offset_ms_list_.push_back(stats.sync_offset_ms); + sync_offset_ms_list_.AddSample(stats.sync_offset_ms); } void set_receive_stream(VideoReceiveStreamInterface* receive_stream) { @@ -151,8 +160,9 @@ class VideoRtcpAndSyncObserver : public test::RtpRtcpObserver, } void PrintResults() { - test::PrintResultList("stream_offset", test_label_, "synchronization", - sync_offset_ms_list_, "ms", false); + GetGlobalMetricsLogger()->LogMetric( + "stream_offset" + test_label_, "synchronization", sync_offset_ms_list_, + Unit::kMilliseconds, ImprovementDirection::kNeitherIsBetter); } private: @@ -161,7 +171,7 @@ class VideoRtcpAndSyncObserver : public test::RtpRtcpObserver, const int64_t creation_time_ms_; int64_t first_time_in_sync_ = -1; VideoReceiveStreamInterface* receive_stream_ = nullptr; - std::vector sync_offset_ms_list_; + SamplesStatsCounter sync_offset_ms_list_; TaskQueueBase* const task_queue_; }; @@ -451,7 +461,7 @@ void CallPerfTest::TestCaptureNtpTime( uint32_t real_capture_timestamp = iter->second; int time_offset_ms = real_capture_timestamp - estimated_capture_timestamp; time_offset_ms = time_offset_ms / 90; - time_offset_ms_list_.push_back(time_offset_ms); + time_offset_ms_list_.AddSample(time_offset_ms); EXPECT_TRUE(std::abs(time_offset_ms) < threshold_ms_); } @@ -495,8 +505,9 @@ void CallPerfTest::TestCaptureNtpTime( void PerformTest() override { EXPECT_TRUE(Wait()) << "Timed out while waiting for estimated capture " "NTP time to be within bounds."; - test::PrintResultList("capture_ntp_time", "", "real - estimated", - time_offset_ms_list_, "ms", true); + GetGlobalMetricsLogger()->LogMetric( + "capture_ntp_time", "real - estimated", time_offset_ms_list_, + Unit::kMilliseconds, ImprovementDirection::kNeitherIsBetter); } Mutex mutex_; @@ -511,7 +522,7 @@ void CallPerfTest::TestCaptureNtpTime( uint32_t rtp_start_timestamp_; typedef std::map FrameCaptureTimeList; FrameCaptureTimeList capture_time_list_ RTC_GUARDED_BY(&mutex_); - std::vector time_offset_ms_list_; + SamplesStatsCounter time_offset_ms_list_; } test(net_config, threshold_ms, start_time_ms, run_time_ms); RunBaseTest(&test); @@ -696,7 +707,7 @@ void CallPerfTest::TestMinTransmitBitrate(bool pad_to_min_bitrate) { observation_complete_.Set(); } if (converged_) - bitrate_kbps_list_.push_back(bitrate_kbps); + bitrate_kbps_list_.AddSample(bitrate_kbps); } })); return SEND_PACKET; @@ -723,11 +734,12 @@ void CallPerfTest::TestMinTransmitBitrate(bool pad_to_min_bitrate) { void PerformTest() override { EXPECT_TRUE(Wait()) << "Timeout while waiting for send-bitrate stats."; - test::PrintResultList( - "bitrate_stats_", - (pad_to_min_bitrate_ ? "min_transmit_bitrate" - : "without_min_transmit_bitrate"), - "bitrate_kbps", bitrate_kbps_list_, "kbps", false); + GetGlobalMetricsLogger()->LogMetric( + std::string("bitrate_stats_") + + (pad_to_min_bitrate_ ? "min_transmit_bitrate" + : "without_min_transmit_bitrate"), + "bitrate_kbps", bitrate_kbps_list_, Unit::kKilobitsPerSecond, + ImprovementDirection::kNeitherIsBetter); } VideoSendStream* send_stream_; @@ -736,7 +748,7 @@ void CallPerfTest::TestMinTransmitBitrate(bool pad_to_min_bitrate) { const int min_acceptable_bitrate_; const int max_acceptable_bitrate_; int num_bitrate_observations_in_range_; - std::vector bitrate_kbps_list_; + SamplesStatsCounter bitrate_kbps_list_; TaskQueueBase* task_queue_; rtc::scoped_refptr task_safety_flag_; } test(pad_to_min_bitrate, task_queue()); @@ -1003,8 +1015,9 @@ void CallPerfTest::TestMinAudioVideoBitrate(int test_bitrate_from, } EXPECT_GT(last_passed_test_bitrate, -1) << "Minimum supported bitrate out of the test scope"; - webrtc::test::PrintResult("min_test_bitrate_", "", "min_bitrate", - last_passed_test_bitrate, "kbps", false); + GetGlobalMetricsLogger()->LogSingleValueMetric( + "min_test_bitrate_", "min_bitrate", last_passed_test_bitrate, + Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); } void OnCallsCreated(Call* sender_call, Call* receiver_call) override { @@ -1130,11 +1143,14 @@ void CallPerfTest::TestEncodeFramerate(VideoEncoderFactory* encoder_factory, input_fps = std::max(configured_framerate.second, input_fps); } for (const auto& encode_frame_rate_list : encode_frame_rate_lists_) { - const std::vector& values = encode_frame_rate_list.second; - test::PrintResultList("substream", "", "encode_frame_rate", values, - "fps", false); - double average_fps = - std::accumulate(values.begin(), values.end(), 0.0) / values.size(); + const SamplesStatsCounter& values = encode_frame_rate_list.second; + GetGlobalMetricsLogger()->LogMetric( + "substream_fps", "encode_frame_rate", values, Unit::kUnitless, + ImprovementDirection::kNeitherIsBetter); + if (values.IsEmpty()) { + continue; + } + double average_fps = values.GetAverage(); uint32_t ssrc = encode_frame_rate_list.first; double expected_fps = configured_framerates_.find(ssrc)->second; if (expected_fps != input_fps) @@ -1149,7 +1165,7 @@ void CallPerfTest::TestEncodeFramerate(VideoEncoderFactory* encoder_factory, task_queue_->PostTask([this, now]() { VideoSendStream::Stats stats = send_stream_->GetStats(); for (const auto& stat : stats.substreams) { - encode_frame_rate_lists_[stat.first].push_back( + encode_frame_rate_lists_[stat.first].AddSample( stat.second.encode_frame_rate); } if (now - start_time_ > kMinRunTime) { @@ -1169,7 +1185,7 @@ void CallPerfTest::TestEncodeFramerate(VideoEncoderFactory* encoder_factory, const Timestamp start_time_; Timestamp last_getstats_time_; VideoSendStream* send_stream_; - std::map> encode_frame_rate_lists_; + std::map encode_frame_rate_lists_; std::map configured_framerates_; } test(encoder_factory, payload_name, max_framerates, task_queue()); diff --git a/call/rampup_tests.cc b/call/rampup_tests.cc index 2c4724378b..eea54ed37b 100644 --- a/call/rampup_tests.cc +++ b/call/rampup_tests.cc @@ -19,6 +19,8 @@ #include "api/task_queue/default_task_queue_factory.h" #include "api/task_queue/task_queue_base.h" #include "api/task_queue/task_queue_factory.h" +#include "api/test/metrics/global_metrics_logger_and_exporter.h" +#include "api/test/metrics/metric.h" #include "call/fake_network_pipe.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" @@ -28,7 +30,6 @@ #include "rtc_base/time_utils.h" #include "test/encoder_settings.h" #include "test/gtest.h" -#include "test/testsupport/perf_test.h" ABSL_FLAG(std::string, ramp_dump_name, @@ -38,6 +39,10 @@ ABSL_FLAG(std::string, namespace webrtc { namespace { +using ::webrtc::test::GetGlobalMetricsLogger; +using ::webrtc::test::ImprovementDirection; +using ::webrtc::test::Unit; + constexpr TimeDelta kPollInterval = TimeDelta::Millis(20); static const int kExpectedHighVideoBitrateBps = 80000; static const int kExpectedHighAudioBitrateBps = 30000; @@ -52,6 +57,7 @@ std::vector GenerateSsrcs(size_t num_streams, uint32_t ssrc_offset) { ssrcs.push_back(static_cast(ssrc_offset + i)); return ssrcs; } + } // namespace RampUpTester::RampUpTester(size_t num_video_streams, @@ -332,12 +338,12 @@ void RampUpTester::PollStats() { void RampUpTester::ReportResult( absl::string_view measurement, size_t value, - absl::string_view units, - test::ImproveDirection improve_direction) const { - webrtc::test::PrintResult( - measurement, "", + Unit unit, + ImprovementDirection improvement_direction) const { + GetGlobalMetricsLogger()->LogSingleValueMetric( + measurement, ::testing::UnitTest::GetInstance()->current_test_info()->name(), value, - units, false); + unit, improvement_direction); } void RampUpTester::AccumulateStats(const VideoSendStream::StreamStats& stream, @@ -392,21 +398,21 @@ void RampUpTester::TriggerTestDone() { } if (report_perf_stats_) { - ReportResult("ramp-up-media-sent", media_sent, "bytes", - test::ImproveDirection::kBiggerIsBetter); - ReportResult("ramp-up-padding-sent", padding_sent, "bytes", - test::ImproveDirection::kSmallerIsBetter); - ReportResult("ramp-up-rtx-media-sent", rtx_media_sent, "bytes", - test::ImproveDirection::kBiggerIsBetter); - ReportResult("ramp-up-rtx-padding-sent", rtx_padding_sent, "bytes", - test::ImproveDirection::kSmallerIsBetter); + ReportResult("ramp-up-media-sent", media_sent, Unit::kBytes, + ImprovementDirection::kBiggerIsBetter); + ReportResult("ramp-up-padding-sent", padding_sent, Unit::kBytes, + ImprovementDirection::kSmallerIsBetter); + ReportResult("ramp-up-rtx-media-sent", rtx_media_sent, Unit::kBytes, + ImprovementDirection::kBiggerIsBetter); + ReportResult("ramp-up-rtx-padding-sent", rtx_padding_sent, Unit::kBytes, + ImprovementDirection::kSmallerIsBetter); if (ramp_up_finished_ms_ >= 0) { ReportResult("ramp-up-time", ramp_up_finished_ms_ - test_start_ms_, - "milliseconds", test::ImproveDirection::kSmallerIsBetter); + Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter); } ReportResult("ramp-up-average-network-latency", - send_transport_->GetAverageDelayMs(), "milliseconds", - test::ImproveDirection::kSmallerIsBetter); + send_transport_->GetAverageDelayMs(), Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter); } } @@ -531,10 +537,10 @@ void RampUpDownUpTester::EvolveTestState(int bitrate_bps, bool suspended) { EXPECT_FALSE(suspended); if (bitrate_bps >= GetExpectedHighBitrate()) { if (report_perf_stats_) { - webrtc::test::PrintResult("ramp_up_down_up", GetModifierString(), - "first_rampup", now - state_start_ms_, "ms", - false, - test::ImproveDirection::kSmallerIsBetter); + GetGlobalMetricsLogger()->LogSingleValueMetric( + "ramp_up_down_up" + GetModifierString(), "first_rampup", + now - state_start_ms_, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter); } // Apply loss during the transition between states if FEC is enabled. forward_transport_config_.loss_percent = loss_rates_[test_state_]; @@ -548,10 +554,10 @@ void RampUpDownUpTester::EvolveTestState(int bitrate_bps, bool suspended) { if (bitrate_bps < kLowBandwidthLimitBps + kLowBitrateMarginBps && suspended == check_suspend_state) { if (report_perf_stats_) { - webrtc::test::PrintResult("ramp_up_down_up", GetModifierString(), - "rampdown", now - state_start_ms_, "ms", - false, - test::ImproveDirection::kSmallerIsBetter); + GetGlobalMetricsLogger()->LogSingleValueMetric( + "ramp_up_down_up" + GetModifierString(), "rampdown", + now - state_start_ms_, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter); } // Apply loss during the transition between states if FEC is enabled. forward_transport_config_.loss_percent = loss_rates_[test_state_]; @@ -563,13 +569,14 @@ void RampUpDownUpTester::EvolveTestState(int bitrate_bps, bool suspended) { case kSecondRampup: if (bitrate_bps >= GetExpectedHighBitrate() && !suspended) { if (report_perf_stats_) { - webrtc::test::PrintResult("ramp_up_down_up", GetModifierString(), - "second_rampup", now - state_start_ms_, - "ms", false, - test::ImproveDirection::kSmallerIsBetter); + GetGlobalMetricsLogger()->LogSingleValueMetric( + "ramp_up_down_up" + GetModifierString(), "second_rampup", + now - state_start_ms_, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter); ReportResult("ramp-up-down-up-average-network-latency", - send_transport_->GetAverageDelayMs(), "milliseconds", - test::ImproveDirection::kSmallerIsBetter); + send_transport_->GetAverageDelayMs(), + Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter); } // Apply loss during the transition between states if FEC is enabled. forward_transport_config_.loss_percent = loss_rates_[test_state_]; @@ -729,4 +736,5 @@ TEST_F(RampUpTest, AudioTransportSequenceNumber) { false, task_queue()); RunBaseTest(&test); } + } // namespace webrtc diff --git a/call/rampup_tests.h b/call/rampup_tests.h index 4e9217fa58..de0022c994 100644 --- a/call/rampup_tests.h +++ b/call/rampup_tests.h @@ -20,13 +20,13 @@ #include "absl/strings/string_view.h" #include "api/rtc_event_log/rtc_event_log.h" #include "api/task_queue/task_queue_base.h" +#include "api/test/metrics/metric.h" #include "api/test/simulated_network.h" #include "call/call.h" #include "call/simulated_network.h" #include "rtc_base/event.h" #include "rtc_base/task_utils/repeating_task.h" #include "test/call_test.h" -#include "test/testsupport/perf_test.h" namespace webrtc { @@ -68,8 +68,8 @@ class RampUpTester : public test::EndToEndTest { void ReportResult(absl::string_view measurement, size_t value, - absl::string_view units, - test::ImproveDirection improve_direction) const; + test::Unit unit, + test::ImprovementDirection improvement_direction) const; void TriggerTestDone(); Clock* const clock_; @@ -166,5 +166,6 @@ class RampUpDownUpTester : public RampUpTester { int sent_bytes_; std::vector loss_rates_; }; + } // namespace webrtc #endif // CALL_RAMPUP_TESTS_H_