From 00376e190a9e3741874f32a29dedab7f0f688ef5 Mon Sep 17 00:00:00 2001 From: Johannes Kron Date: Mon, 25 Nov 2019 10:25:42 +0100 Subject: [PATCH] Add totalInterFrameDelay to RTCInboundRTPStreamStats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: webrtc:11108 Change-Id: I0e0168ba303b127a8db3946d5fa5f97a1c90fb27 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/160042 Reviewed-by: Niels Moller Reviewed-by: Ilya Nikolaevskiy Reviewed-by: Henrik Boström Commit-Queue: Johannes Kron Cr-Commit-Position: refs/heads/master@{#29894} --- api/stats/rtcstats_objects.h | 2 + call/video_receive_stream.h | 6 +++ media/base/media_channel.h | 2 + media/engine/webrtc_video_engine.cc | 2 + media/engine/webrtc_video_engine_unittest.cc | 16 ++++++ pc/rtc_stats_collector.cc | 4 ++ pc/rtc_stats_collector_unittest.cc | 5 ++ pc/rtc_stats_integrationtest.cc | 7 +++ stats/rtcstats_objects.cc | 6 +++ video/receive_statistics_proxy.cc | 4 ++ video/receive_statistics_proxy_unittest.cc | 57 ++++++++++++++++++++ 11 files changed, 111 insertions(+) diff --git a/api/stats/rtcstats_objects.h b/api/stats/rtcstats_objects.h index 6ae46812d6..d5202042fe 100644 --- a/api/stats/rtcstats_objects.h +++ b/api/stats/rtcstats_objects.h @@ -444,6 +444,8 @@ class RTC_EXPORT RTCInboundRTPStreamStats final : public RTCRTPStreamStats { RTCStatsMember frames_decoded; RTCStatsMember key_frames_decoded; RTCStatsMember total_decode_time; + RTCStatsMember total_inter_frame_delay; + RTCStatsMember total_squared_inter_frame_delay; // https://henbos.github.io/webrtc-provisional-stats/#dom-rtcinboundrtpstreamstats-contenttype RTCStatsMember content_type; // TODO(asapersson): Currently only populated if audio/video sync is enabled. diff --git a/call/video_receive_stream.h b/call/video_receive_stream.h index cff812637f..2959f67c0d 100644 --- a/call/video_receive_stream.h +++ b/call/video_receive_stream.h @@ -91,6 +91,12 @@ class VideoReceiveStream { uint32_t frames_decoded = 0; // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-totaldecodetime uint64_t total_decode_time_ms = 0; + // Total inter frame delay in seconds. + // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-totalinterframedelay + double total_inter_frame_delay = 0; + // Total squared inter frame delay in seconds^2. + // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-totalsqauredinterframedelay + double total_squared_inter_frame_delay = 0; int64_t first_frame_received_to_decoded_ms = -1; absl::optional qp_sum; diff --git a/media/base/media_channel.h b/media/base/media_channel.h index c49f2ec069..026d371f3a 100644 --- a/media/base/media_channel.h +++ b/media/base/media_channel.h @@ -613,6 +613,8 @@ struct VideoReceiverInfo : public MediaReceiverInfo { absl::optional qp_sum; // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-totaldecodetime uint64_t total_decode_time_ms = 0; + double total_inter_frame_delay = 0; + double total_squared_inter_frame_delay = 0; int64_t interframe_delay_max_ms = -1; uint32_t freeze_count = 0; uint32_t pause_count = 0; diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc index cab4e122e8..16640924c0 100644 --- a/media/engine/webrtc_video_engine.cc +++ b/media/engine/webrtc_video_engine.cc @@ -2846,6 +2846,8 @@ WebRtcVideoChannel::WebRtcVideoReceiveStream::GetVideoReceiverInfo( stats.estimated_playout_ntp_timestamp_ms; info.first_frame_received_to_decoded_ms = stats.first_frame_received_to_decoded_ms; + info.total_inter_frame_delay = stats.total_inter_frame_delay; + info.total_squared_inter_frame_delay = stats.total_squared_inter_frame_delay; info.interframe_delay_max_ms = stats.interframe_delay_max_ms; info.freeze_count = stats.freeze_count; info.pause_count = stats.pause_count; diff --git a/media/engine/webrtc_video_engine_unittest.cc b/media/engine/webrtc_video_engine_unittest.cc index b60ab953e9..362268aaee 100644 --- a/media/engine/webrtc_video_engine_unittest.cc +++ b/media/engine/webrtc_video_engine_unittest.cc @@ -5279,6 +5279,22 @@ TEST_F(WebRtcVideoChannelTest, GetStatsTranslatesDecodeStatsCorrectly) { EXPECT_EQ(stats.total_decode_time_ms, info.receivers[0].total_decode_time_ms); } +TEST_F(WebRtcVideoChannelTest, + GetStatsTranslatesInterFrameDelayStatsCorrectly) { + FakeVideoReceiveStream* stream = AddRecvStream(); + webrtc::VideoReceiveStream::Stats stats; + stats.total_inter_frame_delay = 0.123; + stats.total_squared_inter_frame_delay = 0.00456; + stream->SetStats(stats); + + cricket::VideoMediaInfo info; + ASSERT_TRUE(channel_->GetStats(&info)); + EXPECT_EQ(stats.total_inter_frame_delay, + info.receivers[0].total_inter_frame_delay); + EXPECT_EQ(stats.total_squared_inter_frame_delay, + info.receivers[0].total_squared_inter_frame_delay); +} + TEST_F(WebRtcVideoChannelTest, GetStatsTranslatesReceivePacketStatsCorrectly) { FakeVideoReceiveStream* stream = AddRecvStream(); webrtc::VideoReceiveStream::Stats stats; diff --git a/pc/rtc_stats_collector.cc b/pc/rtc_stats_collector.cc index 481e25541d..98b3bd42d4 100644 --- a/pc/rtc_stats_collector.cc +++ b/pc/rtc_stats_collector.cc @@ -320,6 +320,10 @@ void SetInboundRTPStreamStatsFromVideoReceiverInfo( inbound_video->total_decode_time = static_cast(video_receiver_info.total_decode_time_ms) / rtc::kNumMillisecsPerSec; + inbound_video->total_inter_frame_delay = + video_receiver_info.total_inter_frame_delay; + inbound_video->total_squared_inter_frame_delay = + video_receiver_info.total_squared_inter_frame_delay; if (video_receiver_info.last_packet_received_timestamp_ms) { inbound_video->last_packet_received_timestamp = static_cast( diff --git a/pc/rtc_stats_collector_unittest.cc b/pc/rtc_stats_collector_unittest.cc index 95c2a9b863..b5e3c6b084 100644 --- a/pc/rtc_stats_collector_unittest.cc +++ b/pc/rtc_stats_collector_unittest.cc @@ -1823,6 +1823,9 @@ TEST_F(RTCStatsCollectorTest, CollectRTCInboundRTPStreamStats_Video) { video_media_info.receivers[0].key_frames_decoded = 3; video_media_info.receivers[0].qp_sum = absl::nullopt; video_media_info.receivers[0].total_decode_time_ms = 9000; + video_media_info.receivers[0].total_inter_frame_delay = 0.123; + video_media_info.receivers[0].total_squared_inter_frame_delay = 0.00456; + video_media_info.receivers[0].last_packet_received_timestamp_ms = absl::nullopt; video_media_info.receivers[0].content_type = VideoContentType::UNSPECIFIED; @@ -1865,6 +1868,8 @@ TEST_F(RTCStatsCollectorTest, CollectRTCInboundRTPStreamStats_Video) { expected_video.key_frames_decoded = 3; // |expected_video.qp_sum| should be undefined. expected_video.total_decode_time = 9.0; + expected_video.total_inter_frame_delay = 0.123; + expected_video.total_squared_inter_frame_delay = 0.00456; // |expected_video.last_packet_received_timestamp| should be undefined. // |expected_video.content_type| should be undefined. // |expected_video.decoder_implementation| should be undefined. diff --git a/pc/rtc_stats_integrationtest.cc b/pc/rtc_stats_integrationtest.cc index 31258a2af4..cd61678ed3 100644 --- a/pc/rtc_stats_integrationtest.cc +++ b/pc/rtc_stats_integrationtest.cc @@ -830,6 +830,10 @@ class RTCStatsReportVerifier { verifier.TestMemberIsDefined(inbound_stream.key_frames_decoded); verifier.TestMemberIsNonNegative( inbound_stream.total_decode_time); + verifier.TestMemberIsNonNegative( + inbound_stream.total_inter_frame_delay); + verifier.TestMemberIsNonNegative( + inbound_stream.total_squared_inter_frame_delay); // The integration test is not set up to test screen share; don't require // this to be present. verifier.MarkMemberTested(inbound_stream.content_type, true); @@ -837,6 +841,9 @@ class RTCStatsReportVerifier { verifier.TestMemberIsUndefined(inbound_stream.frames_decoded); verifier.TestMemberIsUndefined(inbound_stream.key_frames_decoded); verifier.TestMemberIsUndefined(inbound_stream.total_decode_time); + verifier.TestMemberIsUndefined(inbound_stream.total_inter_frame_delay); + verifier.TestMemberIsUndefined( + inbound_stream.total_squared_inter_frame_delay); verifier.TestMemberIsUndefined(inbound_stream.content_type); } return verifier.ExpectAllMembersSuccessfullyTested(); diff --git a/stats/rtcstats_objects.cc b/stats/rtcstats_objects.cc index 1037077e0c..4de2aa125c 100644 --- a/stats/rtcstats_objects.cc +++ b/stats/rtcstats_objects.cc @@ -616,6 +616,8 @@ WEBRTC_RTCSTATS_IMPL( &frames_decoded, &key_frames_decoded, &total_decode_time, + &total_inter_frame_delay, + &total_squared_inter_frame_delay, &content_type, &estimated_playout_timestamp, &decoder_implementation) @@ -650,6 +652,8 @@ RTCInboundRTPStreamStats::RTCInboundRTPStreamStats(std::string&& id, frames_decoded("framesDecoded"), key_frames_decoded("keyFramesDecoded"), total_decode_time("totalDecodeTime"), + total_inter_frame_delay("totalInterFrameDelay"), + total_squared_inter_frame_delay("totalSquaredInterFrameDelay"), content_type("contentType"), estimated_playout_timestamp("estimatedPlayoutTimestamp"), decoder_implementation("decoderImplementation") {} @@ -679,6 +683,8 @@ RTCInboundRTPStreamStats::RTCInboundRTPStreamStats( frames_decoded(other.frames_decoded), key_frames_decoded(other.key_frames_decoded), total_decode_time(other.total_decode_time), + total_inter_frame_delay(other.total_inter_frame_delay), + total_squared_inter_frame_delay(other.total_squared_inter_frame_delay), content_type(other.content_type), estimated_playout_timestamp(other.estimated_playout_timestamp), decoder_implementation(other.decoder_implementation) {} diff --git a/video/receive_statistics_proxy.cc b/video/receive_statistics_proxy.cc index 657e98dd08..82951c8a50 100644 --- a/video/receive_statistics_proxy.cc +++ b/video/receive_statistics_proxy.cc @@ -777,6 +777,10 @@ void ReceiveStatisticsProxy::OnDecodedFrame(const VideoFrame& frame, if (last_decoded_frame_time_ms_) { int64_t interframe_delay_ms = now_ms - *last_decoded_frame_time_ms_; RTC_DCHECK_GE(interframe_delay_ms, 0); + double interframe_delay = interframe_delay_ms / 1000.0; + stats_.total_inter_frame_delay += interframe_delay; + stats_.total_squared_inter_frame_delay += + interframe_delay * interframe_delay; interframe_delay_max_moving_.Add(interframe_delay_ms, now_ms); content_specific_stats->interframe_delay_counter.Add(interframe_delay_ms); content_specific_stats->interframe_delay_percentiles.Add( diff --git a/video/receive_statistics_proxy_unittest.cc b/video/receive_statistics_proxy_unittest.cc index eb7c8655ab..a775624656 100644 --- a/video/receive_statistics_proxy_unittest.cc +++ b/video/receive_statistics_proxy_unittest.cc @@ -184,6 +184,63 @@ TEST_F(ReceiveStatisticsProxyTest, ReportsContentType) { statistics_proxy_->GetStats().content_type)); } +TEST_F(ReceiveStatisticsProxyTest, ReportsMaxTotalInterFrameDelay) { + webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); + const TimeDelta kInterFrameDelay1 = TimeDelta::ms(100); + const TimeDelta kInterFrameDelay2 = TimeDelta::ms(200); + const TimeDelta kInterFrameDelay3 = TimeDelta::ms(300); + double expected_total_inter_frame_delay = 0; + double expected_total_squared_inter_frame_delay = 0; + EXPECT_EQ(expected_total_inter_frame_delay, + statistics_proxy_->GetStats().total_inter_frame_delay); + EXPECT_EQ(expected_total_squared_inter_frame_delay, + statistics_proxy_->GetStats().total_squared_inter_frame_delay); + + statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, 0, + VideoContentType::UNSPECIFIED); + EXPECT_DOUBLE_EQ(expected_total_inter_frame_delay, + statistics_proxy_->GetStats().total_inter_frame_delay); + EXPECT_DOUBLE_EQ( + expected_total_squared_inter_frame_delay, + statistics_proxy_->GetStats().total_squared_inter_frame_delay); + + fake_clock_.AdvanceTime(kInterFrameDelay1); + statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, 0, + VideoContentType::UNSPECIFIED); + expected_total_inter_frame_delay += kInterFrameDelay1.seconds(); + expected_total_squared_inter_frame_delay += + pow(kInterFrameDelay1.seconds(), 2.0); + EXPECT_DOUBLE_EQ(expected_total_inter_frame_delay, + statistics_proxy_->GetStats().total_inter_frame_delay); + EXPECT_DOUBLE_EQ( + expected_total_squared_inter_frame_delay, + statistics_proxy_->GetStats().total_squared_inter_frame_delay); + + fake_clock_.AdvanceTime(kInterFrameDelay2); + statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, 0, + VideoContentType::UNSPECIFIED); + expected_total_inter_frame_delay += kInterFrameDelay2.seconds(); + expected_total_squared_inter_frame_delay += + pow(kInterFrameDelay2.seconds(), 2.0); + EXPECT_DOUBLE_EQ(expected_total_inter_frame_delay, + statistics_proxy_->GetStats().total_inter_frame_delay); + EXPECT_DOUBLE_EQ( + expected_total_squared_inter_frame_delay, + statistics_proxy_->GetStats().total_squared_inter_frame_delay); + + fake_clock_.AdvanceTime(kInterFrameDelay3); + statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, 0, + VideoContentType::UNSPECIFIED); + expected_total_inter_frame_delay += kInterFrameDelay3.seconds(); + expected_total_squared_inter_frame_delay += + pow(kInterFrameDelay3.seconds(), 2.0); + EXPECT_DOUBLE_EQ(expected_total_inter_frame_delay, + statistics_proxy_->GetStats().total_inter_frame_delay); + EXPECT_DOUBLE_EQ( + expected_total_squared_inter_frame_delay, + statistics_proxy_->GetStats().total_squared_inter_frame_delay); +} + TEST_F(ReceiveStatisticsProxyTest, ReportsMaxInterframeDelay) { webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); const int64_t kInterframeDelayMs1 = 100;