From 44125faba5bf646116c5a6335ac46c3b544808e8 Mon Sep 17 00:00:00 2001 From: Henrik Lundin Date: Mon, 29 Apr 2019 17:00:46 +0200 Subject: [PATCH] Reland "Piping audio interruption metrics to API layer" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The metrics are now added as RTCNonStandardStatsMember objects in RTCMediaStreamTrackStats. Unit tests are updated. This is a reland of https://webrtc-review.googlesource.com/c/src/+/134303, with fixes. TBR=kwiberg@webrtc.org Bug: webrtc:10549 Change-Id: I29dcc6fbfc69156715664e71acfa054c1b2d9038 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/134500 Commit-Queue: Henrik Lundin Reviewed-by: Henrik Boström Reviewed-by: Ivo Creusen Cr-Commit-Position: refs/heads/master@{#27806} --- api/stats/rtcstats_objects.h | 4 ++++ audio/audio_receive_stream.cc | 2 ++ call/audio_receive_stream.h | 2 ++ media/base/media_channel.h | 4 ++++ media/engine/webrtc_voice_engine.cc | 2 ++ modules/audio_coding/acm2/acm_receiver.cc | 3 +++ .../include/audio_coding_module_typedefs.h | 4 ++++ modules/audio_coding/neteq/include/neteq.h | 4 ++-- .../audio_coding/neteq/neteq_impl_unittest.cc | 2 +- .../neteq/statistics_calculator_unittest.cc | 18 +++++++++--------- .../neteq/tools/neteq_stats_plotter.cc | 5 ++--- pc/rtc_stats_collector.cc | 7 +++++++ pc/rtc_stats_collector_unittest.cc | 4 ++++ pc/rtc_stats_integrationtest.cc | 7 +++++++ stats/rtcstats_objects.cc | 6 ++++++ 15 files changed, 59 insertions(+), 15 deletions(-) diff --git a/api/stats/rtcstats_objects.h b/api/stats/rtcstats_objects.h index 4ddfa93618..96c7a033e8 100644 --- a/api/stats/rtcstats_objects.h +++ b/api/stats/rtcstats_objects.h @@ -321,6 +321,10 @@ class RTC_EXPORT RTCMediaStreamTrackStats final : public RTCStats { RTCNonStandardStatsMember jitter_buffer_flushes; RTCNonStandardStatsMember delayed_packet_outage_samples; RTCNonStandardStatsMember relative_packet_arrival_delay; + // TODO(henrik.lundin): Add description of the interruption metrics at + // https://github.com/henbos/webrtc-provisional-stats/issues/17 + RTCNonStandardStatsMember interruption_count; + RTCNonStandardStatsMember total_interruption_duration; // Non-standard video-only members. // https://henbos.github.io/webrtc-provisional-stats/#RTCVideoReceiverStats-dict* RTCNonStandardStatsMember freeze_count; diff --git a/audio/audio_receive_stream.cc b/audio/audio_receive_stream.cc index b4948eed40..677ee207ee 100644 --- a/audio/audio_receive_stream.cc +++ b/audio/audio_receive_stream.cc @@ -226,6 +226,8 @@ webrtc::AudioReceiveStream::Stats AudioReceiveStream::GetStats() const { stats.relative_packet_arrival_delay_seconds = static_cast(ns.relativePacketArrivalDelayMs) / static_cast(rtc::kNumMillisecsPerSec); + stats.interruption_count = ns.interruptionCount; + stats.total_interruption_duration_ms = ns.totalInterruptionDurationMs; auto ds = channel_receive_->GetDecodingCallStatistics(); stats.decoding_calls_to_silence_generator = ds.calls_to_silence_generator; diff --git a/call/audio_receive_stream.h b/call/audio_receive_stream.h index 257042b56d..9091afd5ce 100644 --- a/call/audio_receive_stream.h +++ b/call/audio_receive_stream.h @@ -79,6 +79,8 @@ class AudioReceiveStream { absl::optional last_packet_received_timestamp_ms; uint64_t jitter_buffer_flushes = 0; double relative_packet_arrival_delay_seconds = 0.0; + int32_t interruption_count = 0; + int32_t total_interruption_duration_ms = 0; }; struct Config { diff --git a/media/base/media_channel.h b/media/base/media_channel.h index 710fd1a713..69570e7e65 100644 --- a/media/base/media_channel.h +++ b/media/base/media_channel.h @@ -514,6 +514,10 @@ struct VoiceReceiverInfo : public MediaReceiverInfo { uint64_t delayed_packet_outage_samples = 0; // Arrival delay of received audio packets. double relative_packet_arrival_delay_seconds = 0.0; + // Count and total duration of audio interruptions (loss-concealement periods + // longer than 150 ms). + int32_t interruption_count = 0; + int32_t total_interruption_duration_ms = 0; }; struct VideoSenderInfo : public MediaSenderInfo { diff --git a/media/engine/webrtc_voice_engine.cc b/media/engine/webrtc_voice_engine.cc index 56cef2c3cc..3b85064dd8 100644 --- a/media/engine/webrtc_voice_engine.cc +++ b/media/engine/webrtc_voice_engine.cc @@ -2264,6 +2264,8 @@ bool WebRtcVoiceMediaChannel::GetStats(VoiceMediaInfo* info) { rinfo.jitter_buffer_flushes = stats.jitter_buffer_flushes; rinfo.relative_packet_arrival_delay_seconds = stats.relative_packet_arrival_delay_seconds; + rinfo.interruption_count = stats.interruption_count; + rinfo.total_interruption_duration_ms = stats.total_interruption_duration_ms; info->receivers.push_back(rinfo); } diff --git a/modules/audio_coding/acm2/acm_receiver.cc b/modules/audio_coding/acm2/acm_receiver.cc index da7d62101b..c10a71ca6b 100644 --- a/modules/audio_coding/acm2/acm_receiver.cc +++ b/modules/audio_coding/acm2/acm_receiver.cc @@ -259,6 +259,9 @@ void AcmReceiver::GetNetworkStatistics(NetworkStatistics* acm_stat) { neteq_lifetime_stat.delayed_packet_outage_samples; acm_stat->relativePacketArrivalDelayMs = neteq_lifetime_stat.relative_packet_arrival_delay_ms; + acm_stat->interruptionCount = neteq_lifetime_stat.interruption_count; + acm_stat->totalInterruptionDurationMs = + neteq_lifetime_stat.total_interruption_duration_ms; NetEqOperationsAndState neteq_operations_and_state = neteq_->GetOperationsAndState(); diff --git a/modules/audio_coding/include/audio_coding_module_typedefs.h b/modules/audio_coding/include/audio_coding_module_typedefs.h index 8063a29620..621c478dc1 100644 --- a/modules/audio_coding/include/audio_coding_module_typedefs.h +++ b/modules/audio_coding/include/audio_coding_module_typedefs.h @@ -130,6 +130,10 @@ struct NetworkStatistics { uint64_t delayedPacketOutageSamples; // arrival delay of incoming packets uint64_t relativePacketArrivalDelayMs; + // number of audio interruptions + int32_t interruptionCount; + // total duration of audio interruptions + int32_t totalInterruptionDurationMs; }; } // namespace webrtc diff --git a/modules/audio_coding/neteq/include/neteq.h b/modules/audio_coding/neteq/include/neteq.h index c0c836f6fb..d91850fd77 100644 --- a/modules/audio_coding/neteq/include/neteq.h +++ b/modules/audio_coding/neteq/include/neteq.h @@ -90,8 +90,8 @@ struct NetEqLifetimeStatistics { // An interruption is a loss-concealment event lasting at least 150 ms. The // two stats below count the number os such events and the total duration of // these events. - uint64_t interruption_count = 0; - uint64_t total_interruption_duration_ms = 0; + int32_t interruption_count = 0; + int32_t total_interruption_duration_ms = 0; }; // Metrics that describe the operations performed in NetEq, and the internal diff --git a/modules/audio_coding/neteq/neteq_impl_unittest.cc b/modules/audio_coding/neteq/neteq_impl_unittest.cc index 025ed693d9..826952635f 100644 --- a/modules/audio_coding/neteq/neteq_impl_unittest.cc +++ b/modules/audio_coding/neteq/neteq_impl_unittest.cc @@ -745,7 +745,7 @@ TEST_F(NetEqImplTest, NoAudioInterruptionLoggedBeforeFirstDecode) { } auto lifetime_stats = neteq_->GetLifetimeStatistics(); - EXPECT_EQ(0u, lifetime_stats.interruption_count); + EXPECT_EQ(0, lifetime_stats.interruption_count); } // This test verifies that NetEq can handle comfort noise and enters/quits codec diff --git a/modules/audio_coding/neteq/statistics_calculator_unittest.cc b/modules/audio_coding/neteq/statistics_calculator_unittest.cc index a851074274..abfa3c5e1c 100644 --- a/modules/audio_coding/neteq/statistics_calculator_unittest.cc +++ b/modules/audio_coding/neteq/statistics_calculator_unittest.cc @@ -135,31 +135,31 @@ TEST(StatisticsCalculator, InterruptionCounter) { stats.DecodedOutputPlayed(); stats.EndExpandEvent(fs_hz); auto lts = stats.GetLifetimeStatistics(); - EXPECT_EQ(0u, lts.interruption_count); - EXPECT_EQ(0u, lts.total_interruption_duration_ms); + EXPECT_EQ(0, lts.interruption_count); + EXPECT_EQ(0, lts.total_interruption_duration_ms); // Add an event that is shorter than 150 ms. Should not be logged. stats.ExpandedVoiceSamples(10 * fs_khz, false); // 10 ms. stats.ExpandedNoiseSamples(139 * fs_khz, false); // 139 ms. stats.EndExpandEvent(fs_hz); lts = stats.GetLifetimeStatistics(); - EXPECT_EQ(0u, lts.interruption_count); + EXPECT_EQ(0, lts.interruption_count); // Add an event that is longer than 150 ms. Should be logged. stats.ExpandedVoiceSamples(140 * fs_khz, false); // 140 ms. stats.ExpandedNoiseSamples(11 * fs_khz, false); // 11 ms. stats.EndExpandEvent(fs_hz); lts = stats.GetLifetimeStatistics(); - EXPECT_EQ(1u, lts.interruption_count); - EXPECT_EQ(151u, lts.total_interruption_duration_ms); + EXPECT_EQ(1, lts.interruption_count); + EXPECT_EQ(151, lts.total_interruption_duration_ms); // Add one more long event. stats.ExpandedVoiceSamples(100 * fs_khz, false); // 100 ms. stats.ExpandedNoiseSamples(5000 * fs_khz, false); // 5000 ms. stats.EndExpandEvent(fs_hz); lts = stats.GetLifetimeStatistics(); - EXPECT_EQ(2u, lts.interruption_count); - EXPECT_EQ(5100u + 151u, lts.total_interruption_duration_ms); + EXPECT_EQ(2, lts.interruption_count); + EXPECT_EQ(5100 + 151, lts.total_interruption_duration_ms); } TEST(StatisticsCalculator, InterruptionCounterDoNotLogBeforeDecoding) { @@ -172,7 +172,7 @@ TEST(StatisticsCalculator, InterruptionCounterDoNotLogBeforeDecoding) { stats.ExpandedVoiceSamples(151 * fs_khz, false); // 151 ms. stats.EndExpandEvent(fs_hz); auto lts = stats.GetLifetimeStatistics(); - EXPECT_EQ(0u, lts.interruption_count); + EXPECT_EQ(0, lts.interruption_count); // Call DecodedOutputPlayed(). Logging should happen after this. stats.DecodedOutputPlayed(); @@ -181,7 +181,7 @@ TEST(StatisticsCalculator, InterruptionCounterDoNotLogBeforeDecoding) { stats.ExpandedVoiceSamples(151 * fs_khz, false); // 151 ms. stats.EndExpandEvent(fs_hz); lts = stats.GetLifetimeStatistics(); - EXPECT_EQ(1u, lts.interruption_count); + EXPECT_EQ(1, lts.interruption_count); } } // namespace webrtc diff --git a/modules/audio_coding/neteq/tools/neteq_stats_plotter.cc b/modules/audio_coding/neteq/tools/neteq_stats_plotter.cc index c7bec9c660..7c3aed8312 100644 --- a/modules/audio_coding/neteq/tools/neteq_stats_plotter.cc +++ b/modules/audio_coding/neteq/tools/neteq_stats_plotter.cc @@ -79,9 +79,8 @@ void NetEqStatsPlotter::SimulationEnded(int64_t simulation_time_ms) { const auto lifetime_stats_vector = stats_getter_->lifetime_stats(); if (!lifetime_stats_vector->empty()) { auto lifetime_stats = lifetime_stats_vector->back().second; - printf(" num_interruptions: %" PRId64 "\n", - lifetime_stats.interruption_count); - printf(" sum_interruption_length_ms: %" PRId64 " ms\n", + printf(" num_interruptions: %d\n", lifetime_stats.interruption_count); + printf(" sum_interruption_length_ms: %d ms\n", lifetime_stats.total_interruption_duration_ms); printf(" interruption ratio: %f%%\n", 100.0 * lifetime_stats.total_interruption_duration_ms / diff --git a/pc/rtc_stats_collector.cc b/pc/rtc_stats_collector.cc index 611dfe373e..8849d8614c 100644 --- a/pc/rtc_stats_collector.cc +++ b/pc/rtc_stats_collector.cc @@ -490,6 +490,13 @@ ProduceMediaStreamTrackStatsFromVoiceReceiverInfo( voice_receiver_info.delayed_packet_outage_samples; audio_track_stats->relative_packet_arrival_delay = voice_receiver_info.relative_packet_arrival_delay_seconds; + audio_track_stats->interruption_count = + voice_receiver_info.interruption_count >= 0 + ? voice_receiver_info.interruption_count + : 0; + audio_track_stats->total_interruption_duration = + static_cast(voice_receiver_info.total_interruption_duration_ms) / + rtc::kNumMillisecsPerSec; return audio_track_stats; } diff --git a/pc/rtc_stats_collector_unittest.cc b/pc/rtc_stats_collector_unittest.cc index ba80732ea2..0539379564 100644 --- a/pc/rtc_stats_collector_unittest.cc +++ b/pc/rtc_stats_collector_unittest.cc @@ -1429,6 +1429,8 @@ TEST_F(RTCStatsCollectorTest, voice_receiver_info.jitter_buffer_flushes = 7; voice_receiver_info.delayed_packet_outage_samples = 15; voice_receiver_info.relative_packet_arrival_delay_seconds = 16; + voice_receiver_info.interruption_count = 7788; + voice_receiver_info.total_interruption_duration_ms = 778899; stats_->CreateMockRtpSendersReceiversAndChannels( {}, {std::make_pair(remote_audio_track.get(), voice_receiver_info)}, {}, @@ -1466,6 +1468,8 @@ TEST_F(RTCStatsCollectorTest, expected_remote_audio_track.jitter_buffer_flushes = 7; expected_remote_audio_track.delayed_packet_outage_samples = 15; expected_remote_audio_track.relative_packet_arrival_delay = 16; + expected_remote_audio_track.interruption_count = 7788; + expected_remote_audio_track.total_interruption_duration = 778.899; ASSERT_TRUE(report->Get(expected_remote_audio_track.id())); EXPECT_EQ(expected_remote_audio_track, report->Get(expected_remote_audio_track.id()) diff --git a/pc/rtc_stats_integrationtest.cc b/pc/rtc_stats_integrationtest.cc index bb13c20bf2..d576c6082a 100644 --- a/pc/rtc_stats_integrationtest.cc +++ b/pc/rtc_stats_integrationtest.cc @@ -654,6 +654,10 @@ class RTCStatsReportVerifier { media_stream_track.delayed_packet_outage_samples); verifier.TestMemberIsNonNegative( media_stream_track.relative_packet_arrival_delay); + verifier.TestMemberIsNonNegative( + media_stream_track.interruption_count); + verifier.TestMemberIsNonNegative( + media_stream_track.total_interruption_duration); } else { verifier.TestMemberIsUndefined(media_stream_track.jitter_buffer_delay); verifier.TestMemberIsUndefined( @@ -666,6 +670,9 @@ class RTCStatsReportVerifier { media_stream_track.delayed_packet_outage_samples); verifier.TestMemberIsUndefined( media_stream_track.relative_packet_arrival_delay); + verifier.TestMemberIsUndefined(media_stream_track.interruption_count); + verifier.TestMemberIsUndefined( + media_stream_track.total_interruption_duration); } return verifier.ExpectAllMembersSuccessfullyTested(); } diff --git a/stats/rtcstats_objects.cc b/stats/rtcstats_objects.cc index 89d5a85d7a..6c914e7768 100644 --- a/stats/rtcstats_objects.cc +++ b/stats/rtcstats_objects.cc @@ -389,6 +389,8 @@ WEBRTC_RTCSTATS_IMPL(RTCMediaStreamTrackStats, RTCStats, "track", &jitter_buffer_flushes, &delayed_packet_outage_samples, &relative_packet_arrival_delay, + &interruption_count, + &total_interruption_duration, &freeze_count, &pause_count, &total_freezes_duration, @@ -442,6 +444,8 @@ RTCMediaStreamTrackStats::RTCMediaStreamTrackStats(std::string&& id, relative_packet_arrival_delay( "relativePacketArrivalDelay", {NonStandardGroupId::kRtcStatsRelativePacketArrivalDelay}), + interruption_count("interruptionCount"), + total_interruption_duration("totalInterruptionDuration"), freeze_count("freezeCount"), pause_count("pauseCount"), total_freezes_duration("totalFreezesDuration"), @@ -484,6 +488,8 @@ RTCMediaStreamTrackStats::RTCMediaStreamTrackStats( jitter_buffer_flushes(other.jitter_buffer_flushes), delayed_packet_outage_samples(other.delayed_packet_outage_samples), relative_packet_arrival_delay(other.relative_packet_arrival_delay), + interruption_count(other.interruption_count), + total_interruption_duration(other.total_interruption_duration), freeze_count(other.freeze_count), pause_count(other.pause_count), total_freezes_duration(other.total_freezes_duration),