From 9a2e906b0c06ee1a9b75c734bf58326469db8e93 Mon Sep 17 00:00:00 2001 From: Gustaf Ullberg Date: Mon, 18 Sep 2017 09:28:20 +0200 Subject: [PATCH] Added RTCMediaStreamTrackStats.concealmentEvents MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The number of concealment events. This counter increases every time a concealed sample is synthesized after a non-concealed sample. That is, multiple consecutive concealed samples will increase the concealedSamples count multiple times but is a single concealment event. Bug: webrtc:8246 Change-Id: I7ef404edab765218b1f11e3128072c5391e83049 Reviewed-on: https://webrtc-review.googlesource.com/1221 Commit-Queue: Gustaf Ullberg Reviewed-by: Henrik Andreassson Reviewed-by: Fredrik Solenberg Reviewed-by: Henrik Lundin Reviewed-by: Taylor Brandstetter Reviewed-by: Henrik Boström Cr-Commit-Position: refs/heads/master@{#19881} --- api/stats/rtcstats_objects.h | 1 + api/statstypes.cc | 2 ++ api/statstypes.h | 1 + audio/audio_receive_stream.cc | 1 + audio/audio_receive_stream_unittest.cc | 7 ++-- call/audio_receive_stream.h | 12 +++---- common_types.h | 4 +++ media/base/mediachannel.h | 13 +++---- media/engine/webrtcvoiceengine.cc | 1 + media/engine/webrtcvoiceengine_unittest.cc | 2 ++ modules/audio_coding/acm2/acm_receiver.cc | 1 + modules/audio_coding/neteq/include/neteq.h | 8 ++--- modules/audio_coding/neteq/neteq_impl.cc | 7 ++-- modules/audio_coding/neteq/neteq_unittest.cc | 34 +++++++++++++++++++ .../neteq/statistics_calculator.cc | 8 +++-- .../neteq/statistics_calculator.h | 4 +-- pc/rtcstats_integrationtest.cc | 7 ++-- pc/rtcstatscollector.cc | 2 ++ pc/rtcstatscollector_unittest.cc | 2 ++ stats/rtcstats_objects.cc | 9 +++-- 20 files changed, 90 insertions(+), 36 deletions(-) diff --git a/api/stats/rtcstats_objects.h b/api/stats/rtcstats_objects.h index a600fb995c..903d266a15 100644 --- a/api/stats/rtcstats_objects.h +++ b/api/stats/rtcstats_objects.h @@ -278,6 +278,7 @@ class RTCMediaStreamTrackStats final : public RTCStats { RTCStatsMember total_samples_received; RTCStatsMember total_samples_duration; RTCStatsMember concealed_samples; + RTCStatsMember concealment_events; }; // https://w3c.github.io/webrtc-stats/#pcstats-dict* diff --git a/api/statstypes.cc b/api/statstypes.cc index 29b3e5e432..37e8aac585 100644 --- a/api/statstypes.cc +++ b/api/statstypes.cc @@ -373,6 +373,8 @@ const char* StatsReport::Value::display_name() const { return "bytesSent"; case kStatsValueNameConcealedSamples: return "concealedSamples"; + case kStatsValueNameConcealmentEvents: + return "concealmentEvents"; case kStatsValueNamePacketsSent: return "packetsSent"; case kStatsValueNameBytesReceived: diff --git a/api/statstypes.h b/api/statstypes.h index 431a1ce6bc..7f69b028db 100644 --- a/api/statstypes.h +++ b/api/statstypes.h @@ -105,6 +105,7 @@ class StatsReport { kStatsValueNameBytesSent, kStatsValueNameCodecImplementationName, kStatsValueNameConcealedSamples, + kStatsValueNameConcealmentEvents, kStatsValueNameDataChannelId, kStatsValueNameFramesDecoded, kStatsValueNameFramesEncoded, diff --git a/audio/audio_receive_stream.cc b/audio/audio_receive_stream.cc index 0c83319ad7..8ff1d589a9 100644 --- a/audio/audio_receive_stream.cc +++ b/audio/audio_receive_stream.cc @@ -196,6 +196,7 @@ webrtc::AudioReceiveStream::Stats AudioReceiveStream::GetStats() const { stats.jitter_buffer_preferred_ms = ns.preferredBufferSize; stats.total_samples_received = ns.totalSamplesReceived; stats.concealed_samples = ns.concealedSamples; + stats.concealment_events = ns.concealmentEvents; stats.expand_rate = Q14ToFloat(ns.currentExpandRate); stats.speech_expand_rate = Q14ToFloat(ns.currentSpeechExpandRate); stats.secondary_decoded_rate = Q14ToFloat(ns.currentSecondaryDecodedRate); diff --git a/audio/audio_receive_stream_unittest.cc b/audio/audio_receive_stream_unittest.cc index 7bcb85adb9..b79ae19a6c 100644 --- a/audio/audio_receive_stream_unittest.cc +++ b/audio/audio_receive_stream_unittest.cc @@ -64,9 +64,9 @@ const CallStatistics kCallStats = { 345, 678, 901, 234, -12, 3456, 7890, 567, 890, 123}; const CodecInst kCodecInst = { 123, "codec_name_recv", 96000, -187, 0, -103}; -const NetworkStatistics kNetworkStats = {123, 456, false, 789012, 3456, 0, {}, - 789, 12, 345, 678, 901, 0, -1, - -1, -1, -1, -1, 0}; +const NetworkStatistics kNetworkStats = {123, 456, false, 789012, 3456, 123, 0, + {}, 789, 12, 345, 678, 901, 0, + -1, -1, -1, -1, -1, 0}; const AudioDecodingCallStats kAudioDecodeStats = MakeAudioDecodeStatsForTest(); struct ConfigHelper { @@ -322,6 +322,7 @@ TEST(AudioReceiveStreamTest, GetStats) { EXPECT_EQ(kNetworkStats.totalSamplesReceived, stats.total_samples_received); EXPECT_EQ(kTotalOutputDuration, stats.total_output_duration); EXPECT_EQ(kNetworkStats.concealedSamples, stats.concealed_samples); + EXPECT_EQ(kNetworkStats.concealmentEvents, stats.concealment_events); EXPECT_EQ(Q14ToFloat(kNetworkStats.currentExpandRate), stats.expand_rate); EXPECT_EQ(Q14ToFloat(kNetworkStats.currentSpeechExpandRate), stats.speech_expand_rate); diff --git a/call/audio_receive_stream.h b/call/audio_receive_stream.h index 096f19611b..baf2b678fe 100644 --- a/call/audio_receive_stream.h +++ b/call/audio_receive_stream.h @@ -50,18 +50,14 @@ class AudioReceiveStream { uint32_t jitter_buffer_preferred_ms = 0; uint32_t delay_estimate_ms = 0; int32_t audio_level = -1; - // See description of "totalAudioEnergy" in the WebRTC stats spec: - // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-totalaudioenergy + // Stats below correspond to similarly-named fields in the WebRTC stats + // spec. https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats double total_output_energy = 0.0; - // See description of "totalSamplesReceived" in the WebRTC stats spec: - // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-totalsamplesreceived uint64_t total_samples_received = 0; - // See description of "totalSamplesDuration" in the WebRTC stats spec: - // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-totalsamplesduration double total_output_duration = 0.0; - // See description of "concealedSamples" in the WebRTC stats spec: - // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-concealedsamples uint64_t concealed_samples = 0; + uint64_t concealment_events = 0; + // Stats below DO NOT correspond directly to anything in the WebRTC stats float expand_rate = 0.0f; float speech_expand_rate = 0.0f; float secondary_decoded_rate = 0.0f; diff --git a/common_types.h b/common_types.h index 4bd9313285..69fc761140 100644 --- a/common_types.h +++ b/common_types.h @@ -375,6 +375,10 @@ struct NetworkStatistics { // conceal packet loss. // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-concealedsamples uint64_t concealedSamples; + // Number of times a concealed sample is synthesized after a non-concealed + // sample. + // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-concealmentevents + uint64_t concealmentEvents; // Loss rate (network + late); fraction between 0 and 1, scaled to Q14. uint16_t currentPacketLossRate; // Late loss rate; fraction between 0 and 1, scaled to Q14. diff --git a/media/base/mediachannel.h b/media/base/mediachannel.h index 631e695e88..103240ec46 100644 --- a/media/base/mediachannel.h +++ b/media/base/mediachannel.h @@ -657,6 +657,7 @@ struct VoiceReceiverInfo : public MediaReceiverInfo { total_samples_received(0), total_output_duration(0.0), concealed_samples(0), + concealment_events(0), expand_rate(0), speech_expand_rate(0), secondary_decoded_rate(0), @@ -678,18 +679,14 @@ struct VoiceReceiverInfo : public MediaReceiverInfo { int jitter_buffer_preferred_ms; int delay_estimate_ms; int audio_level; - // See description of "totalAudioEnergy" in the WebRTC stats spec: - // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-totalaudioenergy + // Stats below correspond to similarly-named fields in the WebRTC stats spec. + // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats double total_output_energy; - // See description of "totalSamplesReceived" in the WebRTC stats spec: - // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-totalsamplesreceived uint64_t total_samples_received; - // See description of "totalSamplesDuration" in the WebRTC stats spec: - // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-totalsamplesduration double total_output_duration; - // See description of "concealedSamples" in the WebRTC stats spec: - // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-concealedsamples uint64_t concealed_samples; + uint64_t concealment_events; + // Stats below DO NOT correspond directly to anything in the WebRTC stats // fraction of synthesized audio inserted through expansion. float expand_rate; // fraction of synthesized speech inserted through expansion. diff --git a/media/engine/webrtcvoiceengine.cc b/media/engine/webrtcvoiceengine.cc index 8fa5fe2654..00c7fd61fc 100644 --- a/media/engine/webrtcvoiceengine.cc +++ b/media/engine/webrtcvoiceengine.cc @@ -2284,6 +2284,7 @@ bool WebRtcVoiceMediaChannel::GetStats(VoiceMediaInfo* info) { rinfo.total_samples_received = stats.total_samples_received; rinfo.total_output_duration = stats.total_output_duration; rinfo.concealed_samples = stats.concealed_samples; + rinfo.concealment_events = stats.concealment_events; rinfo.expand_rate = stats.expand_rate; rinfo.speech_expand_rate = stats.speech_expand_rate; rinfo.secondary_decoded_rate = stats.secondary_decoded_rate; diff --git a/media/engine/webrtcvoiceengine_unittest.cc b/media/engine/webrtcvoiceengine_unittest.cc index 2c31253114..4e80788919 100644 --- a/media/engine/webrtcvoiceengine_unittest.cc +++ b/media/engine/webrtcvoiceengine_unittest.cc @@ -622,6 +622,7 @@ class WebRtcVoiceEngineTestFake : public testing::Test { stats.audio_level = 1234; stats.total_samples_received = 5678901; stats.concealed_samples = 234; + stats.concealment_events = 12; stats.expand_rate = 5.67f; stats.speech_expand_rate = 8.90f; stats.secondary_decoded_rate = 1.23f; @@ -661,6 +662,7 @@ class WebRtcVoiceEngineTestFake : public testing::Test { EXPECT_EQ(info.audio_level, stats.audio_level); EXPECT_EQ(info.total_samples_received, stats.total_samples_received); EXPECT_EQ(info.concealed_samples, stats.concealed_samples); + EXPECT_EQ(info.concealment_events, stats.concealment_events); EXPECT_EQ(info.expand_rate, stats.expand_rate); EXPECT_EQ(info.speech_expand_rate, stats.speech_expand_rate); EXPECT_EQ(info.secondary_decoded_rate, stats.secondary_decoded_rate); diff --git a/modules/audio_coding/acm2/acm_receiver.cc b/modules/audio_coding/acm2/acm_receiver.cc index 8be2471f2c..d999df027b 100644 --- a/modules/audio_coding/acm2/acm_receiver.cc +++ b/modules/audio_coding/acm2/acm_receiver.cc @@ -336,6 +336,7 @@ void AcmReceiver::GetNetworkStatistics(NetworkStatistics* acm_stat) { NetEqLifetimeStatistics neteq_lifetime_stat = neteq_->GetLifetimeStatistics(); acm_stat->totalSamplesReceived = neteq_lifetime_stat.total_samples_received; acm_stat->concealedSamples = neteq_lifetime_stat.concealed_samples; + acm_stat->concealmentEvents = neteq_lifetime_stat.concealment_events; } int AcmReceiver::DecoderByPayloadType(uint8_t payload_type, diff --git a/modules/audio_coding/neteq/include/neteq.h b/modules/audio_coding/neteq/include/neteq.h index 56f82042a0..b349f20455 100644 --- a/modules/audio_coding/neteq/include/neteq.h +++ b/modules/audio_coding/neteq/include/neteq.h @@ -61,13 +61,11 @@ struct NetEqNetworkStatistics { // NetEq statistics that persist over the lifetime of the class. // These metrics are never reset. struct NetEqLifetimeStatistics { - // Total number of audio samples received, including synthesized samples. - // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-totalsamplesreceived + // Stats below correspond to similarly-named fields in the WebRTC stats spec. + // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats uint64_t total_samples_received = 0; - // Total number of inbound audio samples that are based on synthesized data to - // conceal packet loss. - // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-concealedsamples uint64_t concealed_samples = 0; + uint64_t concealment_events = 0; }; enum NetEqPlayoutMode { diff --git a/modules/audio_coding/neteq/neteq_impl.cc b/modules/audio_coding/neteq/neteq_impl.cc index 3cdcf458af..cdf590baa3 100644 --- a/modules/audio_coding/neteq/neteq_impl.cc +++ b/modules/audio_coding/neteq/neteq_impl.cc @@ -847,7 +847,7 @@ int NetEqImpl::GetAudioInternal(AudioFrame* audio_frame, bool* muted) { : timestamp_scaler_->ToExternal(playout_timestamp_) - static_cast(audio_frame->samples_per_channel_); audio_frame->num_channels_ = sync_buffer_->Channels(); - stats_.ExpandedNoiseSamples(output_size_samples_); + stats_.ExpandedNoiseSamples(output_size_samples_, false); *muted = true; return 0; } @@ -1573,14 +1573,15 @@ int NetEqImpl::DoExpand(bool play_dtmf) { algorithm_buffer_->Clear(); int return_value = expand_->Process(algorithm_buffer_.get()); size_t length = algorithm_buffer_->Size(); + bool is_new_concealment_event = (last_mode_ != kModeExpand); // Update in-call and post-call statistics. if (expand_->MuteFactor(0) == 0) { // Expand operation generates only noise. - stats_.ExpandedNoiseSamples(length); + stats_.ExpandedNoiseSamples(length, is_new_concealment_event); } else { // Expand operation generates more than only noise. - stats_.ExpandedVoiceSamples(length); + stats_.ExpandedVoiceSamples(length, is_new_concealment_event); } last_mode_ = kModeExpand; diff --git a/modules/audio_coding/neteq/neteq_unittest.cc b/modules/audio_coding/neteq/neteq_unittest.cc index 657b8edf71..764a505611 100644 --- a/modules/audio_coding/neteq/neteq_unittest.cc +++ b/modules/audio_coding/neteq/neteq_unittest.cc @@ -1634,4 +1634,38 @@ TEST_F(NetEqDecodingTest, LastDecodedTimestampsTwoDecoded) { neteq_->LastDecodedTimestamps()); } +TEST_F(NetEqDecodingTest, TestConcealmentEvents) { + const int kNumConcealmentEvents = 19; + const size_t kSamples = 10 * 16; + const size_t kPayloadBytes = kSamples * 2; + int seq_no = 0; + RTPHeader rtp_info; + rtp_info.ssrc = 0x1234; // Just an arbitrary SSRC. + rtp_info.payloadType = 94; // PCM16b WB codec. + rtp_info.markerBit = 0; + const uint8_t payload[kPayloadBytes] = {0}; + bool muted; + + for (int i = 0; i < kNumConcealmentEvents; i++) { + // Insert some packets of 10 ms size. + for (int j = 0; j < 10; j++) { + rtp_info.sequenceNumber = seq_no++; + rtp_info.timestamp = rtp_info.sequenceNumber * kSamples; + neteq_->InsertPacket(rtp_info, payload, 0); + neteq_->GetAudio(&out_frame_, &muted); + } + + // Lose a number of packets. + int num_lost = 1 + i; + for (int j = 0; j < num_lost; j++) { + seq_no++; + neteq_->GetAudio(&out_frame_, &muted); + } + } + + // Check number of concealment events. + NetEqLifetimeStatistics stats = neteq_->GetLifetimeStatistics(); + EXPECT_EQ(kNumConcealmentEvents, static_cast(stats.concealment_events)); +} + } // namespace webrtc diff --git a/modules/audio_coding/neteq/statistics_calculator.cc b/modules/audio_coding/neteq/statistics_calculator.cc index 163cfff332..bbd6e249f2 100644 --- a/modules/audio_coding/neteq/statistics_calculator.cc +++ b/modules/audio_coding/neteq/statistics_calculator.cc @@ -151,14 +151,18 @@ void StatisticsCalculator::ResetMcu() { timestamps_since_last_report_ = 0; } -void StatisticsCalculator::ExpandedVoiceSamples(size_t num_samples) { +void StatisticsCalculator::ExpandedVoiceSamples(size_t num_samples, + bool is_new_concealment_event) { expanded_speech_samples_ += num_samples; lifetime_stats_.concealed_samples += num_samples; + lifetime_stats_.concealment_events += is_new_concealment_event; } -void StatisticsCalculator::ExpandedNoiseSamples(size_t num_samples) { +void StatisticsCalculator::ExpandedNoiseSamples(size_t num_samples, + bool is_new_concealment_event) { expanded_noise_samples_ += num_samples; lifetime_stats_.concealed_samples += num_samples; + lifetime_stats_.concealment_events += is_new_concealment_event; } void StatisticsCalculator::ExpandedVoiceSamplesCorrection(int num_samples) { diff --git a/modules/audio_coding/neteq/statistics_calculator.h b/modules/audio_coding/neteq/statistics_calculator.h index e3e274913d..cec02bbf2d 100644 --- a/modules/audio_coding/neteq/statistics_calculator.h +++ b/modules/audio_coding/neteq/statistics_calculator.h @@ -39,11 +39,11 @@ class StatisticsCalculator { // Reports that |num_samples| samples were produced through expansion, and // that the expansion produced other than just noise samples. - void ExpandedVoiceSamples(size_t num_samples); + void ExpandedVoiceSamples(size_t num_samples, bool is_new_concealment_event); // Reports that |num_samples| samples were produced through expansion, and // that the expansion produced only noise samples. - void ExpandedNoiseSamples(size_t num_samples); + void ExpandedNoiseSamples(size_t num_samples, bool is_new_concealment_event); // Corrects the statistics for number of samples produced through non-noise // expansion by adding |num_samples| (negative or positive) to the current diff --git a/pc/rtcstats_integrationtest.cc b/pc/rtcstats_integrationtest.cc index a498dc84db..62d316dcdb 100644 --- a/pc/rtcstats_integrationtest.cc +++ b/pc/rtcstats_integrationtest.cc @@ -560,17 +560,20 @@ class RTCStatsReportVerifier { verifier.MarkMemberTested( media_stream_track.echo_return_loss_enhancement, true); } - // totalSamplesReceived and concealedSamples are only present on inbound - // audio tracks. + // totalSamplesReceived, concealedSamples and concealmentEvents are only + // present on inbound audio tracks. if (*media_stream_track.kind == RTCMediaStreamTrackKind::kAudio && *media_stream_track.remote_source) { verifier.TestMemberIsNonNegative( media_stream_track.total_samples_received); verifier.TestMemberIsNonNegative( media_stream_track.concealed_samples); + verifier.TestMemberIsNonNegative( + media_stream_track.concealment_events); } else { verifier.TestMemberIsUndefined(media_stream_track.total_samples_received); verifier.TestMemberIsUndefined(media_stream_track.concealed_samples); + verifier.TestMemberIsUndefined(media_stream_track.concealment_events); } return verifier.ExpectAllMembersSuccessfullyTested(); } diff --git a/pc/rtcstatscollector.cc b/pc/rtcstatscollector.cc index 9a6b26c0a2..161d224bc5 100644 --- a/pc/rtcstatscollector.cc +++ b/pc/rtcstatscollector.cc @@ -417,6 +417,8 @@ ProduceMediaStreamTrackStatsFromVoiceReceiverInfo( audio_track_stats->total_samples_duration = voice_receiver_info.total_output_duration; audio_track_stats->concealed_samples = voice_receiver_info.concealed_samples; + audio_track_stats->concealment_events = + voice_receiver_info.concealment_events; return audio_track_stats; } diff --git a/pc/rtcstatscollector_unittest.cc b/pc/rtcstatscollector_unittest.cc index cb5a79f77a..14f669cc35 100644 --- a/pc/rtcstatscollector_unittest.cc +++ b/pc/rtcstatscollector_unittest.cc @@ -1555,6 +1555,7 @@ TEST_F(RTCStatsCollectorTest, voice_receiver_info.total_samples_received = 4567; voice_receiver_info.total_output_duration = 0.25; voice_receiver_info.concealed_samples = 123; + voice_receiver_info.concealment_events = 12; test_->CreateMockRtpSendersReceiversAndChannels( { std::make_pair(local_audio_track.get(), voice_sender_info_ssrc1), @@ -1631,6 +1632,7 @@ TEST_F(RTCStatsCollectorTest, expected_remote_audio_track.total_samples_received = 4567; expected_remote_audio_track.total_samples_duration = 0.25; expected_remote_audio_track.concealed_samples = 123; + expected_remote_audio_track.concealment_events = 12; ASSERT_TRUE(report->Get(expected_remote_audio_track.id())); EXPECT_EQ(expected_remote_audio_track, report->Get(expected_remote_audio_track.id())->cast_to< diff --git a/stats/rtcstats_objects.cc b/stats/rtcstats_objects.cc index f407c0600b..e643e121e6 100644 --- a/stats/rtcstats_objects.cc +++ b/stats/rtcstats_objects.cc @@ -383,7 +383,8 @@ WEBRTC_RTCSTATS_IMPL(RTCMediaStreamTrackStats, RTCStats, "track", &echo_return_loss_enhancement, &total_samples_received, &total_samples_duration, - &concealed_samples); + &concealed_samples, + &concealment_events); // clang-format on RTCMediaStreamTrackStats::RTCMediaStreamTrackStats( @@ -416,7 +417,8 @@ RTCMediaStreamTrackStats::RTCMediaStreamTrackStats(std::string&& id, echo_return_loss_enhancement("echoReturnLossEnhancement"), total_samples_received("totalSamplesReceived"), total_samples_duration("totalSamplesDuration"), - concealed_samples("concealedSamples") { + concealed_samples("concealedSamples"), + concealment_events("concealmentEvents") { RTC_DCHECK(kind == RTCMediaStreamTrackKind::kAudio || kind == RTCMediaStreamTrackKind::kVideo); } @@ -445,7 +447,8 @@ RTCMediaStreamTrackStats::RTCMediaStreamTrackStats( echo_return_loss_enhancement(other.echo_return_loss_enhancement), total_samples_received(other.total_samples_received), total_samples_duration(other.total_samples_duration), - concealed_samples(other.concealed_samples) {} + concealed_samples(other.concealed_samples), + concealment_events(other.concealment_events) {} RTCMediaStreamTrackStats::~RTCMediaStreamTrackStats() { }