zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add histogram stats for AV sync stream offset:

Browse Source 
"WebRTC.Video.AVSyncOffsetInMs"

The absolute value of the sync offset between a rendered video frame and the latest played audio frame is measured per video frame. The average offset per received video stream is recorded when a stream is removed.

Updated sync tests in call_perf_tests.cc to use this implementation.

BUG=webrtc:5493

Review URL: https://codereview.webrtc.org/1756193005

Cr-Commit-Position: refs/heads/master@{#11993}
This commit is contained in:
asapersson
2016-03-15 01:00:47 -07:00
committed by Commit bot
parent a1cf366ea9
commit f8cdd184d5
11 changed files with 111 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

135
webrtc/call/call_perf_tests.cc
View File

@ -7,7 +7,9 @@
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <algorithm>
#include <limits>
#include <memory>
#include <sstream>
#include <string>
@ -33,6 +35,7 @@
#include "webrtc/test/fake_encoder.h"
#include "webrtc/test/frame_generator.h"
#include "webrtc/test/frame_generator_capturer.h"
#include "webrtc/test/histogram.h"
#include "webrtc/test/rtp_rtcp_observer.h"
#include "webrtc/test/testsupport/fileutils.h"
#include "webrtc/test/testsupport/perf_test.h"
@ -71,100 +74,35 @@ class CallPerfTest : public test::CallTest {
int run_time_ms);
};
class SyncRtcpObserver : public test::RtpRtcpObserver {
public:
SyncRtcpObserver() : test::RtpRtcpObserver(CallPerfTest::kLongTimeoutMs) {}
Action OnSendRtcp(const uint8_t* packet, size_t length) override {
RTCPUtility::RTCPParserV2 parser(packet, length, true);
EXPECT_TRUE(parser.IsValid());
for (RTCPUtility::RTCPPacketTypes packet_type = parser.Begin();
packet_type != RTCPUtility::RTCPPacketTypes::kInvalid;
packet_type = parser.Iterate()) {
if (packet_type == RTCPUtility::RTCPPacketTypes::kSr) {
const RTCPUtility::RTCPPacket& packet = parser.Packet();
RtcpMeasurement ntp_rtp_pair(
packet.SR.NTPMostSignificant,
packet.SR.NTPLeastSignificant,
packet.SR.RTPTimestamp);
StoreNtpRtpPair(ntp_rtp_pair);
}
}
return SEND_PACKET;
}
int64_t RtpTimestampToNtp(uint32_t timestamp) const {
rtc::CritScope lock(&crit_);
int64_t timestamp_in_ms = -1;
if (ntp_rtp_pairs_.size() == 2) {
// TODO(stefan): We can't EXPECT_TRUE on this call due to a bug in the
// RTCP sender where it sends RTCP SR before any RTP packets, which leads
// to a bogus NTP/RTP mapping.
RtpToNtpMs(timestamp, ntp_rtp_pairs_, &timestamp_in_ms);
return timestamp_in_ms;
}
return -1;
}
private:
void StoreNtpRtpPair(RtcpMeasurement ntp_rtp_pair) {
rtc::CritScope lock(&crit_);
for (RtcpList::iterator it = ntp_rtp_pairs_.begin();
it != ntp_rtp_pairs_.end();
++it) {
if (ntp_rtp_pair.ntp_secs == it->ntp_secs &&
ntp_rtp_pair.ntp_frac == it->ntp_frac) {
// This RTCP has already been added to the list.
return;
}
}
// We need two RTCP SR reports to map between RTP and NTP. More than two
// will not improve the mapping.
if (ntp_rtp_pairs_.size() == 2) {
ntp_rtp_pairs_.pop_back();
}
ntp_rtp_pairs_.push_front(ntp_rtp_pair);
}
rtc::CriticalSection crit_;
RtcpList ntp_rtp_pairs_ GUARDED_BY(crit_);
};
class VideoRtcpAndSyncObserver : public SyncRtcpObserver, public VideoRenderer {
class VideoRtcpAndSyncObserver : public test::RtpRtcpObserver,
public VideoRenderer {
static const int kInSyncThresholdMs = 50;
static const int kStartupTimeMs = 2000;
static const int kMinRunTimeMs = 30000;
public:
VideoRtcpAndSyncObserver(Clock* clock,
int voe_channel,
VoEVideoSync* voe_sync,
SyncRtcpObserver* audio_observer)
: clock_(clock),
voe_channel_(voe_channel),
voe_sync_(voe_sync),
audio_observer_(audio_observer),
explicit VideoRtcpAndSyncObserver(Clock* clock)
: test::RtpRtcpObserver(CallPerfTest::kLongTimeoutMs),
clock_(clock),
creation_time_ms_(clock_->TimeInMilliseconds()),
first_time_in_sync_(-1) {}
first_time_in_sync_(-1),
receive_stream_(nullptr) {}
void RenderFrame(const VideoFrame& video_frame,
int time_to_render_ms) override {
VideoReceiveStream::Stats stats;
{
rtc::CritScope lock(&crit_);
if (receive_stream_)
stats = receive_stream_->GetStats();
}
if (stats.sync_offset_ms == std::numeric_limits<int>::max())
return;
int64_t now_ms = clock_->TimeInMilliseconds();
uint32_t playout_timestamp = 0;
if (voe_sync_->GetPlayoutTimestamp(voe_channel_, playout_timestamp) != 0)
return;
int64_t latest_audio_ntp =
audio_observer_->RtpTimestampToNtp(playout_timestamp);
int64_t latest_video_ntp = RtpTimestampToNtp(video_frame.timestamp());
if (latest_audio_ntp < 0 || latest_video_ntp < 0)
return;
int time_until_render_ms =
std::max(0, static_cast<int>(video_frame.render_time_ms() - now_ms));
latest_video_ntp += time_until_render_ms;
int64_t stream_offset = latest_audio_ntp - latest_video_ntp;
std::stringstream ss;
ss << stream_offset;
ss << stats.sync_offset_ms;
webrtc::test::PrintResult("stream_offset",
"",
"synchronization",
@ -176,7 +114,7 @@ class VideoRtcpAndSyncObserver : public SyncRtcpObserver, public VideoRenderer {
// estimated as being synchronized. We don't want to trigger on those.
if (time_since_creation < kStartupTimeMs)
return;
if (std::abs(latest_audio_ntp - latest_video_ntp) < kInSyncThresholdMs) {
if (std::abs(stats.sync_offset_ms) < kInSyncThresholdMs) {
if (first_time_in_sync_ == -1) {
first_time_in_sync_ = now_ms;
webrtc::test::PrintResult("sync_convergence_time",
@ -193,13 +131,17 @@ class VideoRtcpAndSyncObserver : public SyncRtcpObserver, public VideoRenderer {
bool IsTextureSupported() const override { return false; }
void set_receive_stream(VideoReceiveStream* receive_stream) {
rtc::CritScope lock(&crit_);
receive_stream_ = receive_stream;
}
private:
Clock* const clock_;
const int voe_channel_;
VoEVideoSync* const voe_sync_;
SyncRtcpObserver* const audio_observer_;
const int64_t creation_time_ms_;
int64_t first_time_in_sync_;
rtc::CriticalSection crit_;
VideoReceiveStream* receive_stream_ GUARDED_BY(crit_);
};
void CallPerfTest::TestAudioVideoSync(FecMode fec,
@ -238,11 +180,11 @@ void CallPerfTest::TestAudioVideoSync(FecMode fec,
std::unique_ptr<RtpHeaderParser> parser_;
};
test::ClearHistograms();
VoiceEngine* voice_engine = VoiceEngine::Create();
VoEBase* voe_base = VoEBase::GetInterface(voice_engine);
VoECodec* voe_codec = VoECodec::GetInterface(voice_engine);
VoENetwork* voe_network = VoENetwork::GetInterface(voice_engine);
VoEVideoSync* voe_sync = VoEVideoSync::GetInterface(voice_engine);
const std::string audio_filename =
test::ResourcePath("voice_engine/audio_long16", "pcm");
ASSERT_STRNE("", audio_filename.c_str());
@ -254,8 +196,6 @@ void CallPerfTest::TestAudioVideoSync(FecMode fec,
int send_channel_id = voe_base->CreateChannel(voe_config);
int recv_channel_id = voe_base->CreateChannel();
SyncRtcpObserver audio_observer;
AudioState::Config send_audio_state_config;
send_audio_state_config.voice_engine = voice_engine;
Call::Config sender_config;
@ -267,14 +207,16 @@ void CallPerfTest::TestAudioVideoSync(FecMode fec,
AudioPacketReceiver voe_send_packet_receiver(send_channel_id, voe_network);
AudioPacketReceiver voe_recv_packet_receiver(recv_channel_id, voe_network);
VideoRtcpAndSyncObserver observer(Clock::GetRealTimeClock());
FakeNetworkPipe::Config net_config;
net_config.queue_delay_ms = 500;
net_config.loss_percent = 5;
test::PacketTransport audio_send_transport(
nullptr, &audio_observer, test::PacketTransport::kSender, net_config);
nullptr, &observer, test::PacketTransport::kSender, net_config);
audio_send_transport.SetReceiver(&voe_recv_packet_receiver);
test::PacketTransport audio_receive_transport(
nullptr, &audio_observer, test::PacketTransport::kReceiver, net_config);
nullptr, &observer, test::PacketTransport::kReceiver, net_config);
audio_receive_transport.SetReceiver(&voe_send_packet_receiver);
internal::TransportAdapter send_transport_adapter(&audio_send_transport);
@ -287,9 +229,6 @@ void CallPerfTest::TestAudioVideoSync(FecMode fec,
EXPECT_EQ(0, voe_network->RegisterExternalTransport(recv_channel_id,
recv_transport_adapter));
VideoRtcpAndSyncObserver observer(Clock::GetRealTimeClock(), recv_channel_id,
voe_sync, &audio_observer);
test::PacketTransport sync_send_transport(sender_call_.get(), &observer,
test::PacketTransport::kSender,
FakeNetworkPipe::Config());
@ -341,7 +280,8 @@ void CallPerfTest::TestAudioVideoSync(FecMode fec,
audio_receive_stream =
receiver_call_->CreateAudioReceiveStream(audio_recv_config);
}
EXPECT_EQ(1u, video_receive_streams_.size());
observer.set_receive_stream(video_receive_streams_[0]);
DriftingClock drifting_clock(clock_, video_ntp_speed);
CreateFrameGeneratorCapturerWithDrift(&drifting_clock, video_rtp_speed);
@ -376,11 +316,12 @@ void CallPerfTest::TestAudioVideoSync(FecMode fec,
voe_base->Release();
voe_codec->Release();
voe_network->Release();
voe_sync->Release();
DestroyCalls();
VoiceEngine::Delete(voice_engine);
EXPECT_EQ(1, test::NumHistogramSamples("WebRTC.Video.AVSyncOffsetInMs"));
}
TEST_F(CallPerfTest, PlaysOutAudioAndVideoInSyncWithVideoNtpDrift) {

4
webrtc/modules/rtp_rtcp/source/remote_ntp_time_estimator.cc
View File

@ -51,10 +51,6 @@ bool RemoteNtpTimeEstimator::UpdateRtcpTimestamp(int64_t rtt,
}
int64_t RemoteNtpTimeEstimator::Estimate(uint32_t rtp_timestamp) {
if (rtcp_list_.size() < 2) {
// We need two RTCP SR reports to calculate NTP.
return -1;
}
int64_t sender_capture_ntp_ms = 0;
if (!RtpToNtpMs(rtp_timestamp, rtcp_list_, &sender_capture_ntp_ms)) {
return -1;

4
webrtc/system_wrappers/source/rtp_to_ntp.cc
View File

@ -95,7 +95,9 @@ bool UpdateRtcpList(uint32_t ntp_secs,
bool RtpToNtpMs(int64_t rtp_timestamp,
const RtcpList& rtcp,
int64_t* rtp_timestamp_in_ms) {
assert(rtcp.size() == 2);
if (rtcp.size() != 2)
return false;
int64_t rtcp_ntp_ms_new = Clock::NtpToMs(rtcp.front().ntp_secs,
rtcp.front().ntp_frac);
int64_t rtcp_ntp_ms_old = Clock::NtpToMs(rtcp.back().ntp_secs,

16
webrtc/system_wrappers/source/rtp_to_ntp_unittest.cc
View File

@ -12,6 +12,10 @@
#include "webrtc/system_wrappers/include/rtp_to_ntp.h"
namespace webrtc {
namespace {
const uint32_t kOneMsInNtpFrac = 4294967;
const uint32_t kTimestampTicksPerMs = 90;
} // namespace
TEST(WrapAroundTests, NoWrap) {
EXPECT_EQ(0, CheckForWrapArounds(0xFFFFFFFF, 0xFFFFFFFE));
@ -38,8 +42,6 @@ TEST(WrapAroundTests, OldRtcpWrapped) {
uint32_t ntp_sec = 0;
uint32_t ntp_frac = 0;
uint32_t timestamp = 0;
const uint32_t kOneMsInNtpFrac = 4294967;
const uint32_t kTimestampTicksPerMs = 90;
rtcp.push_front(RtcpMeasurement(ntp_sec, ntp_frac, timestamp));
ntp_frac += kOneMsInNtpFrac;
timestamp -= kTimestampTicksPerMs;
@ -57,8 +59,6 @@ TEST(WrapAroundTests, NewRtcpWrapped) {
uint32_t ntp_sec = 0;
uint32_t ntp_frac = 0;
uint32_t timestamp = 0xFFFFFFFF;
const uint32_t kOneMsInNtpFrac = 4294967;
const uint32_t kTimestampTicksPerMs = 90;
rtcp.push_front(RtcpMeasurement(ntp_sec, ntp_frac, timestamp));
ntp_frac += kOneMsInNtpFrac;
timestamp += kTimestampTicksPerMs;
@ -71,8 +71,6 @@ TEST(WrapAroundTests, NewRtcpWrapped) {
}
TEST(WrapAroundTests, RtpWrapped) {
const uint32_t kOneMsInNtpFrac = 4294967;
const uint32_t kTimestampTicksPerMs = 90;
RtcpList rtcp;
uint32_t ntp_sec = 0;
uint32_t ntp_frac = 0;
@ -91,8 +89,6 @@ TEST(WrapAroundTests, RtpWrapped) {
}
TEST(WrapAroundTests, OldRtp_RtcpsWrapped) {
const uint32_t kOneMsInNtpFrac = 4294967;
const uint32_t kTimestampTicksPerMs = 90;
RtcpList rtcp;
uint32_t ntp_sec = 0;
uint32_t ntp_frac = 0;
@ -108,8 +104,6 @@ TEST(WrapAroundTests, OldRtp_RtcpsWrapped) {
}
TEST(WrapAroundTests, OldRtp_NewRtcpWrapped) {
const uint32_t kOneMsInNtpFrac = 4294967;
const uint32_t kTimestampTicksPerMs = 90;
RtcpList rtcp;
uint32_t ntp_sec = 0;
uint32_t ntp_frac = 0;
@ -128,8 +122,6 @@ TEST(WrapAroundTests, OldRtp_NewRtcpWrapped) {
}
TEST(WrapAroundTests, OldRtp_OldRtcpWrapped) {
const uint32_t kOneMsInNtpFrac = 4294967;
const uint32_t kTimestampTicksPerMs = 90;
RtcpList rtcp;
uint32_t ntp_sec = 0;
uint32_t ntp_frac = 0;

10
webrtc/video/receive_statistics_proxy.cc
View File

@ -59,6 +59,10 @@ void ReceiveStatisticsProxy::UpdateHistograms() {
RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.ReceivedWidthInPixels", width);
RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.ReceivedHeightInPixels", height);
}
int sync_offset_ms = sync_offset_counter_.Avg(kMinRequiredSamples);
if (sync_offset_ms != -1)
RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.AVSyncOffsetInMs", sync_offset_ms);
int qp = qp_counters_.vp8.Avg(kMinRequiredSamples);
if (qp != -1)
RTC_HISTOGRAM_COUNTS_200("WebRTC.Video.Decoded.Vp8.Qp", qp);
@ -239,6 +243,12 @@ void ReceiveStatisticsProxy::OnRenderedFrame(const VideoFrame& frame) {
}
}
void ReceiveStatisticsProxy::OnSyncOffsetUpdated(int64_t sync_offset_ms) {
rtc::CritScope lock(&crit_);
sync_offset_counter_.Add(std::abs(sync_offset_ms));
stats_.sync_offset_ms = sync_offset_ms;
}
void ReceiveStatisticsProxy::OnReceiveRatesUpdated(uint32_t bitRate,
uint32_t frameRate) {
}

2
webrtc/video/receive_statistics_proxy.h
View File

@ -46,6 +46,7 @@ class ReceiveStatisticsProxy : public VCMReceiveStatisticsCallback,
void OnDecodedFrame();
void OnRenderedFrame(const VideoFrame& frame);
void OnSyncOffsetUpdated(int64_t sync_offset_ms);
void OnIncomingPayloadType(int payload_type);
void OnDecoderImplementationName(const char* implementation_name);
void OnIncomingRate(unsigned int framerate, unsigned int bitrate_bps);
@ -106,6 +107,7 @@ class ReceiveStatisticsProxy : public VCMReceiveStatisticsCallback,
rtc::RateTracker render_pixel_tracker_ GUARDED_BY(crit_);
SampleCounter render_width_counter_ GUARDED_BY(crit_);
SampleCounter render_height_counter_ GUARDED_BY(crit_);
SampleCounter sync_offset_counter_ GUARDED_BY(crit_);
SampleCounter decode_time_counter_ GUARDED_BY(crit_);
SampleCounter delay_counter_ GUARDED_BY(crit_);
ReportBlockStats report_block_stats_ GUARDED_BY(crit_);

4
webrtc/video/stream_synchronization.cc
View File

@ -60,10 +60,6 @@ bool StreamSynchronization::ComputeRelativeDelay(
const Measurements& video_measurement,
int* relative_delay_ms) {
assert(relative_delay_ms);
if (audio_measurement.rtcp.size() < 2 || video_measurement.rtcp.size() < 2) {
// We need two RTCP SR reports per stream to do synchronization.
return false;
}
int64_t audio_last_capture_time_ms;
if (!RtpToNtpMs(audio_measurement.latest_timestamp,
audio_measurement.rtcp,

4
webrtc/video/video_receive_stream.cc
View File

@ -384,6 +384,10 @@ void VideoReceiveStream::FrameCallback(VideoFrame* video_frame) {
int VideoReceiveStream::RenderFrame(const uint32_t /*stream_id*/,
const VideoFrame& video_frame) {
int64_t sync_offset_ms;
if (vie_sync_.GetStreamSyncOffsetInMs(video_frame, &sync_offset_ms))
stats_proxy_.OnSyncOffsetUpdated(sync_offset_ms);
// TODO(pbos): Wire up config_.render->IsTextureSupported() and convert if not
// supported. Or provide methods for converting a texture frame in
// VideoFrame.

42
webrtc/video/vie_sync_module.cc
View File

@ -16,11 +16,13 @@
#include "webrtc/modules/rtp_rtcp/include/rtp_receiver.h"
#include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h"
#include "webrtc/modules/video_coding/include/video_coding.h"
#include "webrtc/system_wrappers/include/clock.h"
#include "webrtc/video/stream_synchronization.h"
#include "webrtc/video_frame.h"
#include "webrtc/voice_engine/include/voe_video_sync.h"
namespace webrtc {
namespace {
int UpdateMeasurements(StreamSynchronization::Measurements* stream,
const RtpRtcp& rtp_rtcp, const RtpReceiver& receiver) {
if (!receiver.Timestamp(&stream->latest_timestamp))
@ -47,16 +49,17 @@ int UpdateMeasurements(StreamSynchronization::Measurements* stream,
return 0;
}
} // namespace
ViESyncModule::ViESyncModule(VideoCodingModule* vcm)
: vcm_(vcm),
clock_(Clock::GetRealTimeClock()),
video_receiver_(NULL),
video_rtp_rtcp_(NULL),
voe_channel_id_(-1),
voe_sync_interface_(NULL),
last_sync_time_(TickTime::Now()),
sync_() {
}
sync_() {}
ViESyncModule::~ViESyncModule() {
}
@ -157,4 +160,37 @@ void ViESyncModule::Process() {
vcm_->SetMinimumPlayoutDelay(target_video_delay_ms);
}
bool ViESyncModule::GetStreamSyncOffsetInMs(const VideoFrame& frame,
int64_t* stream_offset_ms) const {
rtc::CritScope lock(&data_cs_);
if (voe_channel_id_ == -1)
return false;
uint32_t playout_timestamp = 0;
if (voe_sync_interface_->GetPlayoutTimestamp(voe_channel_id_,
playout_timestamp) != 0) {
return false;
}
int64_t latest_audio_ntp;
if (!RtpToNtpMs(playout_timestamp, audio_measurement_.rtcp,
&latest_audio_ntp)) {
return false;
}
int64_t latest_video_ntp;
if (!RtpToNtpMs(frame.timestamp(), video_measurement_.rtcp,
&latest_video_ntp)) {
return false;
}
int64_t time_to_render_ms =
frame.render_time_ms() - clock_->TimeInMilliseconds();
if (time_to_render_ms > 0)
latest_video_ntp += time_to_render_ms;
*stream_offset_ms = latest_audio_ntp - latest_video_ntp;
return true;
}
} // namespace webrtc

8
webrtc/video/vie_sync_module.h
View File

@ -24,8 +24,10 @@
namespace webrtc {
class Clock;
class RtpRtcp;
class VideoCodingModule;
class VideoFrame;
class ViEChannel;
class VoEVideoSync;
@ -43,9 +45,15 @@ class ViESyncModule : public Module {
int64_t TimeUntilNextProcess() override;
void Process() override;
// Gets the sync offset between the current played out audio frame and the
// video |frame|. Returns true on success, false otherwise.
bool GetStreamSyncOffsetInMs(const VideoFrame& frame,
int64_t* stream_offset_ms) const;
private:
rtc::CriticalSection data_cs_;
VideoCodingModule* const vcm_;
Clock* const clock_;
RtpReceiver* video_receiver_;
RtpRtcp* video_rtp_rtcp_;
int voe_channel_id_;

3
webrtc/video_receive_stream.h
View File

@ -11,6 +11,7 @@
#ifndef WEBRTC_VIDEO_RECEIVE_STREAM_H_
#define WEBRTC_VIDEO_RECEIVE_STREAM_H_
#include <limits>
#include <map>
#include <string>
#include <vector>
@ -66,6 +67,8 @@ class VideoReceiveStream : public ReceiveStream {
int total_bitrate_bps = 0;
int discarded_packets = 0;
int sync_offset_ms = std::numeric_limits<int>::max();
uint32_t ssrc = 0;
std::string c_name;
StreamDataCounters rtp_stats;