From 791eec7424b5614162fff3e4434cd38d65a97dcd Mon Sep 17 00:00:00 2001 From: "stefan@webrtc.org" Date: Tue, 11 Oct 2011 07:53:43 +0000 Subject: [PATCH] Add API to get the number of packets discarded by the video jitter buffer due to being too late. BUG= TEST= Review URL: http://webrtc-codereview.appspot.com/200001 git-svn-id: http://webrtc.googlecode.com/svn/trunk@723 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../main/interface/video_coding.h | 6 + .../video_coding/main/source/jitter_buffer.cc | 11 +- .../video_coding/main/source/jitter_buffer.h | 6 + .../video_coding/main/source/receiver.cc | 4 + .../video_coding/main/source/receiver.h | 1 + .../main/source/video_coding_impl.cc | 525 ++++++++++++------ .../main/source/video_coding_impl.h | 90 +-- .../main/test/jitter_buffer_test.cc | 33 +- src/video_engine/main/interface/vie_codec.h | 4 + src/video_engine/main/source/vie_channel.cc | 8 + src/video_engine/main/source/vie_channel.h | 2 + .../main/source/vie_codec_impl.cc | 19 + src/video_engine/main/source/vie_codec_impl.h | 2 + 13 files changed, 506 insertions(+), 205 deletions(-) diff --git a/src/modules/video_coding/main/interface/video_coding.h b/src/modules/video_coding/main/interface/video_coding.h index 5b0cc18126..6e139ca049 100644 --- a/src/modules/video_coding/main/interface/video_coding.h +++ b/src/modules/video_coding/main/interface/video_coding.h @@ -490,6 +490,12 @@ public: // Return value : VCM_OK, on success. // <0, on error. virtual WebRtc_Word32 ReceivedFrameCount(VCMFrameCount& frameCount) const = 0; + + // Returns the number of packets discarded by the jitter buffer due to being + // too late. This can include duplicated packets which arrived after the + // frame was sent to the decoder. Therefore packets which were prematurely + // NACKed will be counted. + virtual WebRtc_UWord32 DiscardedPackets() const = 0; }; } // namespace webrtc diff --git a/src/modules/video_coding/main/source/jitter_buffer.cc b/src/modules/video_coding/main/source/jitter_buffer.cc index 0399f0f8fa..cfada1ace4 100644 --- a/src/modules/video_coding/main/source/jitter_buffer.cc +++ b/src/modules/video_coding/main/source/jitter_buffer.cc @@ -81,6 +81,7 @@ VCMJitterBuffer::VCMJitterBuffer(WebRtc_Word32 vcmId, WebRtc_Word32 receiverId, _dropCount(0), _numConsecutiveOldFrames(0), _numConsecutiveOldPackets(0), + _discardedPackets(0), _jitterEstimate(vcmId, receiverId), _rttMs(0), _nackMode(kNoNack), @@ -134,6 +135,7 @@ VCMJitterBuffer::operator=(const VCMJitterBuffer& rhs) _dropCount = rhs._dropCount; _numConsecutiveOldFrames = rhs._numConsecutiveOldFrames; _numConsecutiveOldPackets = rhs._numConsecutiveOldPackets; + _discardedPackets = rhs._discardedPackets; _jitterEstimate = rhs._jitterEstimate; _delayEstimate = rhs._delayEstimate; _waitingForCompletion = rhs._waitingForCompletion; @@ -210,6 +212,7 @@ VCMJitterBuffer::Start() _numConsecutiveOldFrames = 0; _numConsecutiveOldPackets = 0; + _discardedPackets = 0; _frameEvent.Reset(); // start in a non-signaled state _packetEvent.Reset(); // start in a non-signaled state @@ -438,6 +441,11 @@ VCMJitterBuffer::GetFrameStatistics(WebRtc_UWord32& receivedDeltaFrames, return 0; } +WebRtc_UWord32 VCMJitterBuffer::DiscardedPackets() const { + CriticalSectionScoped cs(_critSect); + return _discardedPackets; +} + // Gets frame to use for this timestamp. If no match, get empty frame. WebRtc_Word32 VCMJitterBuffer::GetFrame(const VCMPacket& packet, VCMEncodedFrame*& frame) @@ -448,11 +456,12 @@ VCMJitterBuffer::GetFrame(const VCMPacket& packet, VCMEncodedFrame*& frame) } _critSect.Enter(); + // Make sure that old empty packets are inserted. if (LatestTimestamp(static_cast(_lastDecodedTimeStamp), packet.timestamp) == _lastDecodedTimeStamp && packet.sizeBytes > 0) - // Make sure that old Empty packets are inserted. { + _discardedPackets++; // Only counts discarded media packets // Trying to get an old frame. _numConsecutiveOldPackets++; if (_numConsecutiveOldPackets > kMaxConsecutiveOldPackets) diff --git a/src/modules/video_coding/main/source/jitter_buffer.h b/src/modules/video_coding/main/source/jitter_buffer.h index bad28d5a8f..0baecb5ac4 100644 --- a/src/modules/video_coding/main/source/jitter_buffer.h +++ b/src/modules/video_coding/main/source/jitter_buffer.h @@ -68,6 +68,9 @@ public: WebRtc_Word32 GetFrameStatistics(WebRtc_UWord32& receivedDeltaFrames, WebRtc_UWord32& receivedKeyFrames) const; + // Get number of packets discarded by the jitter buffer + WebRtc_UWord32 DiscardedPackets() const; + // Statistics, Calculate frame and bit rates WebRtc_Word32 GetUpdate(WebRtc_UWord32& frameRate, WebRtc_UWord32& bitRate); @@ -223,6 +226,9 @@ private: WebRtc_UWord32 _numConsecutiveOldFrames; // Number of packets in a row that have been too old WebRtc_UWord32 _numConsecutiveOldPackets; + // Number of packets discarded by the jitter buffer + WebRtc_UWord32 _discardedPackets; + // Filters for estimating jitter VCMJitterEstimator _jitterEstimate; // Calculates network delays used for jitter calculations diff --git a/src/modules/video_coding/main/source/receiver.cc b/src/modules/video_coding/main/source/receiver.cc index 2e78aebf33..69a49a4077 100644 --- a/src/modules/video_coding/main/source/receiver.cc +++ b/src/modules/video_coding/main/source/receiver.cc @@ -344,6 +344,10 @@ VCMReceiver::ReceivedFrameCount(VCMFrameCount& frameCount) const frameCount.numKeyFrames); } +WebRtc_UWord32 VCMReceiver::DiscardedPackets() const { + return _jitterBuffer.DiscardedPackets(); +} + void VCMReceiver::SetNackMode(VCMNackMode nackMode) { diff --git a/src/modules/video_coding/main/source/receiver.h b/src/modules/video_coding/main/source/receiver.h index 0ca6994a1e..6373e64664 100644 --- a/src/modules/video_coding/main/source/receiver.h +++ b/src/modules/video_coding/main/source/receiver.h @@ -57,6 +57,7 @@ public: void ReleaseFrame(VCMEncodedFrame* frame); WebRtc_Word32 ReceiveStatistics(WebRtc_UWord32& bitRate, WebRtc_UWord32& frameRate); WebRtc_Word32 ReceivedFrameCount(VCMFrameCount& frameCount) const; + WebRtc_UWord32 DiscardedPackets() const; // NACK void SetNackMode(VCMNackMode nackMode); diff --git a/src/modules/video_coding/main/source/video_coding_impl.cc b/src/modules/video_coding/main/source/video_coding_impl.cc index 214a896a60..b094222f69 100644 --- a/src/modules/video_coding/main/source/video_coding_impl.cc +++ b/src/modules/video_coding/main/source/video_coding_impl.cc @@ -31,8 +31,9 @@ VCMProcessTimer::Period() const WebRtc_UWord32 VCMProcessTimer::TimeUntilProcess() const { - return static_cast(VCM_MAX(static_cast(_periodMs) - - (VCMTickTime::MillisecondTimestamp() - _latestMs), 0)); + return static_cast( + VCM_MAX(static_cast(_periodMs) - + (VCMTickTime::MillisecondTimestamp() - _latestMs), 0)); } void @@ -105,8 +106,10 @@ VideoCodingModuleImpl::~VideoCodingModuleImpl() VideoCodingModule* VideoCodingModule::Create(const WebRtc_Word32 id) { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(id), - "VideoCodingModule::Create()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(id), + "VideoCodingModule::Create()"); return new VideoCodingModuleImpl(id); } @@ -115,9 +118,10 @@ VideoCodingModule::Destroy(VideoCodingModule* module) { if (module != NULL) { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, - static_cast(module)->Id(), - "VideoCodingModule::Destroy()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + static_cast(module)->Id(), + "VideoCodingModule::Destroy()"); delete static_cast(module); } } @@ -135,7 +139,8 @@ VideoCodingModuleImpl::Process() { WebRtc_UWord32 bitRate; WebRtc_UWord32 frameRate; - const WebRtc_Word32 ret = _receiver.ReceiveStatistics(bitRate, frameRate); + const WebRtc_Word32 ret = _receiver.ReceiveStatistics(bitRate, + frameRate); if (ret == 0) { _receiveStatsCallback->ReceiveStatistics(bitRate, frameRate); @@ -157,8 +162,10 @@ VideoCodingModuleImpl::Process() WebRtc_UWord32 frameRate; { CriticalSectionScoped cs(_sendCritSect); - bitRate = static_cast(_mediaOpt.SentBitRate() + 0.5f); - frameRate = static_cast(_mediaOpt.SentFrameRate() + 0.5f); + bitRate = static_cast( + _mediaOpt.SentBitRate() + 0.5f); + frameRate = static_cast( + _mediaOpt.SentFrameRate() + 0.5f); } _sendStatsCallback->SendStatistics(bitRate, frameRate); } @@ -207,11 +214,16 @@ VideoCodingModuleImpl::Version(WebRtc_Word8* version, WebRtc_UWord32& remainingBufferInBytes, WebRtc_UWord32& position) const { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), "Version()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "Version()"); if (version == NULL) { - WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceVideoCoding, VCMId(_id), - "Invalid buffer pointer in argument to Version()"); + WEBRTC_TRACE(webrtc::kTraceWarning, + webrtc::kTraceVideoCoding, + VCMId(_id), + "Invalid buffer pointer in argument to Version()"); return VCM_PARAMETER_ERROR; } WebRtc_Word8 ourVersion[] = "VideoCodingModule 1.1.0\n"; @@ -225,7 +237,9 @@ VideoCodingModuleImpl::Version(WebRtc_Word8* version, position += ourLength; // Safe to truncate here. - WebRtc_Word32 ret = _codecDataBase.Version(version, remainingBufferInBytes, position); + WebRtc_Word32 ret = _codecDataBase.Version(version, + remainingBufferInBytes, + position); if (ret < 0) { return ret; @@ -242,7 +256,10 @@ VideoCodingModuleImpl::Version(WebRtc_Word8* version, WebRtc_Word32 VideoCodingModuleImpl::Id() const { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), "Id()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "Id()"); CriticalSectionScoped receiveCs(_receiveCritSect); { CriticalSectionScoped sendCs(_sendCritSect); @@ -254,7 +271,10 @@ VideoCodingModuleImpl::Id() const WebRtc_Word32 VideoCodingModuleImpl::ChangeUniqueId(const WebRtc_Word32 id) { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), "ChangeUniqueId()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "ChangeUniqueId()"); CriticalSectionScoped receiveCs(_receiveCritSect); { CriticalSectionScoped sendCs(_sendCritSect); @@ -263,15 +283,19 @@ VideoCodingModuleImpl::ChangeUniqueId(const WebRtc_Word32 id) } } -// Returns the number of milliseconds until the module wants a worker thread to call Process +// Returns the number of milliseconds until the module wants a worker thread to +// call Process WebRtc_Word32 VideoCodingModuleImpl::TimeUntilNextProcess() { - WebRtc_UWord32 timeUntilNextProcess = VCM_MIN(_receiveStatsTimer.TimeUntilProcess(), - _sendStatsTimer.TimeUntilProcess()); - if ((_receiver.NackMode() != kNoNack) || (_dualReceiver.State() != kPassive)) + WebRtc_UWord32 timeUntilNextProcess = VCM_MIN( + _receiveStatsTimer.TimeUntilProcess(), + _sendStatsTimer.TimeUntilProcess()); + if ((_receiver.NackMode() != kNoNack) || + (_dualReceiver.State() != kPassive)) { - // We need a Process call more often if we are relying on retransmissions + // We need a Process call more often if we are relying on + // retransmissions timeUntilNextProcess = VCM_MIN(timeUntilNextProcess, _retransmissionTimer.TimeUntilProcess()); } @@ -285,7 +309,10 @@ VideoCodingModuleImpl::TimeUntilNextProcess() WebRtc_UWord8 VideoCodingModule::NumberOfCodecs() { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, -1, "NumberOfCodecs()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + -1, + "NumberOfCodecs()"); return VCMCodecDataBase::NumberOfCodecs(); } @@ -293,7 +320,10 @@ VideoCodingModule::NumberOfCodecs() WebRtc_Word32 VideoCodingModule::Codec(WebRtc_UWord8 listId, VideoCodec* codec) { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, -1, "Codec()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + -1, + "Codec()"); if (codec == NULL) { return VCM_PARAMETER_ERROR; @@ -305,7 +335,10 @@ VideoCodingModule::Codec(WebRtc_UWord8 listId, VideoCodec* codec) WebRtc_Word32 VideoCodingModule::Codec(VideoCodecType codecType, VideoCodec* codec) { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, -1, "Codec()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + -1, + "Codec()"); if (codec == NULL) { return VCM_PARAMETER_ERROR; @@ -321,7 +354,10 @@ VideoCodingModule::Codec(VideoCodecType codecType, VideoCodec* codec) WebRtc_Word32 VideoCodingModuleImpl::InitializeSender() { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), "InitializeSender()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "InitializeSender()"); CriticalSectionScoped cs(_sendCritSect); _codecDataBase.ResetSender(); _encoder = NULL; @@ -336,7 +372,10 @@ VideoCodingModuleImpl::InitializeSender() WebRtc_Word32 VideoCodingModuleImpl::ResetEncoder() { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), "ResetEncoder()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "ResetEncoder()"); CriticalSectionScoped cs(_sendCritSect); if (_encoder != NULL) { @@ -351,7 +390,10 @@ VideoCodingModuleImpl::RegisterSendCodec(const VideoCodec* sendCodec, WebRtc_UWord32 numberOfCores, WebRtc_UWord32 maxPayloadSize) { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), "RegisterSendCodec()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "RegisterSendCodec()"); CriticalSectionScoped cs(_sendCritSect); if (sendCodec == NULL) { @@ -368,8 +410,10 @@ VideoCodingModuleImpl::RegisterSendCodec(const VideoCodec* sendCodec, _encoder = _codecDataBase.SetEncoder(sendCodec, &_encodedFrameCallback); if (_encoder == NULL) { - WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoCoding, VCMId(_id), - "Failed to initialize encoder"); + WEBRTC_TRACE(webrtc::kTraceError, + webrtc::kTraceVideoCoding, + VCMId(_id), + "Failed to initialize encoder"); return VCM_CODEC_ERROR; } _sendCodecType = sendCodec->codecType; @@ -388,7 +432,10 @@ VideoCodingModuleImpl::RegisterSendCodec(const VideoCodec* sendCodec, WebRtc_Word32 VideoCodingModuleImpl::SendCodec(VideoCodec* currentSendCodec) const { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), "SendCodec()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "SendCodec()"); CriticalSectionScoped cs(_sendCritSect); if (currentSendCodec == NULL) @@ -402,7 +449,10 @@ VideoCodingModuleImpl::SendCodec(VideoCodec* currentSendCodec) const VideoCodecType VideoCodingModuleImpl::SendCodec() const { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), "SendCodec()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "SendCodec()"); CriticalSectionScoped cs(_sendCritSect); return _codecDataBase.SendCodec(); @@ -415,15 +465,18 @@ VideoCodingModuleImpl::RegisterExternalEncoder(VideoEncoder* externalEncoder, WebRtc_UWord8 payloadType, bool internalSource /*= false*/) { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), - "RegisterExternalEncoder()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "RegisterExternalEncoder()"); CriticalSectionScoped cs(_sendCritSect); if (externalEncoder == NULL) { bool wasSendCodec = false; - const WebRtc_Word32 ret = _codecDataBase.DeRegisterExternalEncoder(payloadType, - wasSendCodec); + const WebRtc_Word32 ret = _codecDataBase.DeRegisterExternalEncoder( + payloadType, + wasSendCodec); if (wasSendCodec) { // Make sure the VCM doesn't use the de-registered codec @@ -438,9 +491,12 @@ VideoCodingModuleImpl::RegisterExternalEncoder(VideoEncoder* externalEncoder, // Get codec config parameters WebRtc_Word32 -VideoCodingModuleImpl::CodecConfigParameters(WebRtc_UWord8* buffer, WebRtc_Word32 size) +VideoCodingModuleImpl::CodecConfigParameters(WebRtc_UWord8* buffer, + WebRtc_Word32 size) { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), "CodecConfigParameters()"); CriticalSectionScoped cs(_sendCritSect); if (_encoder != NULL) @@ -454,7 +510,10 @@ VideoCodingModuleImpl::CodecConfigParameters(WebRtc_UWord8* buffer, WebRtc_Word3 WebRtc_UWord32 VideoCodingModuleImpl::Bitrate() const { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), "Bitrate()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "Bitrate()"); CriticalSectionScoped cs(_sendCritSect); // return the bit rate which the encoder is set to if (_encoder != NULL) @@ -468,7 +527,10 @@ VideoCodingModuleImpl::Bitrate() const WebRtc_UWord32 VideoCodingModuleImpl::FrameRate() const { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), "FrameRate()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "FrameRate()"); CriticalSectionScoped cs(_sendCritSect); // input frame rate, not compensated if (_encoder != NULL) @@ -484,8 +546,10 @@ VideoCodingModuleImpl::SetChannelParameters(WebRtc_UWord32 availableBandWidth, WebRtc_UWord8 lossRate, WebRtc_UWord32 RTT) { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), - "SetChannelParameters()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "SetChannelParameters()"); WebRtc_Word32 ret = 0; { CriticalSectionScoped sendCs(_sendCritSect); @@ -499,7 +563,8 @@ VideoCodingModuleImpl::SetChannelParameters(WebRtc_UWord32 availableBandWidth, { return ret; } - ret = (WebRtc_Word32)_encoder->SetRates(targetRate, _mediaOpt.InputFrameRate()); + ret = (WebRtc_Word32)_encoder->SetRates(targetRate, + _mediaOpt.InputFrameRate()); if (ret < 0) { return ret; @@ -516,55 +581,68 @@ VideoCodingModuleImpl::SetChannelParameters(WebRtc_UWord32 availableBandWidth, WebRtc_Word32 VideoCodingModuleImpl::SetReceiveChannelParameters(WebRtc_UWord32 RTT) { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), - "SetReceiveChannelParameters()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "SetReceiveChannelParameters()"); CriticalSectionScoped receiveCs(_receiveCritSect); _receiver.UpdateRtt(RTT); return 0; } -// Register a transport callback which will be called to deliver the encoded buffers +// Register a transport callback which will be called to deliver the encoded +// buffers WebRtc_Word32 -VideoCodingModuleImpl::RegisterTransportCallback(VCMPacketizationCallback* transport) +VideoCodingModuleImpl::RegisterTransportCallback( + VCMPacketizationCallback* transport) { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), - "RegisterTransportCallback()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "RegisterTransportCallback()"); CriticalSectionScoped cs(_sendCritSect); _encodedFrameCallback.SetMediaOpt(&_mediaOpt); _encodedFrameCallback.SetTransportCallback(transport); return VCM_OK; } -// Register video output information callback which will be called to deliver information -// about the video stream produced by the encoder, for instance the average frame rate and -// bit rate. +// Register video output information callback which will be called to deliver +// information about the video stream produced by the encoder, for instance the +// average frame rate and bit rate. WebRtc_Word32 -VideoCodingModuleImpl::RegisterSendStatisticsCallback(VCMSendStatisticsCallback* sendStats) +VideoCodingModuleImpl::RegisterSendStatisticsCallback( + VCMSendStatisticsCallback* sendStats) { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), - "RegisterSendStatisticsCallback()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "RegisterSendStatisticsCallback()"); CriticalSectionScoped cs(_sendCritSect); _sendStatsCallback = sendStats; return VCM_OK; } -// Register a video quality settings callback which will be called when frame rate/dimensions -// need to be updated for video quality optimization +// Register a video quality settings callback which will be called when frame +// rate/dimensions need to be updated for video quality optimization WebRtc_Word32 -VideoCodingModuleImpl::RegisterVideoQMCallback(VCMQMSettingsCallback* videoQMSettings) +VideoCodingModuleImpl::RegisterVideoQMCallback( + VCMQMSettingsCallback* videoQMSettings) { CriticalSectionScoped cs(_sendCritSect); return _mediaOpt.RegisterVideoQMCallback(videoQMSettings); } -// Register a video protection callback which will be called to deliver the requested FEC rate -// and NACK status (on/off). +// Register a video protection callback which will be called to deliver the +// requested FEC rate and NACK status (on/off). WebRtc_Word32 -VideoCodingModuleImpl::RegisterProtectionCallback(VCMProtectionCallback* protection) +VideoCodingModuleImpl::RegisterProtectionCallback( + VCMProtectionCallback* protection) { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), - "RegisterProtectionCallback()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "RegisterProtectionCallback()"); CriticalSectionScoped cs(_sendCritSect); _mediaOpt.RegisterProtectionCallback(protection); return VCM_OK; @@ -572,10 +650,13 @@ VideoCodingModuleImpl::RegisterProtectionCallback(VCMProtectionCallback* protect // Enable or disable a video protection method. WebRtc_Word32 -VideoCodingModuleImpl::SetVideoProtection(VCMVideoProtection videoProtection, bool enable) +VideoCodingModuleImpl::SetVideoProtection(VCMVideoProtection videoProtection, + bool enable) { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), - "SetVideoProtection()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "SetVideoProtection()"); switch (videoProtection) { @@ -708,7 +789,10 @@ VideoCodingModuleImpl::AddVideoFrame(const VideoFrame& videoFrame, const VideoContentMetrics* _contentMetrics, const CodecSpecificInfo* codecSpecificInfo) { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), "AddVideoFrame()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "AddVideoFrame()"); CriticalSectionScoped cs(_sendCritSect); if (_encoder == NULL) @@ -725,8 +809,10 @@ VideoCodingModuleImpl::AddVideoFrame(const VideoFrame& videoFrame, if (_mediaOpt.DropFrame()) { - WEBRTC_TRACE(webrtc::kTraceStream, webrtc::kTraceVideoCoding, VCMId(_id), - "Drop frame due to bitrate"); + WEBRTC_TRACE(webrtc::kTraceStream, + webrtc::kTraceVideoCoding, + VCMId(_id), + "Drop frame due to bitrate"); } else { @@ -738,13 +824,16 @@ VideoCodingModuleImpl::AddVideoFrame(const VideoFrame& videoFrame, requestedFrameType); if (_encoderInputFile != NULL) { - fwrite(videoFrame.Buffer(), 1, videoFrame.Length(), _encoderInputFile); + fwrite(videoFrame.Buffer(), 1, videoFrame.Length(), + _encoderInputFile); } if (ret < 0) { _nextFrameType = requestedFrameType; - WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoCoding,VCMId(_id), - "Encode error: %d", ret); + WEBRTC_TRACE(webrtc::kTraceError, + webrtc::kTraceVideoCoding, + VCMId(_id), + "Encode error: %d", ret); return ret; } } @@ -757,13 +846,16 @@ VideoCodingModuleImpl::AddVideoFrame(const VideoFrame& videoFrame, WebRtc_Word32 VideoCodingModuleImpl::FrameTypeRequest(FrameType frameType) { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), "FrameTypeRequest()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "FrameTypeRequest()"); CriticalSectionScoped cs(_sendCritSect); _nextFrameType = frameType; if (_encoder != NULL && _encoder->InternalSource()) { - // Try to request the frame if we have an external encoder with internal source - // since AddVideoFrame never will be called. + // Try to request the frame if we have an external encoder with + // internal source since AddVideoFrame never will be called. if (_encoder->RequestFrame(_nextFrameType) == WEBRTC_VIDEO_CODEC_OK) { _nextFrameType = kVideoFrameDelta; @@ -784,7 +876,10 @@ VideoCodingModuleImpl::EnableFrameDropper(bool enable) WebRtc_Word32 VideoCodingModuleImpl::SentFrameCount(VCMFrameCount &frameCount) const { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), "SentFrameCount()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "SentFrameCount()"); CriticalSectionScoped cs(_sendCritSect); return _mediaOpt.SentFrameCount(frameCount); } @@ -793,8 +888,10 @@ VideoCodingModuleImpl::SentFrameCount(VCMFrameCount &frameCount) const WebRtc_Word32 VideoCodingModuleImpl::InitializeReceiver() { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), - "InitializeReceiver()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "InitializeReceiver()"); CriticalSectionScoped cs(_receiveCritSect); WebRtc_Word32 ret = _receiver.Initialize(); if (ret < 0) @@ -823,13 +920,16 @@ VideoCodingModuleImpl::InitializeReceiver() return VCM_OK; } -// Register a receive callback. Will be called whenever there is a new frame ready -// for rendering. +// Register a receive callback. Will be called whenever there is a new frame +// ready for rendering. WebRtc_Word32 -VideoCodingModuleImpl::RegisterReceiveCallback(VCMReceiveCallback* receiveCallback) +VideoCodingModuleImpl::RegisterReceiveCallback( + VCMReceiveCallback* receiveCallback) { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), - "RegisterReceiveCallback()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "RegisterReceiveCallback()"); CriticalSectionScoped cs(_receiveCritSect); _decodedFrameCallback.SetUserReceiveCallback(receiveCallback); return VCM_OK; @@ -839,8 +939,10 @@ WebRtc_Word32 VideoCodingModuleImpl::RegisterReceiveStatisticsCallback( VCMReceiveStatisticsCallback* receiveStats) { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), - "RegisterReceiveStatisticsCallback()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "RegisterReceiveStatisticsCallback()"); CriticalSectionScoped cs(_receiveCritSect); _receiveStatsCallback = receiveStats; return VCM_OK; @@ -874,30 +976,39 @@ VideoCodingModuleImpl::RegisterExternalDecoder(VideoDecoder* externalDecoder, // Register a frame type request callback. WebRtc_Word32 -VideoCodingModuleImpl::RegisterFrameTypeCallback(VCMFrameTypeCallback* frameTypeCallback) +VideoCodingModuleImpl::RegisterFrameTypeCallback( + VCMFrameTypeCallback* frameTypeCallback) { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), - "RegisterFrameTypeCallback()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "RegisterFrameTypeCallback()"); CriticalSectionScoped cs(_receiveCritSect); _frameTypeCallback = frameTypeCallback; return VCM_OK; } WebRtc_Word32 -VideoCodingModuleImpl::RegisterFrameStorageCallback(VCMFrameStorageCallback* frameStorageCallback) +VideoCodingModuleImpl::RegisterFrameStorageCallback( + VCMFrameStorageCallback* frameStorageCallback) { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), - "RegisterFrameStorageCallback()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "RegisterFrameStorageCallback()"); CriticalSectionScoped cs(_receiveCritSect); _frameStorageCallback = frameStorageCallback; return VCM_OK; } WebRtc_Word32 -VideoCodingModuleImpl::RegisterPacketRequestCallback(VCMPacketRequestCallback* callback) +VideoCodingModuleImpl::RegisterPacketRequestCallback( + VCMPacketRequestCallback* callback) { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), - "RegisterPacketRequestCallback()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "RegisterPacketRequestCallback()"); CriticalSectionScoped cs(_receiveCritSect); _packetRequestCallback = callback; return VCM_OK; @@ -908,9 +1019,11 @@ VideoCodingModuleImpl::RegisterPacketRequestCallback(VCMPacketRequestCallback* c WebRtc_Word32 VideoCodingModuleImpl::Decode(WebRtc_UWord16 maxWaitTimeMs) { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), "Decode()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "Decode()"); WebRtc_Word64 nextRenderTimeMs; - { CriticalSectionScoped cs(_receiveCritSect); if (!_receiverInited) @@ -923,20 +1036,23 @@ VideoCodingModuleImpl::Decode(WebRtc_UWord16 maxWaitTimeMs) } } - const bool dualReceiverEnabledNotReceiving = _dualReceiver.State() != kReceiving && - _dualReceiver.NackMode() == kNackInfinite; + const bool dualReceiverEnabledNotReceiving = + (_dualReceiver.State() != kReceiving && + _dualReceiver.NackMode() == kNackInfinite); - VCMEncodedFrame* frame = _receiver.FrameForDecoding(maxWaitTimeMs, - nextRenderTimeMs, - _codecDataBase.RenderTiming(), - &_dualReceiver); + VCMEncodedFrame* frame = _receiver.FrameForDecoding( + maxWaitTimeMs, + nextRenderTimeMs, + _codecDataBase.RenderTiming(), + &_dualReceiver); if (dualReceiverEnabledNotReceiving && _dualReceiver.State() == kReceiving) { - // Dual receiver is enabled (kNACK enabled), but was not receiving before the call to - // FrameForDecoding(). After the call the state changed to receiving, and therefore - // we must copy the primary decoder state to the dual decoder to make it possible - // for the dual decoder to start decoding retransmitted frames and recover. + // Dual receiver is enabled (kNACK enabled), but was not receiving + // before the call to FrameForDecoding(). After the call the state + // changed to receiving, and therefore we must copy the primary decoder + // state to the dual decoder to make it possible for the dual decoder to + // start decoding retransmitted frames and recover. CriticalSectionScoped cs(_receiveCritSect); if (_dualDecoder != NULL) { @@ -945,7 +1061,8 @@ VideoCodingModuleImpl::Decode(WebRtc_UWord16 maxWaitTimeMs) _dualDecoder = _codecDataBase.CreateDecoderCopy(); if (_dualDecoder != NULL) { - _dualDecoder->RegisterDecodeCompleteCallback(&_dualDecodedFrameCallback); + _dualDecoder->RegisterDecodeCompleteCallback( + &_dualDecodedFrameCallback); } } @@ -954,12 +1071,14 @@ VideoCodingModuleImpl::Decode(WebRtc_UWord16 maxWaitTimeMs) CriticalSectionScoped cs(_receiveCritSect); // If this frame was too late, we should adjust the delay accordingly - _timing.UpdateCurrentDelay(frame->RenderTimeMs(), VCMTickTime::MillisecondTimestamp()); + _timing.UpdateCurrentDelay(frame->RenderTimeMs(), + VCMTickTime::MillisecondTimestamp()); if (_bitStreamBeforeDecoder != NULL) { // Write bit stream to file for debugging purposes - fwrite(frame->Buffer(), 1, frame->Length(), _bitStreamBeforeDecoder); + fwrite(frame->Buffer(), 1, frame->Length(), + _bitStreamBeforeDecoder); } if (_frameStorageCallback != NULL) { @@ -982,23 +1101,31 @@ VideoCodingModuleImpl::Decode(WebRtc_UWord16 maxWaitTimeMs) } WebRtc_Word32 -VideoCodingModuleImpl::RequestSliceLossIndication(const WebRtc_UWord64 pictureID) const +VideoCodingModuleImpl::RequestSliceLossIndication( + const WebRtc_UWord64 pictureID) const { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), - "RegisterSliceLossIndication()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "RegisterSliceLossIndication()"); if (_frameTypeCallback != NULL) { - const WebRtc_Word32 ret = _frameTypeCallback->SliceLossIndicationRequest(pictureID); + const WebRtc_Word32 ret = + _frameTypeCallback->SliceLossIndicationRequest(pictureID); if (ret < 0) { - WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoCoding, VCMId(_id), - "Failed to request key frame"); + WEBRTC_TRACE(webrtc::kTraceError, + webrtc::kTraceVideoCoding, + VCMId(_id), + "Failed to request key frame"); return ret; } } else { - WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceVideoCoding, VCMId(_id), - "No frame type request callback registered"); + WEBRTC_TRACE(webrtc::kTraceWarning, + webrtc::kTraceVideoCoding, + VCMId(_id), + "No frame type request callback registered"); return VCM_MISSING_CALLBACK; } return VCM_OK; @@ -1009,19 +1136,24 @@ VideoCodingModuleImpl::RequestKeyFrame() { if (_frameTypeCallback != NULL) { - const WebRtc_Word32 ret = _frameTypeCallback->FrameTypeRequest(kVideoFrameKey); + const WebRtc_Word32 ret = _frameTypeCallback->FrameTypeRequest( + kVideoFrameKey); if (ret < 0) { - WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoCoding, VCMId(_id), - "Failed to request key frame"); + WEBRTC_TRACE(webrtc::kTraceError, + webrtc::kTraceVideoCoding, + VCMId(_id), + "Failed to request key frame"); return ret; } _scheduleKeyRequest = false; } else { - WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceVideoCoding, VCMId(_id), - "No frame type request callback registered"); + WEBRTC_TRACE(webrtc::kTraceWarning, + webrtc::kTraceVideoCoding, + VCMId(_id), + "No frame type request callback registered"); return VCM_MISSING_CALLBACK; } return VCM_OK; @@ -1030,27 +1162,38 @@ VideoCodingModuleImpl::RequestKeyFrame() WebRtc_Word32 VideoCodingModuleImpl::DecodeDualFrame(WebRtc_UWord16 maxWaitTimeMs) { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), "DecodeDualFrame()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "DecodeDualFrame()"); CriticalSectionScoped cs(_receiveCritSect); - if (_dualReceiver.State() != kReceiving || _dualReceiver.NackMode() != kNackInfinite) + if (_dualReceiver.State() != kReceiving || + _dualReceiver.NackMode() != kNackInfinite) { - // The dual receiver is currently not receiving or dual decoder mode is disabled. + // The dual receiver is currently not receiving or + // dual decoder mode is disabled. return VCM_OK; } WebRtc_Word64 dummyRenderTime; WebRtc_Word32 decodeCount = 0; - VCMEncodedFrame* dualFrame = _dualReceiver.FrameForDecoding(maxWaitTimeMs, - dummyRenderTime); + VCMEncodedFrame* dualFrame = _dualReceiver.FrameForDecoding( + maxWaitTimeMs, + dummyRenderTime); if (dualFrame != NULL && _dualDecoder != NULL) { - WEBRTC_TRACE(webrtc::kTraceStream, webrtc::kTraceVideoCoding, VCMId(_id), - "Decoding frame %u with dual decoder", dualFrame->TimeStamp()); + WEBRTC_TRACE(webrtc::kTraceStream, + webrtc::kTraceVideoCoding, + VCMId(_id), + "Decoding frame %u with dual decoder", + dualFrame->TimeStamp()); // Decode dualFrame and try to catch up WebRtc_Word32 ret = _dualDecoder->Decode(*dualFrame); if (ret != WEBRTC_VIDEO_CODEC_OK) { - WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceVideoCoding, VCMId(_id), - "Failed to decode frame with dual decoder"); + WEBRTC_TRACE(webrtc::kTraceWarning, + webrtc::kTraceVideoCoding, + VCMId(_id), + "Failed to decode frame with dual decoder"); _dualReceiver.ReleaseFrame(dualFrame); return VCM_CODEC_ERROR; } @@ -1058,8 +1201,10 @@ VideoCodingModuleImpl::DecodeDualFrame(WebRtc_UWord16 maxWaitTimeMs) { // Copy the complete decoder state of the dual decoder // to the primary decoder. - WEBRTC_TRACE(webrtc::kTraceStream, webrtc::kTraceVideoCoding, VCMId(_id), - "Dual decoder caught up"); + WEBRTC_TRACE(webrtc::kTraceStream, + webrtc::kTraceVideoCoding, + VCMId(_id), + "Dual decoder caught up"); _codecDataBase.CopyDecoder(*_dualDecoder); _codecDataBase.ReleaseDecoder(_dualDecoder); _dualDecoder = NULL; @@ -1077,7 +1222,8 @@ VideoCodingModuleImpl::Decode(const VCMEncodedFrame& frame) { // Change decoder if payload type has changed const bool renderTimingBefore = _codecDataBase.RenderTiming(); - _decoder = _codecDataBase.SetDecoder(frame.PayloadType(), _decodedFrameCallback); + _decoder = _codecDataBase.SetDecoder(frame.PayloadType(), + _decodedFrameCallback); if (renderTimingBefore != _codecDataBase.RenderTiming()) { // Make sure we reset the decode time estimate since it will @@ -1101,14 +1247,18 @@ VideoCodingModuleImpl::Decode(const VCMEncodedFrame& frame) } else { - WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoCoding, VCMId(_id), - "Failed to decode frame %u, requesting key frame", frame.TimeStamp()); + WEBRTC_TRACE(webrtc::kTraceError, + webrtc::kTraceVideoCoding, + VCMId(_id), + "Failed to decode frame %u, requesting key frame", + frame.TimeStamp()); ret = RequestKeyFrame(); } } else if (ret == VCM_REQUEST_SLI) { - ret = RequestSliceLossIndication(_decodedFrameCallback.LastReceivedPictureID() + 1); + ret = RequestSliceLossIndication( + _decodedFrameCallback.LastReceivedPictureID() + 1); } if (!frame.Complete() || frame.MissingFrame()) { @@ -1136,9 +1286,13 @@ VideoCodingModuleImpl::Decode(const VCMEncodedFrame& frame) } WebRtc_Word32 -VideoCodingModuleImpl::DecodeFromStorage(const EncodedVideoData& frameFromStorage) +VideoCodingModuleImpl::DecodeFromStorage( + const EncodedVideoData& frameFromStorage) { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), "DecodeFromStorage()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "DecodeFromStorage()"); CriticalSectionScoped cs(_receiveCritSect); WebRtc_Word32 ret = _frameFromFile.ExtractFromStorage(frameFromStorage); if (ret < 0) @@ -1152,7 +1306,10 @@ VideoCodingModuleImpl::DecodeFromStorage(const EncodedVideoData& frameFromStorag WebRtc_Word32 VideoCodingModuleImpl::ResetDecoder() { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), "ResetDecoder()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "ResetDecoder()"); CriticalSectionScoped cs(_receiveCritSect); if (_decoder != NULL) { @@ -1179,21 +1336,27 @@ VideoCodingModuleImpl::RegisterReceiveCodec(const VideoCodec* receiveCodec, WebRtc_Word32 numberOfCores, bool requireKeyFrame) { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), - "RegisterReceiveCodec()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "RegisterReceiveCodec()"); CriticalSectionScoped cs(_receiveCritSect); if (receiveCodec == NULL) { return VCM_PARAMETER_ERROR; } - return _codecDataBase.RegisterReceiveCodec(receiveCodec, numberOfCores, requireKeyFrame); + return _codecDataBase.RegisterReceiveCodec(receiveCodec, numberOfCores, + requireKeyFrame); } // Get current received codec WebRtc_Word32 VideoCodingModuleImpl::ReceiveCodec(VideoCodec* currentReceiveCodec) const { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), "ReceiveCodec()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "ReceiveCodec()"); CriticalSectionScoped cs(_receiveCritSect); if (currentReceiveCodec == NULL) { @@ -1206,7 +1369,10 @@ VideoCodingModuleImpl::ReceiveCodec(VideoCodec* currentReceiveCodec) const VideoCodecType VideoCodingModuleImpl::ReceiveCodec() const { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), "ReceiveCodec()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "ReceiveCodec()"); CriticalSectionScoped cs(_receiveCritSect); return _codecDataBase.ReceiveCodec(); } @@ -1217,7 +1383,10 @@ VideoCodingModuleImpl::IncomingPacket(const WebRtc_UWord8* incomingPayload, WebRtc_UWord32 payloadLength, const WebRtcRTPHeader& rtpInfo) { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), "IncomingPacket()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "IncomingPacket()"); const VCMPacket packet(incomingPayload, payloadLength, rtpInfo); WebRtc_Word32 ret; if (_dualReceiver.State() != kPassive) @@ -1230,7 +1399,8 @@ VideoCodingModuleImpl::IncomingPacket(const WebRtc_UWord8* incomingPayload, return ret; } } - ret = _receiver.InsertPacket(packet, rtpInfo.type.Video.width, rtpInfo.type.Video.height); + ret = _receiver.InsertPacket(packet, rtpInfo.type.Video.width, + rtpInfo.type.Video.height); if (ret < 0) { return ret; @@ -1241,18 +1411,24 @@ VideoCodingModuleImpl::IncomingPacket(const WebRtc_UWord8* incomingPayload, // Set codec config parameters WebRtc_Word32 VideoCodingModuleImpl::SetCodecConfigParameters(WebRtc_UWord8 payloadType, - const WebRtc_UWord8* buffer, - WebRtc_Word32 length) + const WebRtc_UWord8* buffer, + WebRtc_Word32 length) { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), - "SetCodecConfigParameters()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "SetCodecConfigParameters()"); CriticalSectionScoped cs(_receiveCritSect); - WebRtc_Word32 ret = _codecDataBase.SetCodecConfigParameters(payloadType, buffer, length); + WebRtc_Word32 ret = _codecDataBase.SetCodecConfigParameters(payloadType, + buffer, + length); if (ret < 0) { - WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoCoding, VCMId(_id), - "SetCodecConfigParameters() failed, %d", ret); + WEBRTC_TRACE(webrtc::kTraceError, + webrtc::kTraceVideoCoding, + VCMId(_id), + "SetCodecConfigParameters() failed, %d", ret); return ret; } return VCM_OK; @@ -1264,8 +1440,10 @@ VideoCodingModuleImpl::SetCodecConfigParameters(WebRtc_UWord8 payloadType, WebRtc_Word32 VideoCodingModuleImpl::SetMinimumPlayoutDelay(WebRtc_UWord32 minPlayoutDelayMs) { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), - "SetMininumPlayoutDelay(%u)", minPlayoutDelayMs); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "SetMininumPlayoutDelay(%u)", minPlayoutDelayMs); _timing.SetMinimumTotalDelay(minPlayoutDelayMs); return VCM_OK; } @@ -1275,8 +1453,10 @@ VideoCodingModuleImpl::SetMinimumPlayoutDelay(WebRtc_UWord32 minPlayoutDelayMs) WebRtc_Word32 VideoCodingModuleImpl::SetRenderDelay(WebRtc_UWord32 timeMS) { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), - "SetRenderDelay(%u)", timeMS); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "SetRenderDelay(%u)", timeMS); _timing.SetRenderDelay(timeMS); return VCM_OK; } @@ -1285,7 +1465,10 @@ VideoCodingModuleImpl::SetRenderDelay(WebRtc_UWord32 timeMS) WebRtc_Word32 VideoCodingModuleImpl::Delay() const { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), "Delay()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "Delay()"); return _timing.TargetVideoDelay(); } @@ -1314,15 +1497,19 @@ VideoCodingModuleImpl::NackList(WebRtc_UWord16* nackList, WebRtc_UWord16& size) { case kNackNeedMoreMemory: { - WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoCoding, VCMId(_id), - "Out of memory"); + WEBRTC_TRACE(webrtc::kTraceError, + webrtc::kTraceVideoCoding, + VCMId(_id), + "Out of memory"); return VCM_MEMORY; } case kNackKeyFrameRequest: { CriticalSectionScoped cs(_receiveCritSect); - WEBRTC_TRACE(webrtc::kTraceWarning, webrtc::kTraceVideoCoding, VCMId(_id), - "Failed to get NACK list, requesting key frame"); + WEBRTC_TRACE(webrtc::kTraceWarning, + webrtc::kTraceVideoCoding, + VCMId(_id), + "Failed to get NACK list, requesting key frame"); return RequestKeyFrame(); } default: @@ -1334,9 +1521,19 @@ VideoCodingModuleImpl::NackList(WebRtc_UWord16* nackList, WebRtc_UWord16& size) WebRtc_Word32 VideoCodingModuleImpl::ReceivedFrameCount(VCMFrameCount& frameCount) const { - WEBRTC_TRACE(webrtc::kTraceModuleCall, webrtc::kTraceVideoCoding, VCMId(_id), - "ReceivedFrameCount()"); + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "ReceivedFrameCount()"); return _receiver.ReceivedFrameCount(frameCount); } +WebRtc_UWord32 VideoCodingModuleImpl::DiscardedPackets() const { + WEBRTC_TRACE(webrtc::kTraceModuleCall, + webrtc::kTraceVideoCoding, + VCMId(_id), + "DiscardedPackets()"); + return _receiver.DiscardedPackets(); } + +} // namespace webrtc diff --git a/src/modules/video_coding/main/source/video_coding_impl.h b/src/modules/video_coding/main/source/video_coding_impl.h index 69f44931fe..8663bfe8e6 100644 --- a/src/modules/video_coding/main/source/video_coding_impl.h +++ b/src/modules/video_coding/main/source/video_coding_impl.h @@ -43,9 +43,11 @@ private: enum VCMKeyRequestMode { - kKeyOnError, // Normal mode, request key frames on decoder error - kKeyOnKeyLoss, // Request key frames on decoder error and on packet loss in key frames - kKeyOnLoss, // Request key frames on decoder error and on packet loss in any frame + kKeyOnError, // Normal mode, request key frames on decoder error + kKeyOnKeyLoss, // Request key frames on decoder error and on packet loss + // in key frames. + kKeyOnLoss, // Request key frames on decoder error and on packet loss + // in any frame }; class VideoCodingModuleImpl : public VideoCodingModule @@ -65,7 +67,8 @@ public: // Change the unique identifier of this object virtual WebRtc_Word32 ChangeUniqueId(const WebRtc_Word32 id); - // Returns the number of milliseconds until the module want a worker thread to call Process + // Returns the number of milliseconds until the module want a worker thread + // to call Process virtual WebRtc_Word32 TimeUntilNextProcess(); virtual WebRtc_Word32 Process(); @@ -97,7 +100,8 @@ public: bool internalSource = false); // Get codec config parameters - virtual WebRtc_Word32 CodecConfigParameters(WebRtc_UWord8* buffer, WebRtc_Word32 size); + virtual WebRtc_Word32 CodecConfigParameters(WebRtc_UWord8* buffer, + WebRtc_Word32 size); // Get encode bitrate virtual WebRtc_UWord32 Bitrate() const; @@ -106,30 +110,38 @@ public: virtual WebRtc_UWord32 FrameRate() const; // Set channel parameters - virtual WebRtc_Word32 SetChannelParameters(WebRtc_UWord32 availableBandWidth, - WebRtc_UWord8 lossRate, - WebRtc_UWord32 RTT); + virtual WebRtc_Word32 SetChannelParameters( + WebRtc_UWord32 availableBandWidth, + WebRtc_UWord8 lossRate, + WebRtc_UWord32 RTT); + // Set recieve channel parameters virtual WebRtc_Word32 SetReceiveChannelParameters(WebRtc_UWord32 RTT); - // Register a transport callback which will be called to deliver the encoded buffers - virtual WebRtc_Word32 RegisterTransportCallback(VCMPacketizationCallback* transport); + // Register a transport callback which will be called to deliver the + // encoded buffers + virtual WebRtc_Word32 RegisterTransportCallback( + VCMPacketizationCallback* transport); - // Register a send statistics callback which will be called to deliver information - // about the video stream produced by the encoder, + // Register a send statistics callback which will be called to deliver + // information about the video stream produced by the encoder, // for instance the average frame rate and bit rate. - virtual WebRtc_Word32 RegisterSendStatisticsCallback(VCMSendStatisticsCallback* sendStats); + virtual WebRtc_Word32 RegisterSendStatisticsCallback( + VCMSendStatisticsCallback* sendStats); // Register a video quality settings callback which will be called when // frame rate/dimensions need to be updated for video quality optimization - virtual WebRtc_Word32 RegisterVideoQMCallback(VCMQMSettingsCallback* videoQMSettings); + virtual WebRtc_Word32 RegisterVideoQMCallback( + VCMQMSettingsCallback* videoQMSettings); // Register a video protection callback which will be called to deliver // the requested FEC rate and NACK status (on/off). - virtual WebRtc_Word32 RegisterProtectionCallback(VCMProtectionCallback* protection); + virtual WebRtc_Word32 RegisterProtectionCallback( + VCMProtectionCallback* protection); // Enable or disable a video protection method. - virtual WebRtc_Word32 SetVideoProtection(VCMVideoProtection videoProtection, bool enable); + virtual WebRtc_Word32 SetVideoProtection(VCMVideoProtection videoProtection, + bool enable); // Add one raw video frame to the encoder, blocking. virtual WebRtc_Word32 AddVideoFrame( @@ -164,31 +176,35 @@ public: WebRtc_UWord8 payloadType, bool internalRenderTiming); - // Register a receive callback. Will be called whenever there are a new frame ready - // for rendering. - virtual WebRtc_Word32 RegisterReceiveCallback(VCMReceiveCallback* receiveCallback); + // Register a receive callback. Will be called whenever there are a new + // frame ready for rendering. + virtual WebRtc_Word32 RegisterReceiveCallback( + VCMReceiveCallback* receiveCallback); - // Register a receive statistics callback which will be called to deliver information - // about the video stream received by the receiving side of the VCM, for instance - // the average frame rate and bit rate. + // Register a receive statistics callback which will be called to deliver + // information about the video stream received by the receiving side of the + // VCM, for instance the average frame rate and bit rate. virtual WebRtc_Word32 RegisterReceiveStatisticsCallback( - VCMReceiveStatisticsCallback* receiveStats); + VCMReceiveStatisticsCallback* receiveStats); // Register a frame type request callback. - virtual WebRtc_Word32 RegisterFrameTypeCallback(VCMFrameTypeCallback* frameTypeCallback); + virtual WebRtc_Word32 RegisterFrameTypeCallback( + VCMFrameTypeCallback* frameTypeCallback); // Register a frame storage callback. virtual WebRtc_Word32 RegisterFrameStorageCallback( - VCMFrameStorageCallback* frameStorageCallback); + VCMFrameStorageCallback* frameStorageCallback); // Nack callback - virtual WebRtc_Word32 RegisterPacketRequestCallback(VCMPacketRequestCallback* callback); + virtual WebRtc_Word32 RegisterPacketRequestCallback( + VCMPacketRequestCallback* callback); // Decode next frame, blocks for a maximum of maxWaitTimeMs milliseconds. // Should be called as often as possible to get the most out of the decoder. virtual WebRtc_Word32 Decode(WebRtc_UWord16 maxWaitTimeMs = 200); - // Decode next dual frame, blocks for a maximum of maxWaitTimeMs milliseconds. + // Decode next dual frame, blocks for a maximum of maxWaitTimeMs + // milliseconds. virtual WebRtc_Word32 DecodeDualFrame(WebRtc_UWord16 maxWaitTimeMs = 200); // Reset the decoder state @@ -207,17 +223,19 @@ public: // A part of an encoded frame to be decoded. // Used in conjunction with VCMFrameStorageCallback. - virtual WebRtc_Word32 DecodeFromStorage(const EncodedVideoData& frameFromStorage); + virtual WebRtc_Word32 DecodeFromStorage( + const EncodedVideoData& frameFromStorage); // Set codec config parameters virtual WebRtc_Word32 SetCodecConfigParameters(WebRtc_UWord8 payloadType, const WebRtc_UWord8* buffer, WebRtc_Word32 length); - // Minimum playout delay (Used for lip-sync). This is the minimum delay required - // to sync with audio. Not included in VideoCodingModule::Delay() + // Minimum playout delay (Used for lip-sync). This is the minimum delay + // required to sync with audio. Not included in VideoCodingModule::Delay() // Defaults to 0 ms. - virtual WebRtc_Word32 SetMinimumPlayoutDelay(WebRtc_UWord32 minPlayoutDelayMs); + virtual WebRtc_Word32 SetMinimumPlayoutDelay( + WebRtc_UWord32 minPlayoutDelayMs); // The estimated delay caused by rendering virtual WebRtc_Word32 SetRenderDelay(WebRtc_UWord32 timeMS); @@ -228,15 +246,19 @@ public: // Received frame counters virtual WebRtc_Word32 ReceivedFrameCount(VCMFrameCount& frameCount) const; + // Returns the number of packets discarded by the jitter buffer. + virtual WebRtc_UWord32 DiscardedPackets() const; + protected: WebRtc_Word32 Decode(const webrtc::VCMEncodedFrame& frame); WebRtc_Word32 RequestKeyFrame(); - WebRtc_Word32 RequestSliceLossIndication(const WebRtc_UWord64 pictureID) const; + WebRtc_Word32 RequestSliceLossIndication( + const WebRtc_UWord64 pictureID) const; WebRtc_Word32 NackList(WebRtc_UWord16* nackList, WebRtc_UWord16& size); private: WebRtc_Word32 _id; - CriticalSectionWrapper& _receiveCritSect; // Critical section for receive side + CriticalSectionWrapper& _receiveCritSect; bool _receiverInited; VCMTiming _timing; VCMTiming _dualTiming; @@ -255,7 +277,7 @@ private: VCMKeyRequestMode _keyRequestMode; bool _scheduleKeyRequest; - CriticalSectionWrapper& _sendCritSect; // Critical section for send side + CriticalSectionWrapper& _sendCritSect; VCMGenericEncoder* _encoder; VCMEncodedFrameCallback _encodedFrameCallback; FrameType _nextFrameType; diff --git a/src/modules/video_coding/main/test/jitter_buffer_test.cc b/src/modules/video_coding/main/test/jitter_buffer_test.cc index af14d91b43..a125228bdf 100644 --- a/src/modules/video_coding/main/test/jitter_buffer_test.cc +++ b/src/modules/video_coding/main/test/jitter_buffer_test.cc @@ -800,11 +800,6 @@ int JitterBufferTest(CmdArgs& args) packet.insertStartCode = false; //printf("DONE H.264 insert start code test 2 packets\n"); - - // Temporarily do this to make the rest of the test work: - timeStamp += 33*90; - seqNum += 4; - // // TEST statistics // @@ -823,8 +818,35 @@ int JitterBufferTest(CmdArgs& args) TEST(frameRate > 30); TEST(bitRate > 10000000); + + // Insert 3 old packets and verify that we have 3 discarded packets + packet.timestamp = timeStamp - 1000; + frameIn = jb.GetFrame(packet); + TEST(frameIn == NULL); + + packet.timestamp = timeStamp - 500; + frameIn = jb.GetFrame(packet); + TEST(frameIn == NULL); + + packet.timestamp = timeStamp - 100; + frameIn = jb.GetFrame(packet); + TEST(frameIn == NULL); + + TEST(jb.DiscardedPackets() == 3); + + jb.Flush(); + + // This statistic shouldn't be reset by a flush. + TEST(jb.DiscardedPackets() == 3); + //printf("DONE Statistics\n"); + + // Temporarily do this to make the rest of the test work: + timeStamp += 33*90; + seqNum += 4; + + // // TEST delta frame 100 packets with seqNum wrap // @@ -833,7 +855,6 @@ int JitterBufferTest(CmdArgs& args) // --------------------------------------- // - // test flush jb.Flush(); // insert first packet diff --git a/src/video_engine/main/interface/vie_codec.h b/src/video_engine/main/interface/vie_codec.h index 0b2d2aee1f..dded4598fb 100644 --- a/src/video_engine/main/interface/vie_codec.h +++ b/src/video_engine/main/interface/vie_codec.h @@ -140,6 +140,10 @@ public: unsigned int& keyFrames, unsigned int& deltaFrames) const = 0; + // Gets the number of packets discarded by the jitter buffer because they + // arrived too late. + virtual unsigned int GetDiscardedPackets(const int videoChannel) const = 0; + // Enables key frame request callback in ViEDecoderObserver. virtual int SetKeyFrameRequestCallbackStatus(const int videoChannel, const bool enable) = 0; diff --git a/src/video_engine/main/source/vie_channel.cc b/src/video_engine/main/source/vie_channel.cc index f813411cc3..fa25f57070 100644 --- a/src/video_engine/main/source/vie_channel.cc +++ b/src/video_engine/main/source/vie_channel.cc @@ -515,6 +515,14 @@ WebRtc_Word32 ViEChannel::ReceiveCodecStatistics(WebRtc_UWord32& numKeyFrames, return 0; } +WebRtc_UWord32 ViEChannel::DiscardedPackets() const { + WEBRTC_TRACE(webrtc::kTraceInfo, + webrtc::kTraceVideo, + ViEId(_engineId, _channelId), + "%s", __FUNCTION__); + return _vcm.DiscardedPackets(); +} + // ---------------------------------------------------------------------------- // WaitForKeyFrame // diff --git a/src/video_engine/main/source/vie_channel.h b/src/video_engine/main/source/vie_channel.h index 5cce009ba8..29e71c9200 100644 --- a/src/video_engine/main/source/vie_channel.h +++ b/src/video_engine/main/source/vie_channel.h @@ -95,6 +95,8 @@ public: WebRtc_Word32 ReceiveCodecStatistics(WebRtc_UWord32& numKeyFrames, WebRtc_UWord32& numDeltaFrames); + WebRtc_UWord32 DiscardedPackets() const; + WebRtc_Word32 WaitForKeyFrame(bool wait); WebRtc_Word32 SetSignalPacketLossStatus(bool enable, bool onlyKeyFrames); diff --git a/src/video_engine/main/source/vie_codec_impl.cc b/src/video_engine/main/source/vie_codec_impl.cc index 7991c09f8f..94af2f07b0 100644 --- a/src/video_engine/main/source/vie_codec_impl.cc +++ b/src/video_engine/main/source/vie_codec_impl.cc @@ -566,6 +566,25 @@ int ViECodecImpl::GetReceiveCodecStastistics(const int videoChannel, } +unsigned int ViECodecImpl::GetDiscardedPackets(const int videoChannel) const { + WEBRTC_TRACE(webrtc::kTraceApiCall, webrtc::kTraceVideo, + ViEId(_instanceId, videoChannel), + "%s(videoChannel: %d, codecType: %d)", __FUNCTION__, + videoChannel); + + ViEChannelManagerScoped cs(_channelManager); + ViEChannel* vieChannel = cs.Channel(videoChannel); + if (vieChannel == NULL) + { + WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideo, + ViEId(_instanceId, videoChannel), "%s: No channel %d", + __FUNCTION__, videoChannel); + SetLastError(kViECodecInvalidChannelId); + return -1; + } + return vieChannel->DiscardedPackets(); +} + // Callbacks // ---------------------------------------------------------------------------- // SetKeyFrameRequestCallbackStatus diff --git a/src/video_engine/main/source/vie_codec_impl.h b/src/video_engine/main/source/vie_codec_impl.h index b47c7a2281..576189a4d6 100644 --- a/src/video_engine/main/source/vie_codec_impl.h +++ b/src/video_engine/main/source/vie_codec_impl.h @@ -72,6 +72,8 @@ public: unsigned int& keyFrames, unsigned int& deltaFrames) const; + virtual unsigned int GetDiscardedPackets(const int videoChannel) const; + // Callbacks virtual int SetKeyFrameRequestCallbackStatus(const int videoChannel, const bool enable);