From 05f84876279e369f1fd7c0913c247a8d9d30551b Mon Sep 17 00:00:00 2001 From: Johannes Kron Date: Thu, 16 Jan 2020 14:09:33 +0100 Subject: [PATCH] Add processing time to VideoFrame Bug: chromium:1011581 Change-Id: Icd675cb98b8b5052933b9a8eebe718be94c2fef2 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/166162 Commit-Queue: Johannes Kron Reviewed-by: Ilya Nikolaevskiy Reviewed-by: Niels Moller Cr-Commit-Position: refs/heads/master@{#30281} --- api/video/video_frame.h | 18 ++++++++++++++++++ modules/video_coding/generic_decoder.cc | 16 +++++++++------- modules/video_coding/generic_decoder.h | 6 +++--- .../video_coding/generic_decoder_unittest.cc | 4 ++-- modules/video_coding/video_receiver.cc | 2 +- modules/video_coding/video_receiver2.cc | 2 +- 6 files changed, 34 insertions(+), 14 deletions(-) diff --git a/api/video/video_frame.h b/api/video/video_frame.h index f312e7add8..08c939d916 100644 --- a/api/video/video_frame.h +++ b/api/video/video_frame.h @@ -72,6 +72,12 @@ class RTC_EXPORT VideoFrame { int scaled_height) const; }; + struct RTC_EXPORT ProcessingTime { + TimeDelta Elapsed() const { return finish - start; } + Timestamp start; + Timestamp finish; + }; + // Preferred way of building VideoFrame objects. class RTC_EXPORT Builder { public: @@ -223,6 +229,13 @@ class RTC_EXPORT VideoFrame { packet_infos_ = std::move(value); } + const absl::optional processing_time() const { + return processing_time_; + } + void set_processing_time(const ProcessingTime& processing_time) { + processing_time_ = processing_time; + } + private: VideoFrame(uint16_t id, const rtc::scoped_refptr& buffer, @@ -252,6 +265,11 @@ class RTC_EXPORT VideoFrame { // MediaStreamTrack, in order to implement getContributingSources(). See: // https://w3c.github.io/webrtc-pc/#dom-rtcrtpreceiver-getcontributingsources RtpPacketInfos packet_infos_; + // Processing timestamps of the frame. For received video frames these are the + // timestamps when the frame is sent to the decoder and the decoded image + // returned from the decoder. + // Currently, not set for locally captured video frames. + absl::optional processing_time_; }; } // namespace webrtc diff --git a/modules/video_coding/generic_decoder.cc b/modules/video_coding/generic_decoder.cc index 2cd3204b04..100686d336 100644 --- a/modules/video_coding/generic_decoder.cc +++ b/modules/video_coding/generic_decoder.cc @@ -99,11 +99,13 @@ void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, decodedImage.set_packet_infos(frameInfo->packet_infos); decodedImage.set_rotation(frameInfo->rotation); - const int64_t now_ms = _clock->TimeInMilliseconds(); + const Timestamp now = _clock->CurrentTime(); + RTC_DCHECK(frameInfo->decodeStart); if (!decode_time_ms) { - decode_time_ms = now_ms - frameInfo->decodeStartTimeMs; + decode_time_ms = (now - *frameInfo->decodeStart).ms(); } - _timing->StopDecodeTimer(*decode_time_ms, now_ms); + _timing->StopDecodeTimer(*decode_time_ms, now.ms()); + decodedImage.set_processing_time({*frameInfo->decodeStart, now}); // Report timing information. TimingFrameInfo timing_frame_info; @@ -147,8 +149,8 @@ void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, } timing_frame_info.flags = frameInfo->timing.flags; - timing_frame_info.decode_start_ms = frameInfo->decodeStartTimeMs; - timing_frame_info.decode_finish_ms = now_ms; + timing_frame_info.decode_start_ms = frameInfo->decodeStart->ms(); + timing_frame_info.decode_finish_ms = now.ms(); timing_frame_info.render_time_ms = frameInfo->renderTimeMs; timing_frame_info.rtp_timestamp = decodedImage.timestamp(); timing_frame_info.receive_start_ms = frameInfo->timing.receive_start_ms; @@ -210,10 +212,10 @@ int32_t VCMGenericDecoder::InitDecode(const VideoCodec* settings, return decoder_->InitDecode(settings, numberOfCores); } -int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, int64_t nowMs) { +int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, Timestamp now) { TRACE_EVENT1("webrtc", "VCMGenericDecoder::Decode", "timestamp", frame.Timestamp()); - _frameInfos[_nextFrameInfoIdx].decodeStartTimeMs = nowMs; + _frameInfos[_nextFrameInfoIdx].decodeStart = now; _frameInfos[_nextFrameInfoIdx].renderTimeMs = frame.RenderTimeMs(); _frameInfos[_nextFrameInfoIdx].rotation = frame.rotation(); _frameInfos[_nextFrameInfoIdx].timing = frame.video_timing(); diff --git a/modules/video_coding/generic_decoder.h b/modules/video_coding/generic_decoder.h index a9d969869b..4b4d83ecd5 100644 --- a/modules/video_coding/generic_decoder.h +++ b/modules/video_coding/generic_decoder.h @@ -30,14 +30,14 @@ enum { kDecoderFrameMemoryLength = 10 }; struct VCMFrameInformation { int64_t renderTimeMs; - int64_t decodeStartTimeMs; + absl::optional decodeStart; void* userData; VideoRotation rotation; VideoContentType content_type; EncodedImage::Timing timing; int64_t ntp_time_ms; RtpPacketInfos packet_infos; - // ColorSpace is not storred here, as it might be modified by decoders. + // ColorSpace is not stored here, as it might be modified by decoders. }; class VCMDecodedFrameCallback : public DecodedImageCallback { @@ -92,7 +92,7 @@ class VCMGenericDecoder { * * inputVideoBuffer reference to encoded video frame */ - int32_t Decode(const VCMEncodedFrame& inputFrame, int64_t nowMs); + int32_t Decode(const VCMEncodedFrame& inputFrame, Timestamp now); /** * Set decode callback. Deregistering while decoding is illegal. diff --git a/modules/video_coding/generic_decoder_unittest.cc b/modules/video_coding/generic_decoder_unittest.cc index 66167eb610..3e07a2a81c 100644 --- a/modules/video_coding/generic_decoder_unittest.cc +++ b/modules/video_coding/generic_decoder_unittest.cc @@ -93,7 +93,7 @@ TEST_F(GenericDecoderTest, PassesPacketInfos) { RtpPacketInfos packet_infos = CreatePacketInfos(3); VCMEncodedFrame encoded_frame; encoded_frame.SetPacketInfos(packet_infos); - generic_decoder_.Decode(encoded_frame, clock_.TimeInMilliseconds()); + generic_decoder_.Decode(encoded_frame, clock_.CurrentTime()); absl::optional decoded_frame = user_callback_.WaitForFrame(10); ASSERT_TRUE(decoded_frame.has_value()); EXPECT_EQ(decoded_frame->packet_infos().size(), 3U); @@ -107,7 +107,7 @@ TEST_F(GenericDecoderTest, PassesPacketInfosForDelayedDecoders) { // Ensure the original frame is destroyed before the decoding is completed. VCMEncodedFrame encoded_frame; encoded_frame.SetPacketInfos(packet_infos); - generic_decoder_.Decode(encoded_frame, clock_.TimeInMilliseconds()); + generic_decoder_.Decode(encoded_frame, clock_.CurrentTime()); } absl::optional decoded_frame = user_callback_.WaitForFrame(200); diff --git a/modules/video_coding/video_receiver.cc b/modules/video_coding/video_receiver.cc index e52abf211a..44d5526a0a 100644 --- a/modules/video_coding/video_receiver.cc +++ b/modules/video_coding/video_receiver.cc @@ -259,7 +259,7 @@ int32_t VideoReceiver::Decode(const VCMEncodedFrame& frame) { if (decoder == nullptr) { return VCM_NO_CODEC_REGISTERED; } - return decoder->Decode(frame, clock_->TimeInMilliseconds()); + return decoder->Decode(frame, clock_->CurrentTime()); } // Register possible receive codecs, can be called multiple times diff --git a/modules/video_coding/video_receiver2.cc b/modules/video_coding/video_receiver2.cc index d1e57d17be..8eaefbb8da 100644 --- a/modules/video_coding/video_receiver2.cc +++ b/modules/video_coding/video_receiver2.cc @@ -91,7 +91,7 @@ int32_t VideoReceiver2::Decode(const VCMEncodedFrame* frame) { if (decoder == nullptr) { return VCM_NO_CODEC_REGISTERED; } - return decoder->Decode(*frame, clock_->TimeInMilliseconds()); + return decoder->Decode(*frame, clock_->CurrentTime()); } // Register possible receive codecs, can be called multiple times