Add processing time to VideoFrame

Bug: chromium:1011581
Change-Id: Icd675cb98b8b5052933b9a8eebe718be94c2fef2
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/166162
Commit-Queue: Johannes Kron <kron@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Niels Moller <nisse@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30281}

api/video/video_frame.h

@@ -72,6 +72,12 @@ class RTC_EXPORT VideoFrame {
                               int scaled_height) const;
   };
 
+  struct RTC_EXPORT ProcessingTime {
+    TimeDelta Elapsed() const { return finish - start; }
+    Timestamp start;
+    Timestamp finish;
+  };
+
   // Preferred way of building VideoFrame objects.
   class RTC_EXPORT Builder {
    public:
@@ -223,6 +229,13 @@ class RTC_EXPORT VideoFrame {
     packet_infos_ = std::move(value);
   }
 
+  const absl::optional<ProcessingTime> processing_time() const {
+    return processing_time_;
+  }
+  void set_processing_time(const ProcessingTime& processing_time) {
+    processing_time_ = processing_time;
+  }
+
  private:
   VideoFrame(uint16_t id,
              const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
@@ -252,6 +265,11 @@ class RTC_EXPORT VideoFrame {
   // MediaStreamTrack, in order to implement getContributingSources(). See:
   // https://w3c.github.io/webrtc-pc/#dom-rtcrtpreceiver-getcontributingsources
   RtpPacketInfos packet_infos_;
+  // Processing timestamps of the frame. For received video frames these are the
+  // timestamps when the frame is sent to the decoder and the decoded image
+  // returned from the decoder.
+  // Currently, not set for locally captured video frames.
+  absl::optional<ProcessingTime> processing_time_;
 };
 
 }  // namespace webrtc

modules/video_coding/generic_decoder.cc

@@ -99,11 +99,13 @@ void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage,
   decodedImage.set_packet_infos(frameInfo->packet_infos);
   decodedImage.set_rotation(frameInfo->rotation);
 
-  const int64_t now_ms = _clock->TimeInMilliseconds();
+  const Timestamp now = _clock->CurrentTime();
+  RTC_DCHECK(frameInfo->decodeStart);
   if (!decode_time_ms) {
-    decode_time_ms = now_ms - frameInfo->decodeStartTimeMs;
+    decode_time_ms = (now - *frameInfo->decodeStart).ms();
   }
-  _timing->StopDecodeTimer(*decode_time_ms, now_ms);
+  _timing->StopDecodeTimer(*decode_time_ms, now.ms());
+  decodedImage.set_processing_time({*frameInfo->decodeStart, now});
 
   // Report timing information.
   TimingFrameInfo timing_frame_info;
@@ -147,8 +149,8 @@ void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage,
   }
 
   timing_frame_info.flags = frameInfo->timing.flags;
-  timing_frame_info.decode_start_ms = frameInfo->decodeStartTimeMs;
+  timing_frame_info.decode_start_ms = frameInfo->decodeStart->ms();
-  timing_frame_info.decode_finish_ms = now_ms;
+  timing_frame_info.decode_finish_ms = now.ms();
   timing_frame_info.render_time_ms = frameInfo->renderTimeMs;
   timing_frame_info.rtp_timestamp = decodedImage.timestamp();
   timing_frame_info.receive_start_ms = frameInfo->timing.receive_start_ms;
@@ -210,10 +212,10 @@ int32_t VCMGenericDecoder::InitDecode(const VideoCodec* settings,
   return decoder_->InitDecode(settings, numberOfCores);
 }
 
-int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, int64_t nowMs) {
+int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, Timestamp now) {
   TRACE_EVENT1("webrtc", "VCMGenericDecoder::Decode", "timestamp",
                frame.Timestamp());
-  _frameInfos[_nextFrameInfoIdx].decodeStartTimeMs = nowMs;
+  _frameInfos[_nextFrameInfoIdx].decodeStart = now;
   _frameInfos[_nextFrameInfoIdx].renderTimeMs = frame.RenderTimeMs();
   _frameInfos[_nextFrameInfoIdx].rotation = frame.rotation();
   _frameInfos[_nextFrameInfoIdx].timing = frame.video_timing();

modules/video_coding/generic_decoder.h

@@ -30,14 +30,14 @@ enum { kDecoderFrameMemoryLength = 10 };
 
 struct VCMFrameInformation {
   int64_t renderTimeMs;
-  int64_t decodeStartTimeMs;
+  absl::optional<Timestamp> decodeStart;
   void* userData;
   VideoRotation rotation;
   VideoContentType content_type;
   EncodedImage::Timing timing;
   int64_t ntp_time_ms;
   RtpPacketInfos packet_infos;
-  // ColorSpace is not storred here, as it might be modified by decoders.
+  // ColorSpace is not stored here, as it might be modified by decoders.
 };
 
 class VCMDecodedFrameCallback : public DecodedImageCallback {
@@ -92,7 +92,7 @@ class VCMGenericDecoder {
    *
    * inputVideoBuffer reference to encoded video frame
    */
-  int32_t Decode(const VCMEncodedFrame& inputFrame, int64_t nowMs);
+  int32_t Decode(const VCMEncodedFrame& inputFrame, Timestamp now);
 
   /**
    * Set decode callback. Deregistering while decoding is illegal.

modules/video_coding/generic_decoder_unittest.cc

@@ -93,7 +93,7 @@ TEST_F(GenericDecoderTest, PassesPacketInfos) {
   RtpPacketInfos packet_infos = CreatePacketInfos(3);
   VCMEncodedFrame encoded_frame;
   encoded_frame.SetPacketInfos(packet_infos);
-  generic_decoder_.Decode(encoded_frame, clock_.TimeInMilliseconds());
+  generic_decoder_.Decode(encoded_frame, clock_.CurrentTime());
   absl::optional<VideoFrame> decoded_frame = user_callback_.WaitForFrame(10);
   ASSERT_TRUE(decoded_frame.has_value());
   EXPECT_EQ(decoded_frame->packet_infos().size(), 3U);
@@ -107,7 +107,7 @@ TEST_F(GenericDecoderTest, PassesPacketInfosForDelayedDecoders) {
     // Ensure the original frame is destroyed before the decoding is completed.
     VCMEncodedFrame encoded_frame;
     encoded_frame.SetPacketInfos(packet_infos);
-    generic_decoder_.Decode(encoded_frame, clock_.TimeInMilliseconds());
+    generic_decoder_.Decode(encoded_frame, clock_.CurrentTime());
   }
 
   absl::optional<VideoFrame> decoded_frame = user_callback_.WaitForFrame(200);

modules/video_coding/video_receiver.cc

@@ -259,7 +259,7 @@ int32_t VideoReceiver::Decode(const VCMEncodedFrame& frame) {
   if (decoder == nullptr) {
     return VCM_NO_CODEC_REGISTERED;
   }
-  return decoder->Decode(frame, clock_->TimeInMilliseconds());
+  return decoder->Decode(frame, clock_->CurrentTime());
 }
 
 // Register possible receive codecs, can be called multiple times

modules/video_coding/video_receiver2.cc

@@ -91,7 +91,7 @@ int32_t VideoReceiver2::Decode(const VCMEncodedFrame* frame) {
   if (decoder == nullptr) {
     return VCM_NO_CODEC_REGISTERED;
   }
-  return decoder->Decode(*frame, clock_->TimeInMilliseconds());
+  return decoder->Decode(*frame, clock_->CurrentTime());
 }
 
 // Register possible receive codecs, can be called multiple times