VideoStreamDecoderImpl implementation, part 3.

This CL implements the functions related to decoding.

Bug: webrtc:8909
Change-Id: Iefa3c1565a9b9ae93f14992b4a1cca141b7c5193
Reviewed-on: https://webrtc-review.googlesource.com/66403
Reviewed-by: Stefan Holmer <stefan@webrtc.org>
Commit-Queue: Philip Eliasson <philipel@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#22747}

video/video_stream_decoder_impl.cc

@@ -11,6 +11,7 @@
 #include "video/video_stream_decoder_impl.h"
 
 #include "rtc_base/logging.h"
+#include "rtc_base/numerics/mod_ops.h"
 #include "rtc_base/ptr_util.h"
 
 namespace webrtc {
@@ -23,15 +24,24 @@ VideoStreamDecoderImpl::VideoStreamDecoderImpl(
       decoder_factory_(decoder_factory),
       decoder_settings_(std::move(decoder_settings)),
       bookkeeping_queue_("video_stream_decoder_bookkeeping_queue"),
+      decode_thread_(&DecodeLoop,
+                     this,
+                     "video_stream_decoder_decode_thread",
+                     rtc::kHighestPriority),
       jitter_estimator_(Clock::GetRealTimeClock()),
       timing_(Clock::GetRealTimeClock()),
       frame_buffer_(Clock::GetRealTimeClock(),
                     &jitter_estimator_,
                     &timing_,
-                    nullptr) {}
+                    nullptr),
+      next_start_time_index_(0) {
+  decode_start_time_.fill({-1, 0});
+  decode_thread_.Start();
+}
 
 VideoStreamDecoderImpl::~VideoStreamDecoderImpl() {
   frame_buffer_.Stop();
+  decode_thread_.Stop();
 }
 
 void VideoStreamDecoderImpl::OnFrame(
@@ -111,4 +121,94 @@ VideoDecoder* VideoStreamDecoderImpl::GetDecoder(int payload_type) {
   return decoder_.get();
 }
 
+// static
+void VideoStreamDecoderImpl::DecodeLoop(void* ptr) {
+  // TODO(philipel): Remove this and use rtc::Event::kForever when it's
+  //                 supported by the |frame_buffer_|.
+  static constexpr int kForever = 100000000;
+
+  int max_wait_time_ms = kForever;
+  bool keyframe_required = true;
+  auto* vs_decoder = static_cast<VideoStreamDecoderImpl*>(ptr);
+  while (true) {
+    DecodeResult decode_result =
+        vs_decoder->DecodeNextFrame(max_wait_time_ms, keyframe_required);
+
+    switch (decode_result) {
+      case kOk: {
+        max_wait_time_ms = kForever;
+        keyframe_required = false;
+        break;
+      }
+      case kDecodeFailure: {
+        max_wait_time_ms = 0;
+        keyframe_required = true;
+        break;
+      }
+      case kNoFrame: {
+        max_wait_time_ms = kForever;
+        // If we end up here it means that we got a decoding error and there is
+        // no keyframe available in the |frame_buffer_|.
+        vs_decoder->bookkeeping_queue_.PostTask([vs_decoder]() {
+          RTC_DCHECK_RUN_ON(&vs_decoder->bookkeeping_queue_);
+          vs_decoder->callbacks_->OnNonDecodableState();
+        });
+        break;
+      }
+      case kNoDecoder: {
+        max_wait_time_ms = kForever;
+        break;
+      }
+      case kShutdown: {
+        return;
+      }
+    }
+  }
+}
+
+VideoStreamDecoderImpl::DecodeResult VideoStreamDecoderImpl::DecodeNextFrame(
+    int max_wait_time_ms,
+    bool keyframe_required) {
+  std::unique_ptr<video_coding::EncodedFrame> frame;
+  video_coding::FrameBuffer::ReturnReason res =
+      frame_buffer_.NextFrame(max_wait_time_ms, &frame, keyframe_required);
+
+  if (res == video_coding::FrameBuffer::ReturnReason::kStopped)
+    return kShutdown;
+
+  if (frame) {
+    VideoDecoder* decoder = GetDecoder(frame->PayloadType());
+    if (!decoder) {
+      RTC_LOG(LS_WARNING) << "Failed to get decoder, dropping frame ("
+                          << frame->id.picture_id << ":"
+                          << frame->id.spatial_layer << ").";
+      return kNoDecoder;
+    }
+
+    int64_t decode_start_time_ms = rtc::TimeMillis();
+    uint32_t frame_timestamp = frame->timestamp;
+    bookkeeping_queue_.PostTask(
+        [this, decode_start_time_ms, frame_timestamp]() {
+          RTC_DCHECK_RUN_ON(&bookkeeping_queue_);
+          // Saving decode start time this way wont work if we decode spatial
+          // layers sequentially.
+          decode_start_time_[next_start_time_index_] = {frame_timestamp,
+                                                        decode_start_time_ms};
+          next_start_time_index_ =
+              Add<kDecodeTimeMemory>(next_start_time_index_, 1);
+        });
+
+    int32_t decode_result =
+        decoder->Decode(frame->EncodedImage(),
+                        false,    // missing_frame
+                        nullptr,  // rtp fragmentation header
+                        nullptr,  // codec specific info
+                        frame->RenderTimeMs());
+
+    return decode_result == WEBRTC_VIDEO_CODEC_OK ? kOk : kDecodeFailure;
+  }
+
+  return kNoFrame;
+}
+
 }  // namespace webrtc

video/video_stream_decoder_impl.h

@@ -20,6 +20,7 @@
 #include "modules/video_coding/frame_buffer2.h"
 #include "modules/video_coding/jitter_estimator.h"
 #include "modules/video_coding/timing.h"
+#include "rtc_base/platform_thread.h"
 #include "rtc_base/task_queue.h"
 #include "rtc_base/thread_checker.h"
 #include "system_wrappers/include/clock.h"
@@ -39,7 +40,17 @@ class VideoStreamDecoderImpl : public VideoStreamDecoder,
   void OnFrame(std::unique_ptr<video_coding::EncodedFrame> frame) override;
 
  private:
+  enum DecodeResult {
+    kOk,
+    kDecodeFailure,
+    kNoFrame,
+    kNoDecoder,
+    kShutdown,
+  };
+
   VideoDecoder* GetDecoder(int payload_type);
+  static void DecodeLoop(void* ptr);
+  DecodeResult DecodeNextFrame(int max_wait_time_ms, bool keyframe_required);
 
   // Implements DecodedImageCallback interface
   int32_t Decoded(VideoFrame& decodedImage) override;
@@ -59,12 +70,21 @@ class VideoStreamDecoderImpl : public VideoStreamDecoder,
   //  - Synchronize with whatever thread that makes the Decoded callback.
   rtc::TaskQueue bookkeeping_queue_;
 
+  rtc::PlatformThread decode_thread_;
   VCMJitterEstimator jitter_estimator_;
   VCMTiming timing_;
   video_coding::FrameBuffer frame_buffer_;
   video_coding::VideoLayerFrameId last_continuous_id_;
   rtc::Optional<int> current_payload_type_;
   std::unique_ptr<VideoDecoder> decoder_;
+
+  // Keep track of the |decode_start_time_| of the last |kDecodeTimeMemory|
+  // number of frames. The |decode_start_time_| array contain
+  // <frame timestamp> --> <decode start time> pairs.
+  static constexpr int kDecodeTimeMemory = 8;
+  std::array<std::pair<int64_t, int64_t>, kDecodeTimeMemory> decode_start_time_
+      RTC_GUARDED_BY(bookkeeping_queue_);
+  int next_start_time_index_ RTC_GUARDED_BY(bookkeeping_queue_);
 };
 
 }  // namespace webrtc