MediaStreamInterface: introduce encoded sinks.

This change adds a new type of sink for consuming encoded data from
a video source.

Bug: chromium:1013590
Change-Id: Ia7c4e372190c3d6bc007a0d4deb05c2d1bce58d2
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/159927
Commit-Queue: Markus Handell <handellm@webrtc.org>
Reviewed-by: Per Kjellander <perkj@webrtc.org>
Reviewed-by: Niels Moller <nisse@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#29856}

api/BUILD.gn

@@ -115,6 +115,7 @@ rtc_library("media_stream_interface") {
     "../rtc_base:checks",
     "../rtc_base:refcount",
     "../rtc_base/system:rtc_export",
+    "video:recordable_encoded_frame",
     "video:video_frame",
     "//third_party/abseil-cpp/absl/types:optional",
   ]

api/media_stream_interface.h

@@ -24,6 +24,7 @@
 #include "absl/types/optional.h"
 #include "api/audio_options.h"
 #include "api/scoped_refptr.h"
+#include "api/video/recordable_encoded_frame.h"
 #include "api/video/video_frame.h"
 #include "api/video/video_sink_interface.h"
 #include "api/video/video_source_interface.h"
@@ -135,6 +136,31 @@ class VideoTrackSourceInterface : public MediaSourceInterface,
   // Implementation should avoid blocking.
   virtual bool GetStats(Stats* stats) = 0;
 
+  // Returns true if encoded output can be enabled in the source.
+  // TODO(bugs.webrtc.org/11114): make pure virtual once downstream project
+  // adapts.
+  virtual bool SupportsEncodedOutput() const { return false; }
+
+  // Reliably cause a key frame to be generated in encoded output.
+  // TODO(bugs.webrtc.org/11115): find optimal naming.
+  // TODO(bugs.webrtc.org/11114): make pure virtual once downstream project
+  // adapts.
+  virtual void GenerateKeyFrame() {}
+
+  // Add an encoded video sink to the source and additionally cause
+  // a key frame to be generated from the source. The sink will be
+  // invoked from a decoder queue.
+  // TODO(bugs.webrtc.org/11114): make pure virtual once downstream project
+  // adapts.
+  virtual void AddEncodedSink(
+      rtc::VideoSinkInterface<RecordableEncodedFrame>* sink) {}
+
+  // Removes an encoded video sink from the source.
+  // TODO(bugs.webrtc.org/11114): make pure virtual once downstream project
+  // adapts.
+  virtual void RemoveEncodedSink(
+      rtc::VideoSinkInterface<RecordableEncodedFrame>* sink) {}
+
  protected:
   ~VideoTrackSourceInterface() override = default;
 };

api/video/BUILD.gn

@@ -57,6 +57,23 @@ rtc_library("video_frame") {
   ]
 }
 
+rtc_source_set("recordable_encoded_frame") {
+  visibility = [ "*" ]
+  sources = [
+    "recordable_encoded_frame.h",
+  ]
+
+  deps = [
+    ":encoded_image",
+    ":video_frame",
+    ":video_rtp_headers",
+    "..:array_view",
+    "..:scoped_refptr",
+    "../../rtc_base:refcount",
+    "../units:timestamp",
+  ]
+}
+
 rtc_source_set("video_frame_type") {
   visibility = [ "*" ]
   sources = [

api/video/DEPS

@@ -18,6 +18,10 @@ specific_include_rules = {
     "+rtc_base/memory/aligned_malloc.h",
   ],
 
+  "recordable_encoded_frame\.h": [
+    "+rtc_base/ref_count.h",
+  ],
+
   "video_frame\.h": [
     "+rtc_base/ref_count.h",
   ],

api/video/recordable_encoded_frame.h

@@ -0,0 +1,59 @@
+/*
+ *  Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_VIDEO_RECORDABLE_ENCODED_FRAME_H_
+#define API_VIDEO_RECORDABLE_ENCODED_FRAME_H_
+
+#include "api/array_view.h"
+#include "api/scoped_refptr.h"
+#include "api/units/timestamp.h"
+#include "api/video/color_space.h"
+#include "api/video/encoded_image.h"
+#include "api/video/video_codec_type.h"
+#include "rtc_base/ref_count.h"
+
+namespace webrtc {
+
+// Interface for accessing recordable elements of an encoded frame.
+class RecordableEncodedFrame {
+ public:
+  // Encoded resolution in pixels
+  struct EncodedResolution {
+    unsigned width;
+    unsigned height;
+  };
+
+  virtual ~RecordableEncodedFrame() = default;
+
+  // Provides access to encoded data
+  virtual rtc::scoped_refptr<const EncodedImageBufferInterface> encoded_buffer()
+      const = 0;
+
+  // Optionally returns the colorspace of the encoded frame. This can differ
+  // from the eventually decoded frame's colorspace.
+  virtual absl::optional<webrtc::ColorSpace> color_space() const = 0;
+
+  // Returns the codec of the encoded frame
+  virtual VideoCodecType codec() const = 0;
+
+  // Returns whether the encoded frame is a key frame
+  virtual bool is_key_frame() const = 0;
+
+  // Returns the frame's encoded resolution. May be 0x0 if the frame
+  // doesn't contain resolution information
+  virtual EncodedResolution resolution() const = 0;
+
+  // Returns the computed render time
+  virtual Timestamp render_time() const = 0;
+};
+
+}  // namespace webrtc
+
+#endif  // API_VIDEO_RECORDABLE_ENCODED_FRAME_H_

api/video_track_source_proxy.h

@@ -34,6 +34,14 @@ PROXY_WORKER_METHOD2(void,
 PROXY_WORKER_METHOD1(void, RemoveSink, rtc::VideoSinkInterface<VideoFrame>*)
 PROXY_METHOD1(void, RegisterObserver, ObserverInterface*)
 PROXY_METHOD1(void, UnregisterObserver, ObserverInterface*)
+PROXY_CONSTMETHOD0(bool, SupportsEncodedOutput)
+PROXY_METHOD0(void, GenerateKeyFrame)
+PROXY_WORKER_METHOD1(void,
+                     AddEncodedSink,
+                     rtc::VideoSinkInterface<RecordableEncodedFrame>*)
+PROXY_WORKER_METHOD1(void,
+                     RemoveEncodedSink,
+                     rtc::VideoSinkInterface<RecordableEncodedFrame>*)
 END_PROXY_MAP()
 
 }  // namespace webrtc