iOS: Add new RTCVideoSource interface

The new RTCVideoSource interface can be used by custom implementations of RTCVideoCapturer.

BUG=webrtc:7177
TBR=tommi

Review-Url: https://codereview.webrtc.org/2745193002
Cr-Commit-Position: refs/heads/master@{#17409}

webrtc/api/video/i420_buffer.cc

@@ -180,8 +180,7 @@ void* I420Buffer::native_handle() const {
 }
 
 rtc::scoped_refptr<VideoFrameBuffer> I420Buffer::NativeToI420Buffer() {
-  RTC_NOTREACHED();
+  return this;
-  return nullptr;
 }
 
 uint8_t* I420Buffer::MutableDataY() {

webrtc/sdk/BUILD.gn

@@ -148,6 +148,7 @@ if (is_ios || is_mac) {
       "objc/Framework/Classes/RTCShader+Private.h",
       "objc/Framework/Classes/RTCShader.h",
       "objc/Framework/Classes/RTCShader.mm",
+      "objc/Framework/Classes/RTCVideoCapturer.m",
       "objc/Framework/Classes/RTCVideoFrame+Private.h",
       "objc/Framework/Classes/RTCVideoFrame.mm",
       "objc/Framework/Classes/RTCVideoRendererAdapter+Private.h",
@@ -161,6 +162,8 @@ if (is_ios || is_mac) {
       "objc/Framework/Classes/avfoundationformatmapper.mm",
       "objc/Framework/Classes/avfoundationvideocapturer.h",
       "objc/Framework/Classes/avfoundationvideocapturer.mm",
+      "objc/Framework/Classes/objcvideotracksource.h",
+      "objc/Framework/Classes/objcvideotracksource.mm",
       "objc/Framework/Classes/videotoolboxvideocodecfactory.cc",
       "objc/Framework/Classes/videotoolboxvideocodecfactory.h",
       "objc/Framework/Headers/WebRTC/RTCAVFoundationVideoSource.h",
@@ -184,6 +187,7 @@ if (is_ios || is_mac) {
       "objc/Framework/Headers/WebRTC/RTCRtpReceiver.h",
       "objc/Framework/Headers/WebRTC/RTCRtpSender.h",
       "objc/Framework/Headers/WebRTC/RTCSessionDescription.h",
+      "objc/Framework/Headers/WebRTC/RTCVideoCapturer.h",
       "objc/Framework/Headers/WebRTC/RTCVideoFrame.h",
       "objc/Framework/Headers/WebRTC/RTCVideoRenderer.h",
       "objc/Framework/Headers/WebRTC/RTCVideoSource.h",

webrtc/sdk/objc/Framework/Classes/RTCPeerConnectionFactory.mm

@@ -21,6 +21,7 @@
 #import "RTCVideoTrack+Private.h"
 #import "WebRTC/RTCLogging.h"
 
+#include "objcvideotracksource.h"
 #include "videotoolboxvideocodecfactory.h"
 
 @implementation RTCPeerConnectionFactory {
@@ -87,6 +88,12 @@
                                                  constraints:constraints];
 }
 
+- (RTCVideoSource *)videoSource {
+  rtc::scoped_refptr<webrtc::ObjcVideoTrackSource> objc_video_track_source(
+      new rtc::RefCountedObject<webrtc::ObjcVideoTrackSource>());
+  return [[RTCVideoSource alloc] initWithNativeVideoSource:objc_video_track_source];
+}
+
 - (RTCVideoTrack *)videoTrackWithSource:(RTCVideoSource *)source
                                 trackId:(NSString *)trackId {
   return [[RTCVideoTrack alloc] initWithFactory:self

webrtc/sdk/objc/Framework/Classes/RTCVideoCapturer.m

@@ -0,0 +1,28 @@
+/*
+ *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#import "WebRTC/RTCVideoCapturer.h"
+
+@implementation RTCVideoCapturer {
+  __weak id<RTCVideoCapturerDelegate> _delegate;
+}
+
+- (instancetype)initWithDelegate:(id<RTCVideoCapturerDelegate>)delegate {
+  if (self = [super init]) {
+    _delegate = delegate;
+  }
+  return self;
+}
+
+- (id<RTCVideoCapturerDelegate>)delegate {
+  return _delegate;
+}
+
+@end

webrtc/sdk/objc/Framework/Classes/RTCVideoFrame+Private.h

@@ -16,6 +16,8 @@ NS_ASSUME_NONNULL_BEGIN
 
 @interface RTCVideoFrame ()
 
+@property(nonatomic, readonly) rtc::scoped_refptr<webrtc::VideoFrameBuffer> videoBuffer;
+
 - (instancetype)initWithVideoBuffer:
                     (rtc::scoped_refptr<webrtc::VideoFrameBuffer>)videoBuffer
                            rotation:(RTCVideoRotation)rotation

webrtc/sdk/objc/Framework/Classes/RTCVideoFrame.mm

@@ -113,4 +113,8 @@
   return self;
 }
 
+- (rtc::scoped_refptr<webrtc::VideoFrameBuffer>)videoBuffer {
+  return _videoBuffer;
+}
+
 @end

webrtc/sdk/objc/Framework/Classes/RTCVideoSource.mm

@@ -11,7 +11,11 @@
 #import "RTCVideoSource+Private.h"
 
 #include "webrtc/base/checks.h"
+#include "webrtc/sdk/objc/Framework/Classes/objcvideotracksource.h"
 
+// TODO(magjed): Refactor this class and target ObjcVideoTrackSource only once
+// RTCAVFoundationVideoSource is gone. See http://crbug/webrtc/7177 for more
+// info.
 @implementation RTCVideoSource {
   rtc::scoped_refptr<webrtc::VideoTrackSourceInterface> _nativeVideoSource;
 }
@@ -38,6 +42,15 @@
   return [NSString stringWithFormat:@"RTCVideoSource( %p ): %@", self, stateString];
 }
 
+- (void)capturer:(RTCVideoCapturer *)capturer didCaptureVideoFrame:(RTCVideoFrame *)frame {
+  static_cast<webrtc::ObjcVideoTrackSource *>(_nativeVideoSource.get())->OnCapturedFrame(frame);
+}
+
+- (void)adaptOutputFormatToWidth:(int)width height:(int)height fps:(int)fps {
+  static_cast<webrtc::ObjcVideoTrackSource *>(_nativeVideoSource.get())
+      ->OnOutputFormatRequest(width, height, fps);
+}
+
 #pragma mark - Private
 
 - (rtc::scoped_refptr<webrtc::VideoTrackSourceInterface>)nativeVideoSource {

webrtc/sdk/objc/Framework/Classes/objcvideotracksource.h

@@ -0,0 +1,51 @@
+/*
+ *  Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_OBJCVIDEOTRACKSOURCE_H_
+#define WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_OBJCVIDEOTRACKSOURCE_H_
+
+#import <WebRTC/RTCVideoFrame.h>
+
+#include "webrtc/base/timestampaligner.h"
+#include "webrtc/media/base/adaptedvideotracksource.h"
+
+namespace webrtc {
+
+class ObjcVideoTrackSource : public rtc::AdaptedVideoTrackSource {
+ public:
+  ObjcVideoTrackSource();
+
+  // This class can not be used for implementing screen casting. Hopefully, this
+  // function will be removed before we add that to iOS/Mac.
+  bool is_screencast() const override { return false; }
+
+  // Indicates that the encoder should denoise video before encoding it.
+  // If it is not set, the default configuration is used which is different
+  // depending on video codec.
+  rtc::Optional<bool> needs_denoising() const override {
+    return rtc::Optional<bool>(false);
+  }
+
+  SourceState state() const override { return SourceState::kLive; }
+
+  bool remote() const override { return false; }
+
+  // Called by RTCVideoSource.
+  void OnCapturedFrame(RTCVideoFrame* frame);
+  void OnOutputFormatRequest(int width, int height, int fps);
+
+ private:
+  rtc::VideoBroadcaster broadcaster_;
+  rtc::TimestampAligner timestamp_aligner_;
+};
+
+}  // namespace webrtc
+
+#endif  // WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_OBJCVIDEOTRACKSOURCE_H_

webrtc/sdk/objc/Framework/Classes/objcvideotracksource.mm

@@ -0,0 +1,70 @@
+/*
+ *  Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/sdk/objc/Framework/Classes/objcvideotracksource.h"
+
+#import "RTCVideoFrame+Private.h"
+
+#include "webrtc/common_video/include/corevideo_frame_buffer.h"
+
+namespace webrtc {
+
+ObjcVideoTrackSource::ObjcVideoTrackSource() {}
+
+void ObjcVideoTrackSource::OnOutputFormatRequest(int width, int height, int fps) {
+  cricket::VideoFormat format(width, height, cricket::VideoFormat::FpsToInterval(fps), 0);
+  video_adapter()->OnOutputFormatRequest(format);
+}
+
+void ObjcVideoTrackSource::OnCapturedFrame(RTCVideoFrame* frame) {
+  const int64_t timestamp_us = frame.timeStampNs / rtc::kNumNanosecsPerMicrosec;
+  const int64_t translated_timestamp_us =
+      timestamp_aligner_.TranslateTimestamp(timestamp_us, rtc::TimeMicros());
+
+  int adapted_width;
+  int adapted_height;
+  int crop_width;
+  int crop_height;
+  int crop_x;
+  int crop_y;
+  if (!AdaptFrame(frame.width, frame.height, timestamp_us, &adapted_width, &adapted_height,
+                  &crop_width, &crop_height, &crop_x, &crop_y)) {
+    return;
+  }
+
+  rtc::scoped_refptr<VideoFrameBuffer> buffer;
+  if (adapted_width == frame.width && adapted_height == frame.height) {
+    // No adaption - optimized path.
+    buffer = frame.videoBuffer;
+  } else if (frame.nativeHandle) {
+    // Adapted CVPixelBuffer frame.
+    buffer = new rtc::RefCountedObject<CoreVideoFrameBuffer>(
+        static_cast<CVPixelBufferRef>(frame.nativeHandle), adapted_width, adapted_height,
+        crop_width, crop_height, crop_x, crop_y);
+  } else {
+    // Adapted I420 frame.
+    // TODO(magjed): Optimize this I420 path.
+    rtc::scoped_refptr<I420Buffer> i420_buffer = I420Buffer::Create(adapted_width, adapted_height);
+    i420_buffer->CropAndScaleFrom(*frame.videoBuffer, crop_x, crop_y, crop_width, crop_height);
+    buffer = i420_buffer;
+  }
+
+  // Applying rotation is only supported for legacy reasons and performance is
+  // not critical here.
+  webrtc::VideoRotation rotation = static_cast<webrtc::VideoRotation>(frame.rotation);
+  if (apply_rotation() && rotation != kVideoRotation_0) {
+    buffer = I420Buffer::Rotate(buffer->NativeToI420Buffer(), rotation);
+    rotation = kVideoRotation_0;
+  }
+
+  OnFrame(webrtc::VideoFrame(buffer, rotation, translated_timestamp_us));
+}
+
+}  // namespace webrtc

webrtc/sdk/objc/Framework/Headers/WebRTC/RTCPeerConnectionFactory.h

@@ -46,6 +46,8 @@ RTC_EXPORT
 - (RTCAVFoundationVideoSource *)avFoundationVideoSourceWithConstraints:
     (nullable RTCMediaConstraints *)constraints;
 
+- (RTCVideoSource *)videoSource;
+
 /** Initialize an RTCVideoTrack with a source and an id. */
 - (RTCVideoTrack *)videoTrackWithSource:(RTCVideoSource *)source
                                 trackId:(NSString *)trackId;

webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoCapturer.h

@@ -0,0 +1,28 @@
+/*
+ *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#import <WebRTC/RTCVideoFrame.h>
+
+NS_ASSUME_NONNULL_BEGIN
+
+@class RTCVideoCapturer;
+
+RTC_EXPORT
+
+@protocol RTCVideoCapturerDelegate
+- (void)capturer:(RTCVideoCapturer *)capturer didCaptureVideoFrame:(RTCVideoFrame *)frame;
+@end
+
+@interface RTCVideoCapturer : NSObject
+- (instancetype)initWithDelegate:(id<RTCVideoCapturerDelegate>)delegate;
+@property(nonatomic, readonly, weak) id<RTCVideoCapturerDelegate> delegate;
+@end
+
+NS_ASSUME_NONNULL_END

webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoSource.h

@@ -12,14 +12,29 @@
 
 #import <WebRTC/RTCMacros.h>
 #import <WebRTC/RTCMediaSource.h>
+#import <WebRTC/RTCVideoCapturer.h>
 
 NS_ASSUME_NONNULL_BEGIN
 
 RTC_EXPORT
-@interface RTCVideoSource : RTCMediaSource
+
+@interface RTCVideoSource : RTCMediaSource<RTCVideoCapturerDelegate>
 
 - (instancetype)init NS_UNAVAILABLE;
 
+// RTCVideoCapturerDelegate protocol implementation.
+- (void)capturer:(RTCVideoCapturer*)capturer didCaptureVideoFrame:(RTCVideoFrame*)frame;
+
+/**
+ * Calling this function will cause frames to be scaled down to the
+ * requested resolution. Also, frames will be cropped to match the
+ * requested aspect ratio, and frames will be dropped to match the
+ * requested fps. The requested aspect ratio is orientation agnostic and
+ * will be adjusted to maintain the input orientation, so it doesn't
+ * matter if e.g. 1280x720 or 720x1280 is requested.
+ */
+- (void)adaptOutputFormatToWidth:(int)width height:(int)height fps:(int)fps;
+
 @end
 
 NS_ASSUME_NONNULL_END