Fix native api in preparation for native_api example.

Add native api conversions for video frames and video renderer. This
also requires some changes to sdk/BUILD to avoid cyclic dependencies.

Bug: webrtc:8832
Change-Id: Ibf21e63bdcae195dcb61d63f9262e6a8dc4fa790
Reviewed-on: https://webrtc-review.googlesource.com/57142
Commit-Queue: Anders Carlsson <andersc@webrtc.org>
Reviewed-by: Kári Helgason <kthelgason@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#22340}

sdk/objc/Framework/Native/src/objc_frame_buffer.h

@@ -39,6 +39,9 @@ class ObjCFrameBuffer : public VideoFrameBuffer {
   int height_;
 };
 
+id<RTCVideoFrameBuffer> ToObjCVideoFrameBuffer(
+    const rtc::scoped_refptr<VideoFrameBuffer>& buffer);
+
 }  // namespace webrtc
 
 #endif  // SDK_OBJC_FRAMEWORK_NATIVE_SRC_OBJC_FRAME_BUFFER_H_

sdk/objc/Framework/Native/src/objc_frame_buffer.mm

@@ -11,6 +11,7 @@
 #include "sdk/objc/Framework/Native/src/objc_frame_buffer.h"
 
 #import "WebRTC/RTCVideoFrameBuffer.h"
+#import "sdk/objc/Framework/Classes/Video/RTCI420Buffer+Private.h"
 
 namespace webrtc {
 
@@ -75,4 +76,12 @@ id<RTCVideoFrameBuffer> ObjCFrameBuffer::wrapped_frame_buffer() const {
   return frame_buffer_;
 }
 
+id<RTCVideoFrameBuffer> ToObjCVideoFrameBuffer(const rtc::scoped_refptr<VideoFrameBuffer>& buffer) {
+  if (buffer->type() == VideoFrameBuffer::Type::kNative) {
+    return static_cast<ObjCFrameBuffer*>(buffer.get())->wrapped_frame_buffer();
+  } else {
+    return [[RTCI420Buffer alloc] initWithFrameBuffer:buffer->ToI420()];
+  }
+}
+
 }  // namespace webrtc

sdk/objc/Framework/Native/src/objc_video_decoder_factory.h

@@ -34,12 +34,11 @@ class ObjCVideoDecoderFactory : public VideoDecoderFactory,
       const SdpVideoFormat& format) override;
 
   // Needed for WebRtcVideoDecoderFactory interface.
-  webrtc::VideoDecoder* CreateVideoDecoderWithParams(
+  VideoDecoder* CreateVideoDecoderWithParams(
       const cricket::VideoCodec& codec,
       cricket::VideoDecoderParams params) override;
-  webrtc::VideoDecoder* CreateVideoDecoder(
-      webrtc::VideoCodecType type) override;
-  void DestroyVideoDecoder(webrtc::VideoDecoder* decoder) override;
+  VideoDecoder* CreateVideoDecoder(VideoCodecType type) override;
+  void DestroyVideoDecoder(VideoDecoder* decoder) override;
 
  private:
   id<RTCVideoDecoderFactory> decoder_factory_;

sdk/objc/Framework/Native/src/objc_video_decoder_factory.mm

@@ -10,14 +10,14 @@
 
 #include "sdk/objc/Framework/Native/src/objc_video_decoder_factory.h"
 
-#import "NSString+StdString.h"
-#import "RTCVideoCodec+Private.h"
-#import "RTCWrappedNativeVideoDecoder.h"
 #import "WebRTC/RTCVideoCodec.h"
 #import "WebRTC/RTCVideoCodecFactory.h"
 #import "WebRTC/RTCVideoCodecH264.h"
 #import "WebRTC/RTCVideoFrame.h"
 #import "WebRTC/RTCVideoFrameBuffer.h"
+#import "sdk/objc/Framework/Classes/Common/NSString+StdString.h"
+#import "sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodec+Private.h"
+#import "sdk/objc/Framework/Classes/PeerConnection/RTCWrappedNativeVideoDecoder.h"
 
 #include "api/video_codecs/sdp_video_format.h"
 #include "api/video_codecs/video_decoder.h"

sdk/objc/Framework/Native/src/objc_video_encoder_factory.h

@@ -36,10 +36,9 @@ class ObjCVideoEncoderFactory : public VideoEncoderFactory,
   CodecInfo QueryVideoEncoder(const SdpVideoFormat& format) const override;
 
   // Needed for WebRtcVideoEncoderFactory interface.
-  webrtc::VideoEncoder* CreateVideoEncoder(
-      const cricket::VideoCodec& codec) override;
+  VideoEncoder* CreateVideoEncoder(const cricket::VideoCodec& codec) override;
   const std::vector<cricket::VideoCodec>& supported_codecs() const override;
-  void DestroyVideoEncoder(webrtc::VideoEncoder* encoder) override;
+  void DestroyVideoEncoder(VideoEncoder* encoder) override;
 
  private:
   id<RTCVideoEncoderFactory> encoder_factory_;

sdk/objc/Framework/Native/src/objc_video_encoder_factory.mm

@@ -12,16 +12,13 @@
 
 #include <string>
 
-#import "NSString+StdString.h"
-#import "RTCI420Buffer+Private.h"
-#import "RTCVideoCodec+Private.h"
-#import "RTCVideoFrame+Private.h"
-#import "RTCWrappedNativeVideoEncoder.h"
 #import "WebRTC/RTCVideoCodec.h"
 #import "WebRTC/RTCVideoCodecFactory.h"
 #import "WebRTC/RTCVideoCodecH264.h"
-#import "WebRTC/RTCVideoFrame.h"
-#import "WebRTC/RTCVideoFrameBuffer.h"
+#import "sdk/objc/Framework/Classes/Common/NSString+StdString.h"
+#import "sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodec+Private.h"
+#import "sdk/objc/Framework/Classes/PeerConnection/RTCWrappedNativeVideoEncoder.h"
+#import "sdk/objc/Framework/Classes/Video/RTCI420Buffer+Private.h"
 
 #include "api/video/video_frame.h"
 #include "api/video_codecs/sdp_video_format.h"
@@ -31,6 +28,7 @@
 #include "modules/video_coding/include/video_error_codes.h"
 #include "rtc_base/logging.h"
 #include "sdk/objc/Framework/Classes/Common/helpers.h"
+#include "sdk/objc/Framework/Native/src/objc_video_frame.h"
 
 namespace webrtc {
 
@@ -92,7 +90,7 @@ class ObjCVideoEncoder : public VideoEncoder {
       [rtcFrameTypes addObject:@(RTCFrameType(frame_types->at(i)))];
     }
 
-    return [encoder_ encode:[[RTCVideoFrame alloc] initWithNativeVideoFrame:frame]
+    return [encoder_ encode:ToObjCVideoFrame(frame)
           codecSpecificInfo:rtcCodecSpecificInfo
                  frameTypes:rtcFrameTypes];
   }

sdk/objc/Framework/Native/src/objc_video_frame.h

@@ -0,0 +1,24 @@
+/*
+ *  Copyright 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef SDK_OBJC_FRAMEWORK_NATIVE_SRC_OBJC_VIDEO_FRAME_H_
+#define SDK_OBJC_FRAMEWORK_NATIVE_SRC_OBJC_VIDEO_FRAME_H_
+
+#import "WebRTC/RTCVideoFrame.h"
+
+#include "api/video/video_frame.h"
+
+namespace webrtc {
+
+RTCVideoFrame* ToObjCVideoFrame(const VideoFrame& frame);
+
+}  // namespace webrtc
+
+#endif  // SDK_OBJC_FRAMEWORK_NATIVE_SRC_OBJC_VIDEO_FRAME_H_

sdk/objc/Framework/Native/src/objc_video_frame.mm

@@ -0,0 +1,28 @@
+/*
+ *  Copyright 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "sdk/objc/Framework/Native/src/objc_video_frame.h"
+
+#include "rtc_base/timeutils.h"
+#include "sdk/objc/Framework/Native/src/objc_frame_buffer.h"
+
+namespace webrtc {
+
+RTCVideoFrame *ToObjCVideoFrame(const VideoFrame &frame) {
+  RTCVideoFrame *videoFrame =
+      [[RTCVideoFrame alloc] initWithBuffer:ToObjCVideoFrameBuffer(frame.video_frame_buffer())
+                                   rotation:RTCVideoRotation(frame.rotation())
+                                timeStampNs:frame.timestamp_us() * rtc::kNumNanosecsPerMicrosec];
+  videoFrame.timeStamp = frame.timestamp();
+
+  return videoFrame;
+}
+
+}  // namespace webrtc

sdk/objc/Framework/Native/src/objc_video_renderer.h

@@ -0,0 +1,36 @@
+/*
+ *  Copyright 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef SDK_OBJC_FRAMEWORK_NATIVE_SRC_OBJC_VIDEO_RENDERER_H_
+#define SDK_OBJC_FRAMEWORK_NATIVE_SRC_OBJC_VIDEO_RENDERER_H_
+
+#import <CoreGraphics/CoreGraphics.h>
+#import <Foundation/Foundation.h>
+
+#include "api/video/video_frame.h"
+#include "api/videosinkinterface.h"
+
+@protocol RTCVideoRenderer;
+
+namespace webrtc {
+
+class ObjCVideoRenderer : public rtc::VideoSinkInterface<VideoFrame> {
+ public:
+  ObjCVideoRenderer(id<RTCVideoRenderer> renderer);
+  void OnFrame(const VideoFrame& nativeVideoFrame) override;
+
+ private:
+  id<RTCVideoRenderer> renderer_;
+  CGSize size_;
+};
+
+}  // namespace webrtc
+
+#endif  // SDK_OBJC_FRAMEWORK_NATIVE_SRC_OBJC_VIDEO_RENDERER_H_

sdk/objc/Framework/Native/src/objc_video_renderer.mm

@@ -0,0 +1,37 @@
+/*
+ *  Copyright 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "sdk/objc/Framework/Native/src/objc_video_renderer.h"
+
+#import "WebRTC/RTCVideoFrame.h"
+#import "WebRTC/RTCVideoRenderer.h"
+
+#include "sdk/objc/Framework/Native/src/objc_video_frame.h"
+
+namespace webrtc {
+
+ObjCVideoRenderer::ObjCVideoRenderer(id<RTCVideoRenderer> renderer)
+    : renderer_(renderer), size_(CGSizeZero) {}
+
+void ObjCVideoRenderer::OnFrame(const VideoFrame& nativeVideoFrame) {
+  RTCVideoFrame* videoFrame = ToObjCVideoFrame(nativeVideoFrame);
+
+  CGSize current_size = (videoFrame.rotation % 180 == 0) ?
+      CGSizeMake(videoFrame.width, videoFrame.height) :
+      CGSizeMake(videoFrame.height, videoFrame.width);
+
+  if (!CGSizeEqualToSize(size_, current_size)) {
+    size_ = current_size;
+    [renderer_ setSize:size_];
+  }
+  [renderer_ renderFrame:videoFrame];
+}
+
+}  // namespace webrtc

sdk/objc/Framework/Native/src/objc_video_track_source.h

@@ -0,0 +1,50 @@
+/*
+ *  Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef SDK_OBJC_FRAMEWORK_CLASSES_VIDEO_OBJCVIDEOTRACKSOURCE_H_
+#define SDK_OBJC_FRAMEWORK_CLASSES_VIDEO_OBJCVIDEOTRACKSOURCE_H_
+
+#include "WebRTC/RTCMacros.h"
+#include "media/base/adaptedvideotracksource.h"
+#include "rtc_base/timestampaligner.h"
+
+RTC_FWD_DECL_OBJC_CLASS(RTCVideoFrame);
+
+namespace webrtc {
+
+class ObjCVideoTrackSource : public rtc::AdaptedVideoTrackSource {
+ public:
+  ObjCVideoTrackSource();
+
+  // This class can not be used for implementing screen casting. Hopefully, this
+  // function will be removed before we add that to iOS/Mac.
+  bool is_screencast() const override { return false; }
+
+  // Indicates that the encoder should denoise video before encoding it.
+  // If it is not set, the default configuration is used which is different
+  // depending on video codec.
+  rtc::Optional<bool> needs_denoising() const override { return false; }
+
+  SourceState state() const override { return SourceState::kLive; }
+
+  bool remote() const override { return false; }
+
+  // Called by RTCVideoSource.
+  void OnCapturedFrame(RTCVideoFrame* frame);
+  void OnOutputFormatRequest(int width, int height, int fps);
+
+ private:
+  rtc::VideoBroadcaster broadcaster_;
+  rtc::TimestampAligner timestamp_aligner_;
+};
+
+}  // namespace webrtc
+
+#endif  // SDK_OBJC_FRAMEWORK_CLASSES_VIDEO_OBJCVIDEOTRACKSOURCE_H_

sdk/objc/Framework/Native/src/objc_video_track_source.mm

@@ -0,0 +1,86 @@
+/*
+ *  Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "sdk/objc/Framework/Native/src/objc_video_track_source.h"
+
+#import "WebRTC/RTCVideoFrame.h"
+#import "WebRTC/RTCVideoFrameBuffer.h"
+
+#include "api/video/i420_buffer.h"
+#include "sdk/objc/Framework/Native/src/objc_frame_buffer.h"
+
+namespace webrtc {
+
+ObjCVideoTrackSource::ObjCVideoTrackSource() {}
+
+void ObjCVideoTrackSource::OnOutputFormatRequest(int width, int height, int fps) {
+  cricket::VideoFormat format(width, height, cricket::VideoFormat::FpsToInterval(fps), 0);
+  video_adapter()->OnOutputFormatRequest(format);
+}
+
+void ObjCVideoTrackSource::OnCapturedFrame(RTCVideoFrame *frame) {
+  const int64_t timestamp_us = frame.timeStampNs / rtc::kNumNanosecsPerMicrosec;
+  const int64_t translated_timestamp_us =
+      timestamp_aligner_.TranslateTimestamp(timestamp_us, rtc::TimeMicros());
+
+  int adapted_width;
+  int adapted_height;
+  int crop_width;
+  int crop_height;
+  int crop_x;
+  int crop_y;
+  if (!AdaptFrame(frame.width,
+                  frame.height,
+                  timestamp_us,
+                  &adapted_width,
+                  &adapted_height,
+                  &crop_width,
+                  &crop_height,
+                  &crop_x,
+                  &crop_y)) {
+    return;
+  }
+
+  rtc::scoped_refptr<VideoFrameBuffer> buffer;
+  if (adapted_width == frame.width && adapted_height == frame.height) {
+    // No adaption - optimized path.
+    buffer = new rtc::RefCountedObject<ObjCFrameBuffer>(frame.buffer);
+  } else if ([frame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) {
+    // Adapted CVPixelBuffer frame.
+    RTCCVPixelBuffer *rtcPixelBuffer = (RTCCVPixelBuffer *)frame.buffer;
+    buffer = new rtc::RefCountedObject<ObjCFrameBuffer>([[RTCCVPixelBuffer alloc]
+        initWithPixelBuffer:rtcPixelBuffer.pixelBuffer
+               adaptedWidth:adapted_width
+              adaptedHeight:adapted_height
+                  cropWidth:crop_width
+                 cropHeight:crop_height
+                      cropX:crop_x + rtcPixelBuffer.cropX
+                      cropY:crop_y + rtcPixelBuffer.cropY]);
+  } else {
+    // Adapted I420 frame.
+    // TODO(magjed): Optimize this I420 path.
+    rtc::scoped_refptr<I420Buffer> i420_buffer = I420Buffer::Create(adapted_width, adapted_height);
+    buffer = new rtc::RefCountedObject<ObjCFrameBuffer>(frame.buffer);
+    i420_buffer->CropAndScaleFrom(*buffer->ToI420(), crop_x, crop_y, crop_width, crop_height);
+    buffer = i420_buffer;
+  }
+
+  // Applying rotation is only supported for legacy reasons and performance is
+  // not critical here.
+  VideoRotation rotation = static_cast<VideoRotation>(frame.rotation);
+  if (apply_rotation() && rotation != kVideoRotation_0) {
+    buffer = I420Buffer::Rotate(*buffer->ToI420(), rotation);
+    rotation = kVideoRotation_0;
+  }
+
+  OnFrame(VideoFrame(buffer, rotation, translated_timestamp_us));
+}
+
+}  // namespace webrtc