diff --git a/webrtc/api/video/i420_buffer.cc b/webrtc/api/video/i420_buffer.cc index 031b15940a..81f5789837 100644 --- a/webrtc/api/video/i420_buffer.cc +++ b/webrtc/api/video/i420_buffer.cc @@ -180,8 +180,7 @@ void* I420Buffer::native_handle() const { } rtc::scoped_refptr I420Buffer::NativeToI420Buffer() { - RTC_NOTREACHED(); - return nullptr; + return this; } uint8_t* I420Buffer::MutableDataY() { diff --git a/webrtc/sdk/BUILD.gn b/webrtc/sdk/BUILD.gn index a789efb57e..5e89d54cfb 100644 --- a/webrtc/sdk/BUILD.gn +++ b/webrtc/sdk/BUILD.gn @@ -148,6 +148,7 @@ if (is_ios || is_mac) { "objc/Framework/Classes/RTCShader+Private.h", "objc/Framework/Classes/RTCShader.h", "objc/Framework/Classes/RTCShader.mm", + "objc/Framework/Classes/RTCVideoCapturer.m", "objc/Framework/Classes/RTCVideoFrame+Private.h", "objc/Framework/Classes/RTCVideoFrame.mm", "objc/Framework/Classes/RTCVideoRendererAdapter+Private.h", @@ -161,6 +162,8 @@ if (is_ios || is_mac) { "objc/Framework/Classes/avfoundationformatmapper.mm", "objc/Framework/Classes/avfoundationvideocapturer.h", "objc/Framework/Classes/avfoundationvideocapturer.mm", + "objc/Framework/Classes/objcvideotracksource.h", + "objc/Framework/Classes/objcvideotracksource.mm", "objc/Framework/Classes/videotoolboxvideocodecfactory.cc", "objc/Framework/Classes/videotoolboxvideocodecfactory.h", "objc/Framework/Headers/WebRTC/RTCAVFoundationVideoSource.h", @@ -184,6 +187,7 @@ if (is_ios || is_mac) { "objc/Framework/Headers/WebRTC/RTCRtpReceiver.h", "objc/Framework/Headers/WebRTC/RTCRtpSender.h", "objc/Framework/Headers/WebRTC/RTCSessionDescription.h", + "objc/Framework/Headers/WebRTC/RTCVideoCapturer.h", "objc/Framework/Headers/WebRTC/RTCVideoFrame.h", "objc/Framework/Headers/WebRTC/RTCVideoRenderer.h", "objc/Framework/Headers/WebRTC/RTCVideoSource.h", diff --git a/webrtc/sdk/objc/Framework/Classes/RTCPeerConnectionFactory.mm b/webrtc/sdk/objc/Framework/Classes/RTCPeerConnectionFactory.mm index 4f542a9fcc..9759269357 100644 --- a/webrtc/sdk/objc/Framework/Classes/RTCPeerConnectionFactory.mm +++ b/webrtc/sdk/objc/Framework/Classes/RTCPeerConnectionFactory.mm @@ -21,6 +21,7 @@ #import "RTCVideoTrack+Private.h" #import "WebRTC/RTCLogging.h" +#include "objcvideotracksource.h" #include "videotoolboxvideocodecfactory.h" @implementation RTCPeerConnectionFactory { @@ -87,6 +88,12 @@ constraints:constraints]; } +- (RTCVideoSource *)videoSource { + rtc::scoped_refptr objc_video_track_source( + new rtc::RefCountedObject()); + return [[RTCVideoSource alloc] initWithNativeVideoSource:objc_video_track_source]; +} + - (RTCVideoTrack *)videoTrackWithSource:(RTCVideoSource *)source trackId:(NSString *)trackId { return [[RTCVideoTrack alloc] initWithFactory:self diff --git a/webrtc/sdk/objc/Framework/Classes/RTCVideoCapturer.m b/webrtc/sdk/objc/Framework/Classes/RTCVideoCapturer.m new file mode 100644 index 0000000000..e0a307b19e --- /dev/null +++ b/webrtc/sdk/objc/Framework/Classes/RTCVideoCapturer.m @@ -0,0 +1,28 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "WebRTC/RTCVideoCapturer.h" + +@implementation RTCVideoCapturer { + __weak id _delegate; +} + +- (instancetype)initWithDelegate:(id)delegate { + if (self = [super init]) { + _delegate = delegate; + } + return self; +} + +- (id)delegate { + return _delegate; +} + +@end diff --git a/webrtc/sdk/objc/Framework/Classes/RTCVideoFrame+Private.h b/webrtc/sdk/objc/Framework/Classes/RTCVideoFrame+Private.h index a7a40386cc..3b36f5b008 100644 --- a/webrtc/sdk/objc/Framework/Classes/RTCVideoFrame+Private.h +++ b/webrtc/sdk/objc/Framework/Classes/RTCVideoFrame+Private.h @@ -16,6 +16,8 @@ NS_ASSUME_NONNULL_BEGIN @interface RTCVideoFrame () +@property(nonatomic, readonly) rtc::scoped_refptr videoBuffer; + - (instancetype)initWithVideoBuffer: (rtc::scoped_refptr)videoBuffer rotation:(RTCVideoRotation)rotation diff --git a/webrtc/sdk/objc/Framework/Classes/RTCVideoFrame.mm b/webrtc/sdk/objc/Framework/Classes/RTCVideoFrame.mm index ea603b9cce..a4eaefb853 100644 --- a/webrtc/sdk/objc/Framework/Classes/RTCVideoFrame.mm +++ b/webrtc/sdk/objc/Framework/Classes/RTCVideoFrame.mm @@ -113,4 +113,8 @@ return self; } +- (rtc::scoped_refptr)videoBuffer { + return _videoBuffer; +} + @end diff --git a/webrtc/sdk/objc/Framework/Classes/RTCVideoSource.mm b/webrtc/sdk/objc/Framework/Classes/RTCVideoSource.mm index 83a8b79ebe..6748580d6d 100644 --- a/webrtc/sdk/objc/Framework/Classes/RTCVideoSource.mm +++ b/webrtc/sdk/objc/Framework/Classes/RTCVideoSource.mm @@ -11,7 +11,11 @@ #import "RTCVideoSource+Private.h" #include "webrtc/base/checks.h" +#include "webrtc/sdk/objc/Framework/Classes/objcvideotracksource.h" +// TODO(magjed): Refactor this class and target ObjcVideoTrackSource only once +// RTCAVFoundationVideoSource is gone. See http://crbug/webrtc/7177 for more +// info. @implementation RTCVideoSource { rtc::scoped_refptr _nativeVideoSource; } @@ -38,6 +42,15 @@ return [NSString stringWithFormat:@"RTCVideoSource( %p ): %@", self, stateString]; } +- (void)capturer:(RTCVideoCapturer *)capturer didCaptureVideoFrame:(RTCVideoFrame *)frame { + static_cast(_nativeVideoSource.get())->OnCapturedFrame(frame); +} + +- (void)adaptOutputFormatToWidth:(int)width height:(int)height fps:(int)fps { + static_cast(_nativeVideoSource.get()) + ->OnOutputFormatRequest(width, height, fps); +} + #pragma mark - Private - (rtc::scoped_refptr)nativeVideoSource { diff --git a/webrtc/sdk/objc/Framework/Classes/objcvideotracksource.h b/webrtc/sdk/objc/Framework/Classes/objcvideotracksource.h new file mode 100644 index 0000000000..0b1e4bafdf --- /dev/null +++ b/webrtc/sdk/objc/Framework/Classes/objcvideotracksource.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_OBJCVIDEOTRACKSOURCE_H_ +#define WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_OBJCVIDEOTRACKSOURCE_H_ + +#import + +#include "webrtc/base/timestampaligner.h" +#include "webrtc/media/base/adaptedvideotracksource.h" + +namespace webrtc { + +class ObjcVideoTrackSource : public rtc::AdaptedVideoTrackSource { + public: + ObjcVideoTrackSource(); + + // This class can not be used for implementing screen casting. Hopefully, this + // function will be removed before we add that to iOS/Mac. + bool is_screencast() const override { return false; } + + // Indicates that the encoder should denoise video before encoding it. + // If it is not set, the default configuration is used which is different + // depending on video codec. + rtc::Optional needs_denoising() const override { + return rtc::Optional(false); + } + + SourceState state() const override { return SourceState::kLive; } + + bool remote() const override { return false; } + + // Called by RTCVideoSource. + void OnCapturedFrame(RTCVideoFrame* frame); + void OnOutputFormatRequest(int width, int height, int fps); + + private: + rtc::VideoBroadcaster broadcaster_; + rtc::TimestampAligner timestamp_aligner_; +}; + +} // namespace webrtc + +#endif // WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_OBJCVIDEOTRACKSOURCE_H_ diff --git a/webrtc/sdk/objc/Framework/Classes/objcvideotracksource.mm b/webrtc/sdk/objc/Framework/Classes/objcvideotracksource.mm new file mode 100644 index 0000000000..942171be29 --- /dev/null +++ b/webrtc/sdk/objc/Framework/Classes/objcvideotracksource.mm @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/sdk/objc/Framework/Classes/objcvideotracksource.h" + +#import "RTCVideoFrame+Private.h" + +#include "webrtc/common_video/include/corevideo_frame_buffer.h" + +namespace webrtc { + +ObjcVideoTrackSource::ObjcVideoTrackSource() {} + +void ObjcVideoTrackSource::OnOutputFormatRequest(int width, int height, int fps) { + cricket::VideoFormat format(width, height, cricket::VideoFormat::FpsToInterval(fps), 0); + video_adapter()->OnOutputFormatRequest(format); +} + +void ObjcVideoTrackSource::OnCapturedFrame(RTCVideoFrame* frame) { + const int64_t timestamp_us = frame.timeStampNs / rtc::kNumNanosecsPerMicrosec; + const int64_t translated_timestamp_us = + timestamp_aligner_.TranslateTimestamp(timestamp_us, rtc::TimeMicros()); + + int adapted_width; + int adapted_height; + int crop_width; + int crop_height; + int crop_x; + int crop_y; + if (!AdaptFrame(frame.width, frame.height, timestamp_us, &adapted_width, &adapted_height, + &crop_width, &crop_height, &crop_x, &crop_y)) { + return; + } + + rtc::scoped_refptr buffer; + if (adapted_width == frame.width && adapted_height == frame.height) { + // No adaption - optimized path. + buffer = frame.videoBuffer; + } else if (frame.nativeHandle) { + // Adapted CVPixelBuffer frame. + buffer = new rtc::RefCountedObject( + static_cast(frame.nativeHandle), adapted_width, adapted_height, + crop_width, crop_height, crop_x, crop_y); + } else { + // Adapted I420 frame. + // TODO(magjed): Optimize this I420 path. + rtc::scoped_refptr i420_buffer = I420Buffer::Create(adapted_width, adapted_height); + i420_buffer->CropAndScaleFrom(*frame.videoBuffer, crop_x, crop_y, crop_width, crop_height); + buffer = i420_buffer; + } + + // Applying rotation is only supported for legacy reasons and performance is + // not critical here. + webrtc::VideoRotation rotation = static_cast(frame.rotation); + if (apply_rotation() && rotation != kVideoRotation_0) { + buffer = I420Buffer::Rotate(buffer->NativeToI420Buffer(), rotation); + rotation = kVideoRotation_0; + } + + OnFrame(webrtc::VideoFrame(buffer, rotation, translated_timestamp_us)); +} + +} // namespace webrtc diff --git a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCPeerConnectionFactory.h b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCPeerConnectionFactory.h index 2088163eec..8b19dd679a 100644 --- a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCPeerConnectionFactory.h +++ b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCPeerConnectionFactory.h @@ -46,6 +46,8 @@ RTC_EXPORT - (RTCAVFoundationVideoSource *)avFoundationVideoSourceWithConstraints: (nullable RTCMediaConstraints *)constraints; +- (RTCVideoSource *)videoSource; + /** Initialize an RTCVideoTrack with a source and an id. */ - (RTCVideoTrack *)videoTrackWithSource:(RTCVideoSource *)source trackId:(NSString *)trackId; diff --git a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoCapturer.h b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoCapturer.h new file mode 100644 index 0000000000..47d5a47acf --- /dev/null +++ b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoCapturer.h @@ -0,0 +1,28 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import + +NS_ASSUME_NONNULL_BEGIN + +@class RTCVideoCapturer; + +RTC_EXPORT + +@protocol RTCVideoCapturerDelegate +- (void)capturer:(RTCVideoCapturer *)capturer didCaptureVideoFrame:(RTCVideoFrame *)frame; +@end + +@interface RTCVideoCapturer : NSObject +- (instancetype)initWithDelegate:(id)delegate; +@property(nonatomic, readonly, weak) id delegate; +@end + +NS_ASSUME_NONNULL_END diff --git a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoSource.h b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoSource.h index 96bb6f3699..b5be132bcb 100644 --- a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoSource.h +++ b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoSource.h @@ -12,14 +12,29 @@ #import #import +#import NS_ASSUME_NONNULL_BEGIN RTC_EXPORT -@interface RTCVideoSource : RTCMediaSource + +@interface RTCVideoSource : RTCMediaSource - (instancetype)init NS_UNAVAILABLE; +// RTCVideoCapturerDelegate protocol implementation. +- (void)capturer:(RTCVideoCapturer*)capturer didCaptureVideoFrame:(RTCVideoFrame*)frame; + +/** + * Calling this function will cause frames to be scaled down to the + * requested resolution. Also, frames will be cropped to match the + * requested aspect ratio, and frames will be dropped to match the + * requested fps. The requested aspect ratio is orientation agnostic and + * will be adjusted to maintain the input orientation, so it doesn't + * matter if e.g. 1280x720 or 720x1280 is requested. + */ +- (void)adaptOutputFormatToWidth:(int)width height:(int)height fps:(int)fps; + @end NS_ASSUME_NONNULL_END