Injectable software video codecs in Obj-C.

When injecting video codec factories in the Obj-C SDK, use the new
peer connection API that uses webrtc::Video{De,En}CoderFactory classes
and does not automatically add internal software codecs. Instead the
injected factory can support internal VP8 and VP9 codecs through the
included Obj-C classes RTCVideo{De,En}coderVP{8,9}.

When not explicitly injecting any video codec factory, the old code
path is still used and injects only H264 as an external codec and
the internal codec factory is used.

Bug: webrtc:7925
Change-Id: I657d30dfde71da9c0be341e213ab9f97a04caa58
Reviewed-on: https://webrtc-review.googlesource.com/3620
Commit-Queue: Anders Carlsson <andersc@webrtc.org>
Reviewed-by: Magnus Jedvert <magjed@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#20175}

sdk/objc/Framework/Classes/VideoToolbox/RTCVideoEncoderH264.mm

@@ -169,9 +169,9 @@ void compressionOutputCallback(void *encoder,
 // specific VideoToolbox profile for the specified level, AutoLevel will be
 // returned. The user must initialize the encoder with a resolution and
 // framerate conforming to the selected H264 level regardless.
-CFStringRef ExtractProfile(const cricket::VideoCodec &codec) {
+CFStringRef ExtractProfile(webrtc::SdpVideoFormat videoFormat) {
   const rtc::Optional<webrtc::H264::ProfileLevelId> profile_level_id =
-      webrtc::H264::ParseSdpProfileLevelId(codec.params);
+      webrtc::H264::ParseSdpProfileLevelId(videoFormat.parameters);
   RTC_DCHECK(profile_level_id);
   switch (profile_level_id->profile) {
     case webrtc::H264::kProfileConstrainedBaseline:
@@ -302,7 +302,7 @@ CFStringRef ExtractProfile(const cricket::VideoCodec &codec) {
     _bitrateAdjuster.reset(new webrtc::BitrateAdjuster(
         webrtc::Clock::GetRealTimeClock(), .5, .95));
     _packetizationMode = RTCH264PacketizationModeNonInterleaved;
-    _profile = ExtractProfile([codecInfo nativeVideoCodec]);
+    _profile = ExtractProfile([codecInfo nativeSdpVideoFormat]);
     LOG(LS_INFO) << "Using profile " << CFStringToString(_profile);
     RTC_CHECK([codecInfo.name isEqualToString:@"H264"]);
 

sdk/objc/Framework/Classes/VideoToolbox/objc_video_decoder_factory.h

@@ -11,6 +11,7 @@
 #ifndef SDK_OBJC_FRAMEWORK_CLASSES_PEERCONNECTION_OBJC_VIDEO_DECODER_FACTORY_H_
 #define SDK_OBJC_FRAMEWORK_CLASSES_PEERCONNECTION_OBJC_VIDEO_DECODER_FACTORY_H_
 
+#include "api/video_codecs/video_decoder_factory.h"
 #include "media/base/codec.h"
 #include "media/engine/webrtcvideodecoderfactory.h"
 
@@ -18,22 +19,30 @@
 
 namespace webrtc {
 
-class ObjCVideoDecoderFactory : public cricket::WebRtcVideoDecoderFactory {
+// TODO(andersc): Remove the inheritance from cricket::WebRtcVideoDecoderFactory
+// when the legacy path in [RTCPeerConnectionFactory init] is no longer needed.
+class ObjCVideoDecoderFactory : public VideoDecoderFactory,
+                                public cricket::WebRtcVideoDecoderFactory {
  public:
   explicit ObjCVideoDecoderFactory(id<RTCVideoDecoderFactory>);
   ~ObjCVideoDecoderFactory();
 
   id<RTCVideoDecoderFactory> wrapped_decoder_factory() const;
 
-  VideoDecoder* CreateVideoDecoderWithParams(
+  std::vector<SdpVideoFormat> GetSupportedFormats() const override;
+  std::unique_ptr<VideoDecoder> CreateVideoDecoder(
+      const SdpVideoFormat& format) override;
+
+  // Needed for WebRtcVideoDecoderFactory interface.
+  webrtc::VideoDecoder* CreateVideoDecoderWithParams(
       const cricket::VideoCodec& codec,
       cricket::VideoDecoderParams params) override;
-
+  webrtc::VideoDecoder* CreateVideoDecoder(
+      webrtc::VideoCodecType type) override;
   void DestroyVideoDecoder(webrtc::VideoDecoder* decoder) override;
 
  private:
   id<RTCVideoDecoderFactory> decoder_factory_;
-  std::vector<cricket::VideoCodec> supported_codecs_;
 };
 
 }  // namespace webrtc

sdk/objc/Framework/Classes/VideoToolbox/objc_video_decoder_factory.mm

@@ -12,12 +12,14 @@
 
 #import "NSString+StdString.h"
 #import "RTCVideoCodec+Private.h"
+#import "RTCWrappedNativeVideoDecoder.h"
 #import "WebRTC/RTCVideoCodec.h"
 #import "WebRTC/RTCVideoCodecFactory.h"
 #import "WebRTC/RTCVideoCodecH264.h"
 #import "WebRTC/RTCVideoFrame.h"
 #import "WebRTC/RTCVideoFrameBuffer.h"
 
+#include "api/video_codecs/sdp_video_format.h"
 #include "api/video_codecs/video_decoder.h"
 #include "modules/include/module_common_types.h"
 #include "modules/video_coding/include/video_codec_interface.h"
@@ -103,19 +105,47 @@ id<RTCVideoDecoderFactory> ObjCVideoDecoderFactory::wrapped_decoder_factory() co
   return decoder_factory_;
 }
 
-VideoDecoder *ObjCVideoDecoderFactory::CreateVideoDecoderWithParams(
-    const cricket::VideoCodec &codec, cricket::VideoDecoderParams params) {
-  NSString *codecName = [NSString stringWithUTF8String:codec.name.c_str()];
+std::unique_ptr<VideoDecoder> ObjCVideoDecoderFactory::CreateVideoDecoder(
+    const SdpVideoFormat &format) {
+  NSString *codecName = [NSString stringWithUTF8String:format.name.c_str()];
   for (RTCVideoCodecInfo *codecInfo in decoder_factory_.supportedCodecs) {
     if ([codecName isEqualToString:codecInfo.name]) {
       id<RTCVideoDecoder> decoder = [decoder_factory_ createDecoder:codecInfo];
-      return new ObjCVideoDecoder(decoder);
+
+      if ([decoder isKindOfClass:[RTCWrappedNativeVideoDecoder class]]) {
+        return [(RTCWrappedNativeVideoDecoder *)decoder releaseWrappedDecoder];
+      } else {
+        return std::unique_ptr<ObjCVideoDecoder>(new ObjCVideoDecoder(decoder));
+      }
     }
   }
 
   return nullptr;
 }
 
+std::vector<SdpVideoFormat> ObjCVideoDecoderFactory::GetSupportedFormats() const {
+  std::vector<SdpVideoFormat> supported_formats;
+  for (RTCVideoCodecInfo *supportedCodec in decoder_factory_.supportedCodecs) {
+    SdpVideoFormat format = [supportedCodec nativeSdpVideoFormat];
+    supported_formats.push_back(format);
+  }
+
+  return supported_formats;
+}
+
+// WebRtcVideoDecoderFactory
+
+VideoDecoder *ObjCVideoDecoderFactory::CreateVideoDecoderWithParams(
+    const cricket::VideoCodec &codec, cricket::VideoDecoderParams params) {
+  return CreateVideoDecoder(SdpVideoFormat(codec.name, codec.params)).release();
+}
+
+VideoDecoder *ObjCVideoDecoderFactory::CreateVideoDecoder(VideoCodecType type) {
+  // This is implemented to avoid hiding an overloaded virtual function
+  RTC_NOTREACHED();
+  return nullptr;
+}
+
 void ObjCVideoDecoderFactory::DestroyVideoDecoder(VideoDecoder *decoder) {
   delete decoder;
   decoder = nullptr;

sdk/objc/Framework/Classes/VideoToolbox/objc_video_encoder_factory.h

@@ -13,19 +13,29 @@
 
 #import <Foundation/Foundation.h>
 
+#include "api/video_codecs/video_encoder_factory.h"
 #include "media/engine/webrtcvideoencoderfactory.h"
 
 @protocol RTCVideoEncoderFactory;
 
 namespace webrtc {
 
-class ObjCVideoEncoderFactory : public cricket::WebRtcVideoEncoderFactory {
+// TODO(andersc): Remove the inheritance from cricket::WebRtcVideoEncoderFactory
+// when the legacy path in [RTCPeerConnectionFactory init] is no longer needed.
+class ObjCVideoEncoderFactory : public VideoEncoderFactory,
+                                public cricket::WebRtcVideoEncoderFactory {
  public:
   explicit ObjCVideoEncoderFactory(id<RTCVideoEncoderFactory>);
   ~ObjCVideoEncoderFactory();
 
   id<RTCVideoEncoderFactory> wrapped_encoder_factory() const;
 
+  std::vector<SdpVideoFormat> GetSupportedFormats() const override;
+  std::unique_ptr<VideoEncoder> CreateVideoEncoder(
+      const SdpVideoFormat& format) override;
+  CodecInfo QueryVideoEncoder(const SdpVideoFormat& format) const override;
+
+  // Needed for WebRtcVideoEncoderFactory interface.
   webrtc::VideoEncoder* CreateVideoEncoder(
       const cricket::VideoCodec& codec) override;
   const std::vector<cricket::VideoCodec>& supported_codecs() const override;
@@ -33,6 +43,8 @@ class ObjCVideoEncoderFactory : public cricket::WebRtcVideoEncoderFactory {
 
  private:
   id<RTCVideoEncoderFactory> encoder_factory_;
+
+  // Needed for WebRtcVideoEncoderFactory interface.
   mutable std::vector<cricket::VideoCodec> supported_codecs_;
 };
 

sdk/objc/Framework/Classes/VideoToolbox/objc_video_encoder_factory.mm

@@ -15,6 +15,8 @@
 #import "NSString+StdString.h"
 #import "RTCI420Buffer+Private.h"
 #import "RTCVideoCodec+Private.h"
+#import "RTCVideoFrame+Private.h"
+#import "RTCWrappedNativeVideoEncoder.h"
 #import "WebRTC/RTCVideoCodec.h"
 #import "WebRTC/RTCVideoCodecFactory.h"
 #import "WebRTC/RTCVideoCodecH264.h"
@@ -22,12 +24,12 @@
 #import "WebRTC/RTCVideoFrameBuffer.h"
 
 #include "api/video/video_frame.h"
+#include "api/video_codecs/sdp_video_format.h"
 #include "api/video_codecs/video_encoder.h"
 #include "modules/include/module_common_types.h"
 #include "modules/video_coding/include/video_codec_interface.h"
 #include "modules/video_coding/include/video_error_codes.h"
 #include "rtc_base/logging.h"
-#include "rtc_base/timeutils.h"
 #include "sdk/objc/Framework/Classes/Common/helpers.h"
 #include "sdk/objc/Framework/Classes/Video/objc_frame_buffer.h"
 
@@ -35,21 +37,6 @@ namespace webrtc {
 
 namespace {
 
-id<RTCVideoFrameBuffer> nativeToRtcFrameBuffer(const rtc::scoped_refptr<VideoFrameBuffer> &buffer) {
-  return buffer->type() == VideoFrameBuffer::Type::kNative ?
-      static_cast<ObjCFrameBuffer *>(buffer.get())->wrapped_frame_buffer() :
-      [[RTCI420Buffer alloc] initWithFrameBuffer:buffer->ToI420()];
-}
-
-RTCVideoFrame *nativeToRtcFrame(const VideoFrame &frame) {
-  RTCVideoFrame *rtcFrame =
-      [[RTCVideoFrame alloc] initWithBuffer:nativeToRtcFrameBuffer(frame.video_frame_buffer())
-                                   rotation:RTCVideoRotation(frame.rotation())
-                                timeStampNs:frame.timestamp_us() * rtc::kNumNanosecsPerMicrosec];
-  rtcFrame.timeStamp = frame.timestamp();
-  return rtcFrame;
-}
-
 class ObjCVideoEncoder : public VideoEncoder {
  public:
   ObjCVideoEncoder(id<RTCVideoEncoder> encoder)
@@ -69,7 +56,7 @@ class ObjCVideoEncoder : public VideoEncoder {
                                 RTCRtpFragmentationHeader *_Nonnull header) {
       EncodedImage encodedImage = [frame nativeEncodedImage];
 
-      // Handle types than can be converted into one of CodecSpecificInfo's hard coded cases.
+      // Handle types that can be converted into one of CodecSpecificInfo's hard coded cases.
       CodecSpecificInfo codecSpecificInfo;
       if ([info isKindOfClass:[RTCCodecSpecificInfoH264 class]]) {
         codecSpecificInfo = [(RTCCodecSpecificInfoH264 *)info nativeCodecSpecificInfo];
@@ -106,7 +93,7 @@ class ObjCVideoEncoder : public VideoEncoder {
       [rtcFrameTypes addObject:@(RTCFrameType(frame_types->at(i)))];
     }
 
-    return [encoder_ encode:nativeToRtcFrame(frame)
+    return [encoder_ encode:[[RTCVideoFrame alloc] initWithNativeVideoFrame:frame]
           codecSpecificInfo:rtcCodecSpecificInfo
                  frameTypes:rtcFrameTypes];
   }
@@ -143,8 +130,44 @@ id<RTCVideoEncoderFactory> ObjCVideoEncoderFactory::wrapped_encoder_factory() co
   return encoder_factory_;
 }
 
+std::vector<SdpVideoFormat> ObjCVideoEncoderFactory::GetSupportedFormats() const {
+  std::vector<SdpVideoFormat> supported_formats;
+  for (RTCVideoCodecInfo *supportedCodec in encoder_factory_.supportedCodecs) {
+    SdpVideoFormat format = [supportedCodec nativeSdpVideoFormat];
+    supported_formats.push_back(format);
+  }
+
+  return supported_formats;
+}
+
+VideoEncoderFactory::CodecInfo ObjCVideoEncoderFactory::QueryVideoEncoder(
+    const SdpVideoFormat &format) const {
+  // TODO(andersc): This is a hack until we figure out how this should be done properly.
+  NSString *formatName = [NSString stringForStdString:format.name];
+  NSSet *wrappedSoftwareFormats = [NSSet setWithObjects:@"VP8", @"VP9", nil];
+
+  CodecInfo codec_info;
+  codec_info.is_hardware_accelerated = ![wrappedSoftwareFormats containsObject:formatName];
+  codec_info.has_internal_source = false;
+  return codec_info;
+}
+
+std::unique_ptr<VideoEncoder> ObjCVideoEncoderFactory::CreateVideoEncoder(
+    const SdpVideoFormat &format) {
+  RTCVideoCodecInfo *info = [[RTCVideoCodecInfo alloc] initWithNativeSdpVideoFormat:format];
+  id<RTCVideoEncoder> encoder = [encoder_factory_ createEncoder:info];
+  if ([encoder isKindOfClass:[RTCWrappedNativeVideoEncoder class]]) {
+    return [(RTCWrappedNativeVideoEncoder *)encoder releaseWrappedEncoder];
+  } else {
+    return std::unique_ptr<ObjCVideoEncoder>(new ObjCVideoEncoder(encoder));
+  }
+}
+
+// WebRtcVideoEncoderFactory
+
 VideoEncoder *ObjCVideoEncoderFactory::CreateVideoEncoder(const cricket::VideoCodec &codec) {
-  RTCVideoCodecInfo *info = [[RTCVideoCodecInfo alloc] initWithNativeVideoCodec:codec];
+  RTCVideoCodecInfo *info = [[RTCVideoCodecInfo alloc]
+      initWithNativeSdpVideoFormat:SdpVideoFormat(codec.name, codec.params)];
   id<RTCVideoEncoder> encoder = [encoder_factory_ createEncoder:info];
   return new ObjCVideoEncoder(encoder);
 }
@@ -152,8 +175,8 @@ VideoEncoder *ObjCVideoEncoderFactory::CreateVideoEncoder(const cricket::VideoCo
 const std::vector<cricket::VideoCodec> &ObjCVideoEncoderFactory::supported_codecs() const {
   supported_codecs_.clear();
   for (RTCVideoCodecInfo *supportedCodec in encoder_factory_.supportedCodecs) {
-    cricket::VideoCodec codec = [supportedCodec nativeVideoCodec];
-    supported_codecs_.push_back(codec);
+    SdpVideoFormat format = [supportedCodec nativeSdpVideoFormat];
+    supported_codecs_.push_back(cricket::VideoCodec(format));
   }
 
   return supported_codecs_;