Add HW fallback option to software decoding.

Permits falling back to software decoding for unsupported resolutions in
bitstreams.

BUG=4625, chromium:487934
R=mflodman@webrtc.org, stefan@webrtc.org

Review URL: https://webrtc-codereview.appspot.com/46269004

Cr-Commit-Position: refs/heads/master@{#9209}

talk/media/webrtc/webrtcvideoengine2.cc

@@ -2223,6 +2223,21 @@ WebRtcVideoChannel2::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream(
   SetRecvCodecs(recv_codecs);
 }
 
+WebRtcVideoChannel2::WebRtcVideoReceiveStream::AllocatedDecoder::
+    AllocatedDecoder(webrtc::VideoDecoder* decoder,
+                     webrtc::VideoCodecType type,
+                     bool external)
+    : decoder(decoder),
+      external_decoder(nullptr),
+      type(type),
+      external(external) {
+  if (external) {
+    external_decoder = decoder;
+    this->decoder =
+        new webrtc::VideoDecoderSoftwareFallbackWrapper(type, external_decoder);
+  }
+}
+
 WebRtcVideoChannel2::WebRtcVideoReceiveStream::~WebRtcVideoReceiveStream() {
   call_->DestroyVideoReceiveStream(stream_);
   ClearDecoders(&allocated_decoders_);
@@ -2330,10 +2345,9 @@ void WebRtcVideoChannel2::WebRtcVideoReceiveStream::ClearDecoders(
   for (size_t i = 0; i < allocated_decoders->size(); ++i) {
     if ((*allocated_decoders)[i].external) {
       external_decoder_factory_->DestroyVideoDecoder(
-          (*allocated_decoders)[i].decoder);
-    } else {
-      delete (*allocated_decoders)[i].decoder;
+          (*allocated_decoders)[i].external_decoder);
     }
+    delete (*allocated_decoders)[i].decoder;
   }
   allocated_decoders->clear();
 }

talk/media/webrtc/webrtcvideoengine2.h

@@ -430,9 +430,10 @@ class WebRtcVideoChannel2 : public rtc::MessageHandler,
     struct AllocatedDecoder {
       AllocatedDecoder(webrtc::VideoDecoder* decoder,
                        webrtc::VideoCodecType type,
-                       bool external)
-          : decoder(decoder), type(type), external(external) {}
+                       bool external);
       webrtc::VideoDecoder* decoder;
+      // Decoder wrapped into a fallback decoder to permit software fallback.
+      webrtc::VideoDecoder* external_decoder;
       webrtc::VideoCodecType type;
       bool external;
     };

webrtc/modules/video_coding/codecs/interface/video_error_codes.h

@@ -26,5 +26,6 @@
 #define WEBRTC_VIDEO_CODEC_TIMEOUT -6
 #define WEBRTC_VIDEO_CODEC_UNINITIALIZED -7
 #define WEBRTC_VIDEO_CODEC_ERR_REQUEST_SLI -12
+#define WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE -13
 
 #endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_INTERFACE_VIDEO_ERROR_CODES_H

webrtc/video/BUILD.gn

@@ -21,6 +21,7 @@ source_set("video") {
     "send_statistics_proxy.h",
     "transport_adapter.cc",
     "transport_adapter.h",
+    "video_decoder.cc",
     "video_receive_stream.cc",
     "video_receive_stream.h",
     "video_send_stream.cc",

webrtc/video/call.cc

@@ -47,17 +47,6 @@ VideoEncoder* VideoEncoder::Create(VideoEncoder::EncoderType codec_type) {
   return nullptr;
 }
 
-VideoDecoder* VideoDecoder::Create(VideoDecoder::DecoderType codec_type) {
-  switch (codec_type) {
-    case kVp8:
-      return VP8Decoder::Create();
-    case kVp9:
-      return VP9Decoder::Create();
-  }
-  RTC_NOTREACHED();
-  return nullptr;
-}
-
 const int Call::Config::kDefaultStartBitrateBps = 300000;
 
 namespace internal {

webrtc/video/video_decoder.cc

@@ -0,0 +1,128 @@
+/*
+ *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/video_decoder.h"
+
+#include "webrtc/base/checks.h"
+#include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h"
+#include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h"
+#include "webrtc/system_wrappers/interface/logging.h"
+
+namespace webrtc {
+VideoDecoder* VideoDecoder::Create(VideoDecoder::DecoderType codec_type) {
+  switch (codec_type) {
+    case kVp8:
+      return VP8Decoder::Create();
+    case kVp9:
+      return VP9Decoder::Create();
+    case kUnsupportedCodec:
+      RTC_NOTREACHED();
+      return nullptr;
+  }
+  RTC_NOTREACHED();
+  return nullptr;
+}
+
+VideoDecoder::DecoderType CodecTypeToDecoderType(VideoCodecType codec_type) {
+  switch (codec_type) {
+    case kVideoCodecVP8:
+      return VideoDecoder::kVp8;
+    case kVideoCodecVP9:
+      return VideoDecoder::kVp9;
+    default:
+      return VideoDecoder::kUnsupportedCodec;
+  }
+}
+
+VideoDecoderSoftwareFallbackWrapper::VideoDecoderSoftwareFallbackWrapper(
+    VideoCodecType codec_type,
+    VideoDecoder* decoder)
+    : decoder_type_(CodecTypeToDecoderType(codec_type)),
+      decoder_(decoder),
+      callback_(nullptr) {
+}
+
+int32_t VideoDecoderSoftwareFallbackWrapper::InitDecode(
+    const VideoCodec* codec_settings,
+    int32_t number_of_cores) {
+  codec_settings_ = *codec_settings;
+  number_of_cores_ = number_of_cores;
+  return decoder_->InitDecode(codec_settings, number_of_cores);
+}
+
+bool VideoDecoderSoftwareFallbackWrapper::InitFallbackDecoder() {
+  CHECK(decoder_type_ != kUnsupportedCodec)
+      << "Decoder requesting fallback to codec not supported in software.";
+  LOG(LS_WARNING) << "Decoder falling back to software decoding.";
+  fallback_decoder_.reset(VideoDecoder::Create(decoder_type_));
+  if (fallback_decoder_->InitDecode(&codec_settings_, number_of_cores_) !=
+      WEBRTC_VIDEO_CODEC_OK) {
+    LOG(LS_ERROR) << "Failed to initialize software-decoder fallback.";
+    fallback_decoder_.reset();
+    return false;
+  }
+  if (callback_ != nullptr)
+    fallback_decoder_->RegisterDecodeCompleteCallback(callback_);
+  return true;
+}
+
+int32_t VideoDecoderSoftwareFallbackWrapper::Decode(
+    const EncodedImage& input_image,
+    bool missing_frames,
+    const RTPFragmentationHeader* fragmentation,
+    const CodecSpecificInfo* codec_specific_info,
+    int64_t render_time_ms) {
+  // Try decoding with the provided decoder on every keyframe or when there's no
+  // fallback decoder. This is the normal case.
+  if (!fallback_decoder_ || input_image._frameType == kKeyFrame) {
+    int32_t ret = decoder_->Decode(input_image, missing_frames, fragmentation,
+                                   codec_specific_info, render_time_ms);
+    if (ret == WEBRTC_VIDEO_CODEC_OK) {
+      if (fallback_decoder_) {
+        // Decode OK -> stop using fallback decoder.
+        fallback_decoder_->Release();
+        fallback_decoder_.reset();
+        return WEBRTC_VIDEO_CODEC_OK;
+      }
+    }
+    if (ret != WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE)
+      return ret;
+    if (!fallback_decoder_) {
+      // Try to initialize fallback decoder.
+      if (!InitFallbackDecoder())
+        return ret;
+    }
+  }
+  return fallback_decoder_->Decode(input_image, missing_frames, fragmentation,
+                                   codec_specific_info, render_time_ms);
+}
+
+int32_t VideoDecoderSoftwareFallbackWrapper::RegisterDecodeCompleteCallback(
+    DecodedImageCallback* callback) {
+  callback_ = callback;
+  int32_t ret = decoder_->RegisterDecodeCompleteCallback(callback);
+  if (fallback_decoder_)
+    return fallback_decoder_->RegisterDecodeCompleteCallback(callback);
+  return ret;
+}
+
+int32_t VideoDecoderSoftwareFallbackWrapper::Release() {
+  if (fallback_decoder_)
+    fallback_decoder_->Release();
+  return decoder_->Release();
+}
+
+int32_t VideoDecoderSoftwareFallbackWrapper::Reset() {
+  if (fallback_decoder_)
+    fallback_decoder_->Reset();
+  return decoder_->Reset();
+}
+
+}  // namespace webrtc

webrtc/video/video_decoder_unittest.cc

@@ -0,0 +1,165 @@
+/*
+ *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/video_decoder.h"
+
+#include "testing/gtest/include/gtest/gtest.h"
+#include "webrtc/modules/video_coding/codecs/interface/video_error_codes.h"
+
+namespace webrtc {
+
+class VideoDecoderSoftwareFallbackWrapperTest : public ::testing::Test {
+ protected:
+  VideoDecoderSoftwareFallbackWrapperTest()
+      : fallback_wrapper_(kVideoCodecVP8, &fake_decoder_) {}
+
+  class CountingFakeDecoder : public VideoDecoder {
+   public:
+    int32_t InitDecode(const VideoCodec* codec_settings,
+                       int32_t number_of_cores) override {
+      ++init_decode_count_;
+      return WEBRTC_VIDEO_CODEC_OK;
+    }
+
+    int32_t Decode(const EncodedImage& input_image,
+                   bool missing_frames,
+                   const RTPFragmentationHeader* fragmentation,
+                   const CodecSpecificInfo* codec_specific_info,
+                   int64_t render_time_ms) override {
+      ++decode_count_;
+      return decode_return_code_;
+    }
+
+    int32_t RegisterDecodeCompleteCallback(
+        DecodedImageCallback* callback) override {
+      decode_complete_callback_ = callback;
+      return WEBRTC_VIDEO_CODEC_OK;
+    }
+
+    int32_t Release() override {
+      ++release_count_;
+      return WEBRTC_VIDEO_CODEC_OK;
+    }
+
+    int32_t Reset() override {
+      ++reset_count_;
+      return WEBRTC_VIDEO_CODEC_OK;
+    }
+    int init_decode_count_ = 0;
+    int decode_count_ = 0;
+    int32_t decode_return_code_ = WEBRTC_VIDEO_CODEC_OK;
+    DecodedImageCallback* decode_complete_callback_ = nullptr;
+    int release_count_ = 0;
+    int reset_count_ = 0;
+  };
+  CountingFakeDecoder fake_decoder_;
+  VideoDecoderSoftwareFallbackWrapper fallback_wrapper_;
+};
+
+TEST_F(VideoDecoderSoftwareFallbackWrapperTest, InitializesDecoder) {
+  VideoCodec codec = {};
+  fallback_wrapper_.InitDecode(&codec, 2);
+  EXPECT_EQ(1, fake_decoder_.init_decode_count_);
+}
+
+TEST_F(VideoDecoderSoftwareFallbackWrapperTest,
+       CanRecoverFromSoftwareFallback) {
+  VideoCodec codec = {};
+  fallback_wrapper_.InitDecode(&codec, 2);
+  // Unfortunately faking a VP8 frame is hard. Rely on no Decode -> using SW
+  // decoder.
+  fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
+  EncodedImage encoded_image;
+  fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
+  EXPECT_EQ(1, fake_decoder_.decode_count_);
+
+  // Fail -> fake_decoder shouldn't be used anymore.
+  fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
+  EXPECT_EQ(1, fake_decoder_.decode_count_)
+      << "Decoder used even though fallback should be active.";
+
+  // Should be able to recover on a keyframe.
+  encoded_image._frameType = kKeyFrame;
+  fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_OK;
+  fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
+  EXPECT_EQ(2, fake_decoder_.decode_count_)
+      << "Wrapper did not try to decode a keyframe using registered decoder.";
+
+  encoded_image._frameType = kDeltaFrame;
+  fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
+  EXPECT_EQ(3, fake_decoder_.decode_count_)
+      << "Decoder not used on future delta frames.";
+}
+
+TEST_F(VideoDecoderSoftwareFallbackWrapperTest, DoesNotFallbackOnEveryError) {
+  VideoCodec codec = {};
+  fallback_wrapper_.InitDecode(&codec, 2);
+  fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_ERROR;
+  EncodedImage encoded_image;
+  EXPECT_EQ(
+      fake_decoder_.decode_return_code_,
+      fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1));
+  EXPECT_EQ(1, fake_decoder_.decode_count_);
+
+  fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
+  EXPECT_EQ(2, fake_decoder_.decode_count_)
+      << "Decoder should be active even though previous decode failed.";
+}
+
+TEST_F(VideoDecoderSoftwareFallbackWrapperTest, ForwardsReleaseCall) {
+  VideoCodec codec = {};
+  fallback_wrapper_.InitDecode(&codec, 2);
+  fallback_wrapper_.Release();
+  EXPECT_EQ(1, fake_decoder_.release_count_);
+
+  fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
+  EncodedImage encoded_image;
+  fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
+  EXPECT_EQ(1, fake_decoder_.release_count_)
+      << "Decoder should not be released during fallback.";
+  fallback_wrapper_.Release();
+  EXPECT_EQ(2, fake_decoder_.release_count_);
+}
+
+TEST_F(VideoDecoderSoftwareFallbackWrapperTest, ForwardsResetCall) {
+  VideoCodec codec = {};
+  fallback_wrapper_.InitDecode(&codec, 2);
+  fallback_wrapper_.Reset();
+  EXPECT_EQ(1, fake_decoder_.reset_count_);
+
+  fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
+  EncodedImage encoded_image;
+  fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
+  fallback_wrapper_.Reset();
+  EXPECT_EQ(2, fake_decoder_.reset_count_)
+      << "Reset not forwarded during fallback.";
+}
+
+// TODO(pbos): Fake a VP8 frame well enough to actually receive a callback from
+// the software encoder.
+TEST_F(VideoDecoderSoftwareFallbackWrapperTest,
+       ForwardsRegisterDecodeCompleteCallback) {
+  class FakeDecodedImageCallback : public DecodedImageCallback {
+    int32_t Decoded(I420VideoFrame& decodedImage) override { return 0; }
+  } callback, callback2;
+
+  VideoCodec codec = {};
+  fallback_wrapper_.InitDecode(&codec, 2);
+  fallback_wrapper_.RegisterDecodeCompleteCallback(&callback);
+  EXPECT_EQ(&callback, fake_decoder_.decode_complete_callback_);
+
+  fake_decoder_.decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
+  EncodedImage encoded_image;
+  fallback_wrapper_.Decode(encoded_image, false, nullptr, nullptr, -1);
+  fallback_wrapper_.RegisterDecodeCompleteCallback(&callback2);
+  EXPECT_EQ(&callback2, fake_decoder_.decode_complete_callback_);
+}
+
+}  // namespace webrtc

webrtc/video/webrtc_video.gypi

@@ -22,6 +22,7 @@
       'video/receive_statistics_proxy.h',
       'video/transport_adapter.cc',
       'video/transport_adapter.h',
+      'video/video_decoder.cc',
       'video/video_receive_stream.cc',
       'video/video_receive_stream.h',
       'video/video_send_stream.cc',

webrtc/video_decoder.h

@@ -40,21 +40,22 @@ class VideoDecoder {
  public:
   enum DecoderType {
     kVp8,
-    kVp9
+    kVp9,
+    kUnsupportedCodec,
   };
 
   static VideoDecoder* Create(DecoderType codec_type);
 
   virtual ~VideoDecoder() {}
 
-  virtual int32_t InitDecode(const VideoCodec* codecSettings,
-                             int32_t numberOfCores) = 0;
+  virtual int32_t InitDecode(const VideoCodec* codec_settings,
+                             int32_t number_of_cores) = 0;
 
-  virtual int32_t Decode(const EncodedImage& inputImage,
-                         bool missingFrames,
+  virtual int32_t Decode(const EncodedImage& input_image,
+                         bool missing_frames,
                          const RTPFragmentationHeader* fragmentation,
-                         const CodecSpecificInfo* codecSpecificInfo = NULL,
-                         int64_t renderTimeMs = -1) = 0;
+                         const CodecSpecificInfo* codec_specific_info = NULL,
+                         int64_t render_time_ms = -1) = 0;
 
   virtual int32_t RegisterDecodeCompleteCallback(
       DecodedImageCallback* callback) = 0;
@@ -70,6 +71,41 @@ class VideoDecoder {
   virtual VideoDecoder* Copy() { return NULL; }
 };
 
+// Class used to wrap external VideoDecoders to provide a fallback option on
+// software decoding when a hardware decoder fails to decode a stream due to
+// hardware restrictions, such as max resolution.
+class VideoDecoderSoftwareFallbackWrapper : public webrtc::VideoDecoder {
+ public:
+  VideoDecoderSoftwareFallbackWrapper(VideoCodecType codec_type,
+                                      VideoDecoder* decoder);
+
+  int32_t InitDecode(const VideoCodec* codec_settings,
+                     int32_t number_of_cores) override;
+
+  int32_t Decode(const EncodedImage& input_image,
+                 bool missing_frames,
+                 const RTPFragmentationHeader* fragmentation,
+                 const CodecSpecificInfo* codec_specific_info,
+                 int64_t render_time_ms) override;
+
+  int32_t RegisterDecodeCompleteCallback(
+      DecodedImageCallback* callback) override;
+
+  int32_t Release() override;
+  int32_t Reset() override;
+
+ private:
+  bool InitFallbackDecoder();
+
+  const DecoderType decoder_type_;
+  VideoDecoder* const decoder_;
+
+  VideoCodec codec_settings_;
+  int32_t number_of_cores_;
+  rtc::scoped_ptr<VideoDecoder> fallback_decoder_;
+  DecodedImageCallback* callback_;
+};
+
 }  // namespace webrtc
 
 #endif  // WEBRTC_VIDEO_DECODER_H_

webrtc/webrtc_tests.gypi

@@ -150,6 +150,7 @@
         'video/bitrate_estimator_tests.cc',
         'video/end_to_end_tests.cc',
         'video/send_statistics_proxy_unittest.cc',
+        'video/video_decoder_unittest.cc',
         'video/video_send_stream_tests.cc',
       ],
       'dependencies': [