Dynamic resolution change for VP8 HW encode.

Off by default for now.

BUG=
R=glaznev@webrtc.org, stefan@webrtc.org
TBR=mflodman@webrtc.org

Review URL: https://webrtc-codereview.appspot.com/45849004

Cr-Commit-Position: refs/heads/master@{#9045}

webrtc/modules/video_coding/BUILD.gn

@@ -94,8 +94,9 @@ source_set("video_coding_utility") {
   sources = [
     "utility/frame_dropper.cc",
     "utility/include/frame_dropper.h",
+    "utility/include/moving_average.h",
+    "utility/include/quality_scaler.h",
     "utility/quality_scaler.cc",
-    "utility/quality_scaler.h",
   ]
 
   configs += [ "../..:common_config" ]

webrtc/modules/video_coding/codecs/i420/main/interface/i420.h

@@ -77,6 +77,8 @@ class I420Encoder : public VideoEncoder {
     return WEBRTC_VIDEO_CODEC_OK;
   }
 
+  void OnDroppedFrame() override {}
+
  private:
   static uint8_t* InsertHeader(uint8_t* buffer, uint16_t width,
                                uint16_t height);

webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc

@@ -501,4 +501,8 @@ bool SimulcastEncoderAdapter::Initialized() const {
   return !streaminfos_.empty();
 }
 
+void SimulcastEncoderAdapter::OnDroppedFrame() {
+  streaminfos_[0].encoder->OnDroppedFrame();
+}
+
 }  // namespace webrtc

webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h

@@ -55,6 +55,8 @@ class SimulcastEncoderAdapter : public VP8Encoder {
                   const CodecSpecificInfo* codecSpecificInfo = NULL,
                   const RTPFragmentationHeader* fragmentation = NULL);
 
+  void OnDroppedFrame() override;
+
  private:
   struct StreamInfo {
     StreamInfo()

webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc

@@ -1035,7 +1035,7 @@ int VP8EncoderImpl::GetEncodedPartitions(
     if (encoded_images_[0]._length > 0) {
       int qp;
       vpx_codec_control(&encoders_[0], VP8E_GET_LAST_QUANTIZER_64, &qp);
-      quality_scaler_.ReportEncodedFrame(qp);
+      quality_scaler_.ReportNormalizedQP(qp);
     } else {
       quality_scaler_.ReportDroppedFrame();
     }

webrtc/modules/video_coding/codecs/vp8/vp8_impl.h

@@ -27,7 +27,7 @@
 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h"
 #include "webrtc/modules/video_coding/codecs/vp8/reference_picture_selection.h"
 #include "webrtc/modules/video_coding/utility/include/frame_dropper.h"
-#include "webrtc/modules/video_coding/utility/quality_scaler.h"
+#include "webrtc/modules/video_coding/utility/include/quality_scaler.h"
 #include "webrtc/video_frame.h"
 
 namespace webrtc {
@@ -56,6 +56,8 @@ class VP8EncoderImpl : public VP8Encoder {
 
   virtual int SetRates(uint32_t new_bitrate_kbit, uint32_t frame_rate);
 
+  void OnDroppedFrame() override {}
+
  private:
   void SetupTemporalLayers(int num_streams, int num_temporal_layers,
                            const VideoCodec& codec);

webrtc/modules/video_coding/codecs/vp9/vp9_impl.h

@@ -41,6 +41,8 @@ class VP9EncoderImpl : public VP9Encoder {
 
   int SetRates(uint32_t new_bitrate_kbit, uint32_t frame_rate) override;
 
+  void OnDroppedFrame() override {}
+
  private:
   // Determine number of encoder threads to use.
   int NumberOfThreads(int width, int height, int number_of_cores);

webrtc/modules/video_coding/main/source/generic_encoder.cc

@@ -197,6 +197,10 @@ VCMGenericEncoder::InternalSource() const
     return internal_source_;
 }
 
+void VCMGenericEncoder::OnDroppedFrame() {
+  encoder_->OnDroppedFrame();
+}
+
  /***************************
   * Callback Implementation
   ***************************/

webrtc/modules/video_coding/main/source/generic_encoder.h

@@ -136,6 +136,8 @@ public:
 
     bool InternalSource() const;
 
+    void OnDroppedFrame();
+
 private:
     VideoEncoder* const encoder_;
     VideoEncoderRateObserver* const rate_observer_;

webrtc/modules/video_coding/main/source/video_sender.cc

@@ -17,6 +17,7 @@
 #include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h"
 #include "webrtc/modules/video_coding/main/source/encoded_frame.h"
 #include "webrtc/modules/video_coding/main/source/video_coding_impl.h"
+#include "webrtc/modules/video_coding/utility/include/quality_scaler.h"
 #include "webrtc/system_wrappers/interface/clock.h"
 #include "webrtc/system_wrappers/interface/logging.h"
 
@@ -346,6 +347,7 @@ int32_t VideoSender::AddVideoFrame(const I420VideoFrame& videoFrame,
     return VCM_OK;
   }
   if (_mediaOpt.DropFrame()) {
+    _encoder->OnDroppedFrame();
     return VCM_OK;
   }
   _mediaOpt.UpdateContentData(contentMetrics);

webrtc/modules/video_coding/utility/include/moving_average.h

@@ -0,0 +1,71 @@
+/*
+ *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef WEBRTC_MODULES_VIDEO_CODING_UTILITY_MOVING_AVERAGE_H_
+#define WEBRTC_MODULES_VIDEO_CODING_UTILITY_MOVING_AVERAGE_H_
+
+#include <list>
+
+#include "webrtc/typedefs.h"
+
+namespace webrtc {
+template<class T>
+class MovingAverage {
+ public:
+  MovingAverage();
+  void AddSample(T sample);
+  bool GetAverage(size_t num_samples, T* average);
+  void Reset();
+  int size();
+
+ private:
+  T sum_;
+  std::list<T> samples_;
+};
+
+template<class T>
+MovingAverage<T>::MovingAverage() : sum_(static_cast<T>(0)) {
+}
+
+template<class T>
+void MovingAverage<T>::AddSample(T sample) {
+  samples_.push_back(sample);
+  sum_ += sample;
+}
+
+template<class T>
+bool MovingAverage<T>::GetAverage(size_t num_samples, T* avg) {
+  if (num_samples > samples_.size())
+    return false;
+
+  // Remove old samples.
+  while (num_samples < samples_.size()) {
+    sum_ -= samples_.front();
+    samples_.pop_front();
+  }
+
+  *avg = sum_ / static_cast<T>(num_samples);
+  return true;
+}
+
+template<class T>
+void MovingAverage<T>::Reset() {
+  sum_ = static_cast<T>(0);
+  samples_.clear();
+}
+
+template<class T>
+int MovingAverage<T>::size() {
+  return samples_.size();
+}
+
+}  // namespace webrtc
+
+#endif  // WEBRTC_MODULES_VIDEO_CODING_MOVING_AVERAGE_SCALER_H_

webrtc/modules/video_coding/utility/include/quality_scaler.h

@@ -11,9 +11,8 @@
 #ifndef WEBRTC_MODULES_VIDEO_CODING_UTILITY_QUALITY_SCALER_H_
 #define WEBRTC_MODULES_VIDEO_CODING_UTILITY_QUALITY_SCALER_H_
 
-#include <list>
-
 #include "webrtc/common_video/libyuv/include/scaler.h"
+#include "webrtc/modules/video_coding/utility/include/moving_average.h"
 
 namespace webrtc {
 class QualityScaler {
@@ -25,27 +24,20 @@ class QualityScaler {
 
   QualityScaler();
   void Init(int max_qp);
-
+  void SetMinResolution(int min_width, int min_height);
   void ReportFramerate(int framerate);
-  void ReportEncodedFrame(int qp);
-  void ReportDroppedFrame();
 
+  // Report QP for SW encoder, report framesize fluctuation for HW encoder,
+  // only one of these two functions should be called, framesize fluctuation
+  // is to be used only if qp isn't available.
+  void ReportNormalizedQP(int qp);
+  void ReportNormalizedFrameSizeFluctuation(double framesize_deviation);
+  void ReportDroppedFrame();
+  void Reset(int framerate, int bitrate, int width, int height);
   Resolution GetScaledResolution(const I420VideoFrame& frame);
   const I420VideoFrame& GetScaledFrame(const I420VideoFrame& frame);
 
  private:
-  class MovingAverage {
-   public:
-    MovingAverage();
-    void AddSample(int sample);
-    bool GetAverage(size_t num_samples, int* average);
-    void Reset();
-
-   private:
-    int sum_;
-    std::list<int> samples_;
-  };
-
   void AdjustScale(bool up);
   void ClearSamples();
 
@@ -53,11 +45,14 @@ class QualityScaler {
   I420VideoFrame scaled_frame_;
 
   size_t num_samples_;
+  int target_framesize_;
   int low_qp_threshold_;
-  MovingAverage average_qp_;
-  MovingAverage framedrop_percent_;
+  MovingAverage<int> framedrop_percent_;
+  MovingAverage<double> frame_quality_;
 
   int downscale_shift_;
+  int min_width_;
+  int min_height_;
 };
 
 }  // namespace webrtc

webrtc/modules/video_coding/utility/quality_scaler.cc

@@ -7,8 +7,7 @@
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
-
-#include "webrtc/modules/video_coding/utility/quality_scaler.h"
+#include "webrtc/modules/video_coding/utility/include/quality_scaler.h"
 
 namespace webrtc {
 
@@ -16,25 +15,41 @@ static const int kMinFps = 10;
 static const int kMeasureSeconds = 5;
 static const int kFramedropPercentThreshold = 60;
 static const int kLowQpThresholdDenominator = 3;
+static const double kFramesizeFlucThreshold = 0.11;
 
 QualityScaler::QualityScaler()
-    : num_samples_(0), low_qp_threshold_(-1), downscale_shift_(0) {
+    : num_samples_(0), low_qp_threshold_(-1), downscale_shift_(0),
+      min_width_(0), min_height_(0) {
 }
 
 void QualityScaler::Init(int max_qp) {
   ClearSamples();
-  downscale_shift_ = 0;
-  low_qp_threshold_ = max_qp / kLowQpThresholdDenominator ;
+  low_qp_threshold_ = max_qp / kLowQpThresholdDenominator;
 }
 
+void QualityScaler::SetMinResolution(int min_width, int min_height) {
+  min_width_ = min_width;
+  min_height_ = min_height;
+}
+
+// TODO(jackychen): target_framesize should be calculated from average bitrate
+// in the measured period of time.
+// Report framerate(fps) and target_bitrate(kbit/s) to estimate # of samples
+// and get target_framesize_.
 void QualityScaler::ReportFramerate(int framerate) {
   num_samples_ = static_cast<size_t>(
       kMeasureSeconds * (framerate < kMinFps ? kMinFps : framerate));
 }
 
-void QualityScaler::ReportEncodedFrame(int qp) {
-  average_qp_.AddSample(qp);
+void QualityScaler::ReportNormalizedQP(int qp) {
   framedrop_percent_.AddSample(0);
+  frame_quality_.AddSample(static_cast<double>(qp) / low_qp_threshold_);
+}
+
+void QualityScaler::ReportNormalizedFrameSizeFluctuation(
+    double framesize_deviation) {
+  framedrop_percent_.AddSample(0);
+  frame_quality_.AddSample(framesize_deviation / kFramesizeFlucThreshold);
 }
 
 void QualityScaler::ReportDroppedFrame() {
@@ -43,23 +58,25 @@ void QualityScaler::ReportDroppedFrame() {
 
 QualityScaler::Resolution QualityScaler::GetScaledResolution(
     const I420VideoFrame& frame) {
-  // Both of these should be set through InitEncode -> Should be set by now.
+  // Should be set through InitEncode -> Should be set by now.
   assert(low_qp_threshold_ >= 0);
   assert(num_samples_ > 0);
-  // Update scale factor.
-  int avg;
-  if (framedrop_percent_.GetAverage(num_samples_, &avg) &&
-      avg >= kFramedropPercentThreshold) {
-    AdjustScale(false);
-  } else if (average_qp_.GetAverage(num_samples_, &avg) &&
-             avg <= low_qp_threshold_) {
-    AdjustScale(true);
-  }
 
   Resolution res;
   res.width = frame.width();
   res.height = frame.height();
 
+  // Update scale factor.
+  int avg_drop;
+  double avg_quality;
+  if (framedrop_percent_.GetAverage(num_samples_, &avg_drop) &&
+      avg_drop >= kFramedropPercentThreshold) {
+    AdjustScale(false);
+  } else if (frame_quality_.GetAverage(num_samples_, &avg_quality) &&
+      avg_quality <= 1.0) {
+    AdjustScale(true);
+  }
+
   assert(downscale_shift_ >= 0);
   for (int shift = downscale_shift_;
        shift > 0 && res.width > 1 && res.height > 1;
@@ -68,6 +85,12 @@ QualityScaler::Resolution QualityScaler::GetScaledResolution(
     res.height >>= 1;
   }
 
+  // Set this limitation for VP8 HW encoder to avoid crash.
+  if (min_width_ > 0 && res.width * res.height < min_width_ * min_height_) {
+    res.width = min_width_;
+    res.height = min_height_;
+  }
+
   return res;
 }
 
@@ -94,37 +117,9 @@ const I420VideoFrame& QualityScaler::GetScaledFrame(
   return scaled_frame_;
 }
 
-QualityScaler::MovingAverage::MovingAverage() : sum_(0) {
-}
-
-void QualityScaler::MovingAverage::AddSample(int sample) {
-  samples_.push_back(sample);
-  sum_ += sample;
-}
-
-bool QualityScaler::MovingAverage::GetAverage(size_t num_samples, int* avg) {
-  assert(num_samples > 0);
-  if (num_samples > samples_.size())
-    return false;
-
-  // Remove old samples.
-  while (num_samples < samples_.size()) {
-    sum_ -= samples_.front();
-    samples_.pop_front();
-  }
-
-  *avg = sum_ / static_cast<int>(num_samples);
-  return true;
-}
-
-void QualityScaler::MovingAverage::Reset() {
-  sum_ = 0;
-  samples_.clear();
-}
-
 void QualityScaler::ClearSamples() {
-  average_qp_.Reset();
   framedrop_percent_.Reset();
+  frame_quality_.Reset();
 }
 
 void QualityScaler::AdjustScale(bool up) {

webrtc/modules/video_coding/utility/quality_scaler_unittest.cc

@@ -8,7 +8,7 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
-#include "webrtc/modules/video_coding/utility/quality_scaler.h"
+#include "webrtc/modules/video_coding/utility/include/quality_scaler.h"
 
 #include "testing/gtest/include/gtest/gtest.h"
 
@@ -40,7 +40,7 @@ class QualityScalerTest : public ::testing::Test {
     for (int i = 0; i < kFramerate * kNumSeconds; ++i) {
       switch (scale_direction) {
         case kScaleUp:
-          qs_.ReportEncodedFrame(kLowQp);
+          qs_.ReportNormalizedQP(kLowQp);
           break;
         case kScaleDown:
           qs_.ReportDroppedFrame();
@@ -93,7 +93,7 @@ TEST_F(QualityScalerTest, DownscalesAfterContinuousFramedrop) {
 
 TEST_F(QualityScalerTest, DownscalesAfterTwoThirdsFramedrop) {
   for (int i = 0; i < kFramerate * kNumSeconds / 3; ++i) {
-    qs_.ReportEncodedFrame(kNormalQp);
+    qs_.ReportNormalizedQP(kNormalQp);
     qs_.ReportDroppedFrame();
     qs_.ReportDroppedFrame();
     if (qs_.GetScaledResolution(input_frame_).width < input_frame_.width())
@@ -105,7 +105,7 @@ TEST_F(QualityScalerTest, DownscalesAfterTwoThirdsFramedrop) {
 
 TEST_F(QualityScalerTest, DoesNotDownscaleOnNormalQp) {
   for (int i = 0; i < kFramerate * kNumSeconds; ++i) {
-    qs_.ReportEncodedFrame(kNormalQp);
+    qs_.ReportNormalizedQP(kNormalQp);
     ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution(input_frame_).width)
         << "Unexpected scale on half framedrop.";
   }
@@ -113,7 +113,7 @@ TEST_F(QualityScalerTest, DoesNotDownscaleOnNormalQp) {
 
 TEST_F(QualityScalerTest, DoesNotDownscaleAfterHalfFramedrop) {
   for (int i = 0; i < kFramerate * kNumSeconds / 2; ++i) {
-    qs_.ReportEncodedFrame(kNormalQp);
+    qs_.ReportNormalizedQP(kNormalQp);
     ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution(input_frame_).width)
         << "Unexpected scale on half framedrop.";
 
@@ -153,7 +153,7 @@ void QualityScalerTest::ContinuouslyDownscalesByHalfDimensionsAndBackUp() {
 
   // Verify we don't start upscaling after further low use.
   for (int i = 0; i < kFramerate * kNumSeconds; ++i) {
-    qs_.ReportEncodedFrame(kLowQp);
+    qs_.ReportNormalizedQP(kLowQp);
     ExpectOriginalFrame();
   }
 }

webrtc/modules/video_coding/utility/video_coding_utility.gyp

@@ -20,8 +20,9 @@
       'sources': [
         'frame_dropper.cc',
         'include/frame_dropper.h',
+        'include/moving_average.h',
         'quality_scaler.cc',
-        'quality_scaler.h',
+        'include/quality_scaler.h',
       ],
     },
   ], # targets