From 194f40a2e77481b727e6656a6945bc23180eb9ff Mon Sep 17 00:00:00 2001 From: kthelgason Date: Wed, 14 Sep 2016 02:14:58 -0700 Subject: [PATCH] Refactor QualityScaler and MovingAverage The MovingAverage class was very specific to the QualityScaler. This commit generalizes the MovingAverage class to be useful in other situations as well, and adapts the QualityScaler to use the new MovingAverage. BUG=webrtc:6304 Review-Url: https://codereview.webrtc.org/2310853002 Cr-Commit-Position: refs/heads/master@{#14207} --- webrtc/modules/BUILD.gn | 1 + webrtc/modules/modules.gyp | 1 + webrtc/modules/video_coding/BUILD.gn | 1 + .../video_coding/utility/moving_average.cc | 46 ++++++ .../video_coding/utility/moving_average.h | 58 ++----- .../utility/moving_average_unittest.cc | 62 ++++++++ .../video_coding/utility/quality_scaler.cc | 149 +++++++++--------- .../video_coding/utility/quality_scaler.h | 17 +- .../utility/quality_scaler_unittest.cc | 2 +- .../utility/video_coding_utility.gyp | 1 + 10 files changed, 208 insertions(+), 130 deletions(-) create mode 100644 webrtc/modules/video_coding/utility/moving_average.cc create mode 100644 webrtc/modules/video_coding/utility/moving_average_unittest.cc diff --git a/webrtc/modules/BUILD.gn b/webrtc/modules/BUILD.gn index 759b9410cf..70e9a13f61 100644 --- a/webrtc/modules/BUILD.gn +++ b/webrtc/modules/BUILD.gn @@ -465,6 +465,7 @@ if (rtc_include_tests) { "video_coding/utility/frame_dropper_unittest.cc", "video_coding/utility/h264_bitstream_parser_unittest.cc", "video_coding/utility/ivf_file_writer_unittest.cc", + "video_coding/utility/moving_average_unittest.cc", "video_coding/utility/quality_scaler_unittest.cc", "video_coding/utility/simulcast_rate_allocator_unittest.cc", "video_coding/video_coding_robustness_unittest.cc", diff --git a/webrtc/modules/modules.gyp b/webrtc/modules/modules.gyp index f829233e06..c40fafcbd3 100644 --- a/webrtc/modules/modules.gyp +++ b/webrtc/modules/modules.gyp @@ -389,6 +389,7 @@ 'video_coding/utility/frame_dropper_unittest.cc', 'video_coding/utility/h264_bitstream_parser_unittest.cc', 'video_coding/utility/ivf_file_writer_unittest.cc', + 'video_coding/utility/moving_average_unittest.cc', 'video_coding/utility/quality_scaler_unittest.cc', 'video_coding/utility/simulcast_rate_allocator_unittest.cc', 'video_coding/video_coding_robustness_unittest.cc', diff --git a/webrtc/modules/video_coding/BUILD.gn b/webrtc/modules/video_coding/BUILD.gn index b73b6e703f..979355d229 100644 --- a/webrtc/modules/video_coding/BUILD.gn +++ b/webrtc/modules/video_coding/BUILD.gn @@ -103,6 +103,7 @@ rtc_source_set("video_coding_utility") { "utility/h264_bitstream_parser.h", "utility/ivf_file_writer.cc", "utility/ivf_file_writer.h", + "utility/moving_average.cc", "utility/moving_average.h", "utility/qp_parser.cc", "utility/qp_parser.h", diff --git a/webrtc/modules/video_coding/utility/moving_average.cc b/webrtc/modules/video_coding/utility/moving_average.cc new file mode 100644 index 0000000000..34e7bb60e8 --- /dev/null +++ b/webrtc/modules/video_coding/utility/moving_average.cc @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/video_coding/utility/moving_average.h" + +#include + +namespace webrtc { + +MovingAverage::MovingAverage(size_t s) : sum_history_(s + 1, 0) {} + +void MovingAverage::AddSample(int sample) { + count_++; + sum_ += sample; + sum_history_[count_ % sum_history_.size()] = sum_; +} + +rtc::Optional MovingAverage::GetAverage() const { + return GetAverage(size()); +} + +rtc::Optional MovingAverage::GetAverage(size_t num_samples) const { + if (num_samples > size() || num_samples == 0) + return rtc::Optional(); + int sum = sum_ - sum_history_[(count_ - num_samples) % sum_history_.size()]; + return rtc::Optional(sum / static_cast(num_samples)); +} + +void MovingAverage::Reset() { + count_ = 0; + sum_ = 0; + std::fill(sum_history_.begin(), sum_history_.end(), 0); +} + +size_t MovingAverage::size() const { + return std::min(count_, sum_history_.size() - 1); +} + +} // namespace webrtc diff --git a/webrtc/modules/video_coding/utility/moving_average.h b/webrtc/modules/video_coding/utility/moving_average.h index cdad50f0f9..dd42385038 100644 --- a/webrtc/modules/video_coding/utility/moving_average.h +++ b/webrtc/modules/video_coding/utility/moving_average.h @@ -11,63 +11,25 @@ #ifndef WEBRTC_MODULES_VIDEO_CODING_UTILITY_MOVING_AVERAGE_H_ #define WEBRTC_MODULES_VIDEO_CODING_UTILITY_MOVING_AVERAGE_H_ -#include +#include -#include - -#include "webrtc/typedefs.h" +#include "webrtc/base/optional.h" namespace webrtc { -template class MovingAverage { public: - MovingAverage(); - void AddSample(T sample); - bool GetAverage(size_t num_samples, T* average); + explicit MovingAverage(size_t s); + void AddSample(int sample); + rtc::Optional GetAverage() const; + rtc::Optional GetAverage(size_t num_samples) const; void Reset(); - int size(); + size_t size() const; private: - T sum_; - std::list samples_; + size_t count_ = 0; + int sum_ = 0; + std::vector sum_history_; }; - -template -MovingAverage::MovingAverage() - : sum_(static_cast(0)) {} - -template -void MovingAverage::AddSample(T sample) { - samples_.push_back(sample); - sum_ += sample; -} - -template -bool MovingAverage::GetAverage(size_t num_samples, T* avg) { - if (num_samples > samples_.size()) - return false; - - // Remove old samples. - while (num_samples < samples_.size()) { - sum_ -= samples_.front(); - samples_.pop_front(); - } - - *avg = sum_ / static_cast(num_samples); - return true; -} - -template -void MovingAverage::Reset() { - sum_ = static_cast(0); - samples_.clear(); -} - -template -int MovingAverage::size() { - return samples_.size(); -} - } // namespace webrtc #endif // WEBRTC_MODULES_VIDEO_CODING_UTILITY_MOVING_AVERAGE_H_ diff --git a/webrtc/modules/video_coding/utility/moving_average_unittest.cc b/webrtc/modules/video_coding/utility/moving_average_unittest.cc new file mode 100644 index 0000000000..fa32a45e46 --- /dev/null +++ b/webrtc/modules/video_coding/utility/moving_average_unittest.cc @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/video_coding/utility/moving_average.h" + +#include "testing/gtest/include/gtest/gtest.h" + +TEST(MovingAverageTest, EmptyAverage) { + webrtc::MovingAverage moving_average(1); + EXPECT_EQ(0u, moving_average.size()); + EXPECT_FALSE(moving_average.GetAverage(0)); +} + +// Test single value. +TEST(MovingAverageTest, OneElement) { + webrtc::MovingAverage moving_average(1); + moving_average.AddSample(3); + EXPECT_EQ(1u, moving_average.size()); + EXPECT_EQ(3, *moving_average.GetAverage()); + EXPECT_EQ(3, *moving_average.GetAverage(1)); + EXPECT_FALSE(moving_average.GetAverage(2)); +} + +TEST(MovingAverageTest, GetAverage) { + webrtc::MovingAverage moving_average(1024); + moving_average.AddSample(1); + moving_average.AddSample(1); + moving_average.AddSample(3); + moving_average.AddSample(3); + EXPECT_EQ(*moving_average.GetAverage(4), 2); + EXPECT_EQ(*moving_average.GetAverage(2), 3); + EXPECT_FALSE(moving_average.GetAverage(0)); +} + +TEST(MovingAverageTest, Reset) { + webrtc::MovingAverage moving_average(5); + moving_average.AddSample(1); + EXPECT_EQ(1, *moving_average.GetAverage(1)); + moving_average.Reset(); + EXPECT_FALSE(moving_average.GetAverage(1)); + EXPECT_FALSE(moving_average.GetAverage(6)); +} + +TEST(MovingAverageTest, ManySamples) { + webrtc::MovingAverage moving_average(10); + for (int i = 1; i < 11; i++) { + moving_average.AddSample(i); + } + EXPECT_EQ(*moving_average.GetAverage(), 5); + moving_average.Reset(); + for (int i = 1; i < 2001; i++) { + moving_average.AddSample(i); + } + EXPECT_EQ(*moving_average.GetAverage(), 1995); +} diff --git a/webrtc/modules/video_coding/utility/quality_scaler.cc b/webrtc/modules/video_coding/utility/quality_scaler.cc index c509e84364..99bc6dad22 100644 --- a/webrtc/modules/video_coding/utility/quality_scaler.cc +++ b/webrtc/modules/video_coding/utility/quality_scaler.cc @@ -10,10 +10,12 @@ #include "webrtc/modules/video_coding/utility/quality_scaler.h" +#include +#include + namespace webrtc { namespace { -static const int kMinFps = 5; // Threshold constant used until first downscale (to permit fast rampup). static const int kMeasureSecondsFastUpscale = 2; static const int kMeasureSecondsUpscale = 5; @@ -46,7 +48,11 @@ const int QualityScaler::kLowH264QpThreshold = 24; const int QualityScaler::kBadH264QpThreshold = 37; #endif -QualityScaler::QualityScaler() : low_qp_threshold_(-1) {} +// Default values. Should immediately get set to something more sensible. +QualityScaler::QualityScaler() + : average_qp_(kMeasureSecondsUpscale * 30), + framedrop_percent_(kMeasureSecondsUpscale * 30), + low_qp_threshold_(-1) {} void QualityScaler::Init(int low_qp_threshold, int high_qp_threshold, @@ -54,14 +60,15 @@ void QualityScaler::Init(int low_qp_threshold, int width, int height, int fps) { - ClearSamples(); low_qp_threshold_ = low_qp_threshold; high_qp_threshold_ = high_qp_threshold; downscale_shift_ = 0; - // Use a faster window for upscaling initially (but be more graceful later). - // This enables faster initial rampups without risking strong up-down - // behavior later. - measure_seconds_upscale_ = kMeasureSecondsFastUpscale; + + fast_rampup_ = true; + + ClearSamples(); + ReportFramerate(fps); + const int init_width = width; const int init_height = height; if (initial_bitrate_kbps > 0) { @@ -76,24 +83,28 @@ void QualityScaler::Init(int low_qp_threshold, height /= 2; } } - - // Zero out width/height so they can be checked against inside - // UpdateTargetResolution. - res_.width = res_.height = 0; UpdateTargetResolution(init_width, init_height); ReportFramerate(fps); } // Report framerate(fps) to estimate # of samples. void QualityScaler::ReportFramerate(int framerate) { - framerate_ = framerate; - UpdateSampleCounts(); + // Use a faster window for upscaling initially. + // This enables faster initial rampups without risking strong up-down + // behavior later. + num_samples_upscale_ = framerate * (fast_rampup_ ? kMeasureSecondsFastUpscale + : kMeasureSecondsUpscale); + num_samples_downscale_ = framerate * kMeasureSecondsDownscale; + + average_qp_ = + MovingAverage(std::max(num_samples_upscale_, num_samples_downscale_)); + framedrop_percent_ = + MovingAverage(std::max(num_samples_upscale_, num_samples_downscale_)); } void QualityScaler::ReportQP(int qp) { framedrop_percent_.AddSample(0); - average_qp_downscale_.AddSample(qp); - average_qp_upscale_.AddSample(qp); + average_qp_.AddSample(qp); } void QualityScaler::ReportDroppedFrame() { @@ -103,34 +114,58 @@ void QualityScaler::ReportDroppedFrame() { void QualityScaler::OnEncodeFrame(int width, int height) { // Should be set through InitEncode -> Should be set by now. RTC_DCHECK_GE(low_qp_threshold_, 0); - RTC_DCHECK_GT(num_samples_upscale_, 0u); - RTC_DCHECK_GT(num_samples_downscale_, 0u); - - // Update scale factor. - int avg_drop = 0; - int avg_qp = 0; - - if ((framedrop_percent_.GetAverage(num_samples_downscale_, &avg_drop) && - avg_drop >= kFramedropPercentThreshold) || - (average_qp_downscale_.GetAverage(num_samples_downscale_, &avg_qp) && - avg_qp > high_qp_threshold_)) { - AdjustScale(false); - } else if (average_qp_upscale_.GetAverage(num_samples_upscale_, &avg_qp) && - avg_qp <= low_qp_threshold_) { - AdjustScale(true); + if (target_res_.width != width || target_res_.height != height) { + UpdateTargetResolution(width, height); + } + + // Check if we should scale down due to high frame drop. + const auto drop_rate = framedrop_percent_.GetAverage(num_samples_downscale_); + if (drop_rate && *drop_rate >= kFramedropPercentThreshold) { + ScaleDown(); + return; + } + + // Check if we should scale up or down based on QP. + const auto avg_qp_down = average_qp_.GetAverage(num_samples_downscale_); + if (avg_qp_down && *avg_qp_down > high_qp_threshold_) { + ScaleDown(); + return; + } + const auto avg_qp_up = average_qp_.GetAverage(num_samples_upscale_); + if (avg_qp_up && *avg_qp_up <= low_qp_threshold_) { + // QP has been low. We want to try a higher resolution. + ScaleUp(); + return; + } +} + +void QualityScaler::ScaleUp() { + downscale_shift_ = std::max(0, downscale_shift_ - 1); + ClearSamples(); +} + +void QualityScaler::ScaleDown() { + downscale_shift_ = std::min(maximum_shift_, downscale_shift_ + 1); + ClearSamples(); + // If we've scaled down, wait longer before scaling up again. + if (fast_rampup_) { + fast_rampup_ = false; + num_samples_upscale_ = (num_samples_upscale_ / kMeasureSecondsFastUpscale) * + kMeasureSecondsUpscale; } - UpdateTargetResolution(width, height); } QualityScaler::Resolution QualityScaler::GetScaledResolution() const { - return res_; + const int frame_width = target_res_.width >> downscale_shift_; + const int frame_height = target_res_.height >> downscale_shift_; + return Resolution{frame_width, frame_height}; } rtc::scoped_refptr QualityScaler::GetScaledBuffer( const rtc::scoped_refptr& frame) { Resolution res = GetScaledResolution(); - int src_width = frame->width(); - int src_height = frame->height(); + const int src_width = frame->width(); + const int src_height = frame->height(); if (res.width == src_width && res.height == src_height) return frame; @@ -142,50 +177,20 @@ rtc::scoped_refptr QualityScaler::GetScaledBuffer( return scaled_buffer; } -void QualityScaler::UpdateTargetResolution(int frame_width, int frame_height) { - RTC_DCHECK_GE(downscale_shift_, 0); - int shifts_performed = 0; - for (int shift = downscale_shift_; - shift > 0 && (frame_width / 2 >= kMinDownscaleDimension) && - (frame_height / 2 >= kMinDownscaleDimension); - --shift, ++shifts_performed) { - frame_width /= 2; - frame_height /= 2; +void QualityScaler::UpdateTargetResolution(int width, int height) { + if (width < kMinDownscaleDimension || height < kMinDownscaleDimension) { + maximum_shift_ = 0; + } else { + maximum_shift_ = static_cast( + std::log2(std::min(width, height) / kMinDownscaleDimension)); } - // Clamp to number of shifts actually performed to not be stuck trying to - // scale way beyond QVGA. - downscale_shift_ = shifts_performed; - if (res_.width == frame_width && res_.height == frame_height) { - // No reset done/needed, using same resolution. - return; - } - res_.width = frame_width; - res_.height = frame_height; - ClearSamples(); + target_res_ = Resolution{width, height}; } void QualityScaler::ClearSamples() { framedrop_percent_.Reset(); - average_qp_downscale_.Reset(); - average_qp_upscale_.Reset(); + average_qp_.Reset(); } -void QualityScaler::UpdateSampleCounts() { - num_samples_downscale_ = static_cast( - kMeasureSecondsDownscale * (framerate_ < kMinFps ? kMinFps : framerate_)); - num_samples_upscale_ = static_cast( - measure_seconds_upscale_ * (framerate_ < kMinFps ? kMinFps : framerate_)); -} - -void QualityScaler::AdjustScale(bool up) { - downscale_shift_ += up ? -1 : 1; - if (downscale_shift_ < 0) - downscale_shift_ = 0; - if (!up) { - // First downscale hit, start using a slower threshold for going up. - measure_seconds_upscale_ = kMeasureSecondsUpscale; - UpdateSampleCounts(); - } -} } // namespace webrtc diff --git a/webrtc/modules/video_coding/utility/quality_scaler.h b/webrtc/modules/video_coding/utility/quality_scaler.h index 7176d4966e..c1ae50b06e 100644 --- a/webrtc/modules/video_coding/utility/quality_scaler.h +++ b/webrtc/modules/video_coding/utility/quality_scaler.h @@ -48,26 +48,25 @@ class QualityScaler { static const int kBadH264QpThreshold; private: - void AdjustScale(bool up); - void UpdateTargetResolution(int frame_width, int frame_height); void ClearSamples(); - void UpdateSampleCounts(); + void ScaleUp(); + void ScaleDown(); + void UpdateTargetResolution(int width, int height); I420BufferPool pool_; size_t num_samples_downscale_; size_t num_samples_upscale_; - int measure_seconds_upscale_; - MovingAverage average_qp_upscale_; - MovingAverage average_qp_downscale_; + bool fast_rampup_; + MovingAverage average_qp_; + MovingAverage framedrop_percent_; - int framerate_; int low_qp_threshold_; int high_qp_threshold_; - MovingAverage framedrop_percent_; - Resolution res_; + Resolution target_res_; int downscale_shift_; + int maximum_shift_; }; } // namespace webrtc diff --git a/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc b/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc index 27ee25cd94..2bbd26d760 100644 --- a/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc +++ b/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc @@ -43,7 +43,7 @@ class QualityScalerTest : public ::testing::Test { QualityScalerTest() { input_frame_ = I420Buffer::Create(kWidth, kHeight); - qs_.Init(kLowQpThreshold, kHighQp, 0, 0, 0, kFramerate); + qs_.Init(kLowQpThreshold, kHighQp, 0, kWidth, kHeight, kFramerate); qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); } diff --git a/webrtc/modules/video_coding/utility/video_coding_utility.gyp b/webrtc/modules/video_coding/utility/video_coding_utility.gyp index c189abcfda..e4ed78f211 100644 --- a/webrtc/modules/video_coding/utility/video_coding_utility.gyp +++ b/webrtc/modules/video_coding/utility/video_coding_utility.gyp @@ -25,6 +25,7 @@ 'h264_bitstream_parser.h', 'ivf_file_writer.cc', 'ivf_file_writer.h', + 'moving_average.cc', 'moving_average.h', 'qp_parser.cc', 'qp_parser.h',