zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make quality scaler downscale faster.

Browse Source 
Include dropped frames by the encoder in the frame drop percentage.

To react faster at low framerates:
- Use ExpFilter instead of MovingAverage to filter QP values.
- Reduce sampling interval while waiting for minimum number of needed frames (when not in fast rampup mode).

A separate slower ExpFilter is used for upscaling.

Bug: webrtc:9169
Change-Id: If7ff6c3bd4201fda2da67125889838fe96ce7061
Reviewed-on: https://webrtc-review.googlesource.com/70761
Commit-Queue: Åsa Persson <asapersson@webrtc.org>
Reviewed-by: Karl Wiberg <kwiberg@webrtc.org>
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#23014}
This commit is contained in:
Åsa Persson
2018-04-24 16:53:25 +02:00
committed by Commit Bot
parent c79268f15a
commit a945aee72e
11 changed files with 504 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
modules/video_coding/utility/quality_scaler.cc
View File

@ -17,7 +17,9 @@
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/numerics/exp_filter.h"
#include "rtc_base/task_queue.h"
#include "rtc_base/timeutils.h"
// TODO(kthelgason): Some versions of Android have issues with log2.
// See https://code.google.com/p/android/issues/detail?id=212634 for details
@ -37,6 +39,33 @@ static const int kMinFramesNeededToScale = 2 * 30;
} // namespace
class QualityScaler::QpSmoother {
public:
explicit QpSmoother(float alpha)
: alpha_(alpha), last_sample_ms_(rtc::TimeMillis()), smoother_(alpha) {}
rtc::Optional<int> GetAvg() const {
float value = smoother_.filtered();
if (value == rtc::ExpFilter::kValueUndefined) {
return rtc::nullopt;
}
return static_cast<int>(value);
}
void Add(float sample) {
int64_t now_ms = rtc::TimeMillis();
smoother_.Apply(static_cast<float>(now_ms - last_sample_ms_), sample);
last_sample_ms_ = now_ms;
}
void Reset() { smoother_.Reset(alpha_); }
private:
const float alpha_;
int64_t last_sample_ms_;
rtc::ExpFilter smoother_;
};
class QualityScaler::CheckQpTask : public rtc::QueuedTask {
public:
explicit CheckQpTask(QualityScaler* scaler) : scaler_(scaler) {
@ -81,8 +110,16 @@ QualityScaler::QualityScaler(AdaptationObserverInterface* observer,
fast_rampup_(true),
// Arbitrarily choose size based on 30 fps for 5 seconds.
average_qp_(5 * 30),
framedrop_percent_(5 * 30) {
framedrop_percent_media_opt_(5 * 30),
framedrop_percent_all_(5 * 30),
experiment_enabled_(QualityScalingExperiment::Enabled()),
observed_enough_frames_(false) {
RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_);
if (experiment_enabled_) {
config_ = QualityScalingExperiment::GetConfig();
qp_smoother_high_.reset(new QpSmoother(config_.alpha_high));
qp_smoother_low_.reset(new QpSmoother(config_.alpha_low));
}
RTC_DCHECK(observer_ != nullptr);
check_qp_task_ = new CheckQpTask(this);
RTC_LOG(LS_INFO) << "QP thresholds: low: " << thresholds_.low
@ -96,19 +133,36 @@ QualityScaler::~QualityScaler() {
int64_t QualityScaler::GetSamplingPeriodMs() const {
RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_);
return fast_rampup_ ? sampling_period_ms_
: (sampling_period_ms_ * kSamplePeriodScaleFactor);
if (fast_rampup_) {
return sampling_period_ms_;
}
if (experiment_enabled_ && !observed_enough_frames_) {
// Use half the interval while waiting for enough frames.
return sampling_period_ms_ / 2;
}
return sampling_period_ms_ * kSamplePeriodScaleFactor;
}
void QualityScaler::ReportDroppedFrame() {
void QualityScaler::ReportDroppedFrameByMediaOpt() {
RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_);
framedrop_percent_.AddSample(100);
framedrop_percent_media_opt_.AddSample(100);
framedrop_percent_all_.AddSample(100);
}
void QualityScaler::ReportDroppedFrameByEncoder() {
RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_);
framedrop_percent_all_.AddSample(100);
}
void QualityScaler::ReportQp(int qp) {
RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_);
framedrop_percent_.AddSample(0);
framedrop_percent_media_opt_.AddSample(0);
framedrop_percent_all_.AddSample(0);
average_qp_.AddSample(qp);
if (qp_smoother_high_)
qp_smoother_high_->Add(qp);
if (qp_smoother_low_)
qp_smoother_low_->Add(qp);
}
void QualityScaler::CheckQp() {
@ -118,11 +172,19 @@ void QualityScaler::CheckQp() {
// If we have not observed at least this many frames we can't make a good
// scaling decision.
if (framedrop_percent_.size() < kMinFramesNeededToScale)
const size_t frames = config_.use_all_drop_reasons
? framedrop_percent_all_.size()
: framedrop_percent_media_opt_.size();
if (frames < kMinFramesNeededToScale) {
observed_enough_frames_ = false;
return;
}
observed_enough_frames_ = true;
// Check if we should scale down due to high frame drop.
const rtc::Optional<int> drop_rate = framedrop_percent_.GetAverage();
const rtc::Optional<int> drop_rate =
config_.use_all_drop_reasons ? framedrop_percent_all_.GetAverage()
: framedrop_percent_media_opt_.GetAverage();
if (drop_rate && *drop_rate >= kFramedropPercentThreshold) {
RTC_LOG(LS_INFO) << "Reporting high QP, framedrop percent " << *drop_rate;
ReportQpHigh();
@ -130,14 +192,19 @@ void QualityScaler::CheckQp() {
}
// Check if we should scale up or down based on QP.
const rtc::Optional<int> avg_qp = average_qp_.GetAverage();
if (avg_qp) {
RTC_LOG(LS_INFO) << "Checking average QP " << *avg_qp;
if (*avg_qp > thresholds_.high) {
const rtc::Optional<int> avg_qp_high = qp_smoother_high_
? qp_smoother_high_->GetAvg()
: average_qp_.GetAverage();
const rtc::Optional<int> avg_qp_low =
qp_smoother_low_ ? qp_smoother_low_->GetAvg() : average_qp_.GetAverage();
if (avg_qp_high && avg_qp_low) {
RTC_LOG(LS_INFO) << "Checking average QP " << *avg_qp_high << " ("
<< *avg_qp_low << ").";
if (*avg_qp_high > thresholds_.high) {
ReportQpHigh();
return;
}
if (*avg_qp <= thresholds_.low) {
if (*avg_qp_low <= thresholds_.low) {
// QP has been low. We want to try a higher resolution.
ReportQpLow();
return;
@ -163,7 +230,12 @@ void QualityScaler::ReportQpHigh() {
void QualityScaler::ClearSamples() {
RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_);
framedrop_percent_.Reset();
framedrop_percent_media_opt_.Reset();
framedrop_percent_all_.Reset();
average_qp_.Reset();
if (qp_smoother_high_)
qp_smoother_high_->Reset();
if (qp_smoother_low_)
qp_smoother_low_->Reset();
}
} // namespace webrtc

18
modules/video_coding/utility/quality_scaler.h
View File

@ -11,12 +11,14 @@
#ifndef MODULES_VIDEO_CODING_UTILITY_QUALITY_SCALER_H_
#define MODULES_VIDEO_CODING_UTILITY_QUALITY_SCALER_H_
#include <memory>
#include <utility>
#include "api/optional.h"
#include "api/video_codecs/video_encoder.h"
#include "common_types.h" // NOLINT(build/include)
#include "modules/video_coding/utility/moving_average.h"
#include "rtc_base/experiments/quality_scaling_experiment.h"
#include "rtc_base/sequenced_task_checker.h"
namespace webrtc {
@ -49,8 +51,9 @@ class QualityScaler {
QualityScaler(AdaptationObserverInterface* observer,
VideoEncoder::QpThresholds thresholds);
virtual ~QualityScaler();
// Should be called each time the encoder drops a frame.
void ReportDroppedFrame();
// Should be called each time a frame is dropped at encoding.
void ReportDroppedFrameByMediaOpt();
void ReportDroppedFrameByEncoder();
// Inform the QualityScaler of the last seen QP.
void ReportQp(int qp);
@ -62,6 +65,7 @@ class QualityScaler {
private:
class CheckQpTask;
class QpSmoother;
void CheckQp();
void ClearSamples();
void ReportQpLow();
@ -76,7 +80,15 @@ class QualityScaler {
const int64_t sampling_period_ms_;
bool fast_rampup_ RTC_GUARDED_BY(&task_checker_);
MovingAverage average_qp_ RTC_GUARDED_BY(&task_checker_);
MovingAverage framedrop_percent_ RTC_GUARDED_BY(&task_checker_);
MovingAverage framedrop_percent_media_opt_ RTC_GUARDED_BY(&task_checker_);
MovingAverage framedrop_percent_all_ RTC_GUARDED_BY(&task_checker_);
// Used by QualityScalingExperiment.
const bool experiment_enabled_;
QualityScalingExperiment::Config config_ RTC_GUARDED_BY(&task_checker_);
std::unique_ptr<QpSmoother> qp_smoother_high_ RTC_GUARDED_BY(&task_checker_);
std::unique_ptr<QpSmoother> qp_smoother_low_ RTC_GUARDED_BY(&task_checker_);
bool observed_enough_frames_ RTC_GUARDED_BY(&task_checker_);
};
} // namespace webrtc

58
modules/video_coding/utility/quality_scaler_unittest.cc
View File

@ -11,9 +11,11 @@
#include "modules/video_coding/utility/quality_scaler.h"
#include <memory>
#include <string>
#include "rtc_base/event.h"
#include "rtc_base/task_queue.h"
#include "test/field_trial.h"
#include "test/gmock.h"
#include "test/gtest.h"
@ -63,7 +65,8 @@ class QualityScalerUnderTest : public QualityScaler {
: QualityScaler(observer, thresholds, 5) {}
};
class QualityScalerTest : public ::testing::Test {
class QualityScalerTest : public ::testing::Test,
public ::testing::WithParamInterface<std::string> {
protected:
enum ScaleDirection {
kKeepScaleAboveLowQp,
@ -74,7 +77,8 @@ class QualityScalerTest : public ::testing::Test {
};
QualityScalerTest()
: q_(new rtc::TaskQueue("QualityScalerTestQueue")),
: scoped_field_trial_(GetParam()),
q_(new rtc::TaskQueue("QualityScalerTestQueue")),
observer_(new MockAdaptationObserver()) {
DO_SYNC(q_, {
qs_ = std::unique_ptr<QualityScaler>(new QualityScalerUnderTest(
@ -96,7 +100,7 @@ class QualityScalerTest : public ::testing::Test {
qs_->ReportQp(kLowQp);
break;
case kScaleDown:
qs_->ReportDroppedFrame();
qs_->ReportDroppedFrameByMediaOpt();
break;
case kKeepScaleAtHighQp:
qs_->ReportQp(kHighQp);
@ -108,37 +112,45 @@ class QualityScalerTest : public ::testing::Test {
}
}
test::ScopedFieldTrials scoped_field_trial_;
std::unique_ptr<rtc::TaskQueue> q_;
std::unique_ptr<QualityScaler> qs_;
std::unique_ptr<MockAdaptationObserver> observer_;
};
TEST_F(QualityScalerTest, DownscalesAfterContinuousFramedrop) {
INSTANTIATE_TEST_CASE_P(
FieldTrials,
QualityScalerTest,
::testing::Values(
"WebRTC-Video-QualityScaling/Enabled-1,2,3,4,5,6,7,8,0.9,0.99,1/",
""));
TEST_P(QualityScalerTest, DownscalesAfterContinuousFramedrop) {
DO_SYNC(q_, { TriggerScale(kScaleDown); });
EXPECT_TRUE(observer_->event.Wait(kDefaultTimeoutMs));
EXPECT_EQ(1, observer_->adapt_down_events_);
EXPECT_EQ(0, observer_->adapt_up_events_);
}
TEST_F(QualityScalerTest, KeepsScaleAtHighQp) {
TEST_P(QualityScalerTest, KeepsScaleAtHighQp) {
DO_SYNC(q_, { TriggerScale(kKeepScaleAtHighQp); });
EXPECT_FALSE(observer_->event.Wait(kDefaultTimeoutMs));
EXPECT_EQ(0, observer_->adapt_down_events_);
EXPECT_EQ(0, observer_->adapt_up_events_);
}
TEST_F(QualityScalerTest, DownscalesAboveHighQp) {
TEST_P(QualityScalerTest, DownscalesAboveHighQp) {
DO_SYNC(q_, { TriggerScale(kScaleDownAboveHighQp); });
EXPECT_TRUE(observer_->event.Wait(kDefaultTimeoutMs));
EXPECT_EQ(1, observer_->adapt_down_events_);
EXPECT_EQ(0, observer_->adapt_up_events_);
}
TEST_F(QualityScalerTest, DownscalesAfterTwoThirdsFramedrop) {
TEST_P(QualityScalerTest, DownscalesAfterTwoThirdsFramedrop) {
DO_SYNC(q_, {
for (int i = 0; i < kFramerate * 5; ++i) {
qs_->ReportDroppedFrame();
qs_->ReportDroppedFrame();
qs_->ReportDroppedFrameByMediaOpt();
qs_->ReportDroppedFrameByMediaOpt();
qs_->ReportQp(kHighQp);
}
});
@ -147,10 +159,10 @@ TEST_F(QualityScalerTest, DownscalesAfterTwoThirdsFramedrop) {
EXPECT_EQ(0, observer_->adapt_up_events_);
}
TEST_F(QualityScalerTest, DoesNotDownscaleAfterHalfFramedrop) {
TEST_P(QualityScalerTest, DoesNotDownscaleAfterHalfFramedrop) {
DO_SYNC(q_, {
for (int i = 0; i < kFramerate * 5; ++i) {
qs_->ReportDroppedFrame();
qs_->ReportDroppedFrameByMediaOpt();
qs_->ReportQp(kHighQp);
}
});
@ -159,21 +171,35 @@ TEST_F(QualityScalerTest, DoesNotDownscaleAfterHalfFramedrop) {
EXPECT_EQ(0, observer_->adapt_up_events_);
}
TEST_F(QualityScalerTest, KeepsScaleOnNormalQp) {
TEST_P(QualityScalerTest, DownscalesAfterTwoThirdsIfFieldTrialEnabled) {
const bool kDownScaleExpected = !GetParam().empty();
DO_SYNC(q_, {
for (int i = 0; i < kFramerate * 5; ++i) {
qs_->ReportDroppedFrameByMediaOpt();
qs_->ReportDroppedFrameByEncoder();
qs_->ReportQp(kHighQp);
}
});
EXPECT_EQ(kDownScaleExpected, observer_->event.Wait(kDefaultTimeoutMs));
EXPECT_EQ(kDownScaleExpected ? 1 : 0, observer_->adapt_down_events_);
EXPECT_EQ(0, observer_->adapt_up_events_);
}
TEST_P(QualityScalerTest, KeepsScaleOnNormalQp) {
DO_SYNC(q_, { TriggerScale(kKeepScaleAboveLowQp); });
EXPECT_FALSE(observer_->event.Wait(kDefaultTimeoutMs));
EXPECT_EQ(0, observer_->adapt_down_events_);
EXPECT_EQ(0, observer_->adapt_up_events_);
}
TEST_F(QualityScalerTest, UpscalesAfterLowQp) {
TEST_P(QualityScalerTest, UpscalesAfterLowQp) {
DO_SYNC(q_, { TriggerScale(kScaleUp); });
EXPECT_TRUE(observer_->event.Wait(kDefaultTimeoutMs));
EXPECT_EQ(0, observer_->adapt_down_events_);
EXPECT_EQ(1, observer_->adapt_up_events_);
}
TEST_F(QualityScalerTest, ScalesDownAndBackUp) {
TEST_P(QualityScalerTest, ScalesDownAndBackUp) {
DO_SYNC(q_, { TriggerScale(kScaleDown); });
EXPECT_TRUE(observer_->event.Wait(kDefaultTimeoutMs));
EXPECT_EQ(1, observer_->adapt_down_events_);
@ -184,7 +210,7 @@ TEST_F(QualityScalerTest, ScalesDownAndBackUp) {
EXPECT_EQ(1, observer_->adapt_up_events_);
}
TEST_F(QualityScalerTest, DoesNotScaleUntilEnoughFramesObserved) {
TEST_P(QualityScalerTest, DoesNotScaleUntilEnoughFramesObserved) {
DO_SYNC(q_, {
// Not enough frames to make a decision.
for (int i = 0; i < kMinFramesNeededToScale - 1; ++i) {
@ -210,7 +236,7 @@ TEST_F(QualityScalerTest, DoesNotScaleUntilEnoughFramesObserved) {
EXPECT_EQ(1, observer_->adapt_up_events_);
}
TEST_F(QualityScalerTest, ScalesDownAndBackUpWithMinFramesNeeded) {
TEST_P(QualityScalerTest, ScalesDownAndBackUpWithMinFramesNeeded) {
DO_SYNC(q_, {
for (int i = 0; i < kMinFramesNeededToScale; ++i) {
qs_->ReportQp(kHighQp + 1);