zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Asynchronous QualityScaler: Callback-based CheckQpTask.

Browse Source 
This CL breaks up the CheckQp() operation into several steps managed
by the inner helper class CheckQpTask, making responding to high or
low QP an asynchronous operation. Why? Reconfiguring the stream in
response to QP overuse will in the future be handled on a separate
task queue. See Call-Level Adaptation Processing for more details:
https://docs.google.com/document/d/1ZyC26yOCknrrcYa839ZWLxD6o6Gig5A3lVTh4E41074/edit?usp=sharing

Instead of "bool AdaptDown()" when high QP is reported,
synchronously returning true or false depending on the result of
adaptation, this CL introduces
  void QualityScalerQpUsageHandlerInterface::OnReportQpUsageHigh(
      rtc::scoped_refptr<QualityScalerQpUsageHandlerCallback>);
Where
  QualityScalerQpUsageHandlerCallback::OnQpUsageHandled(
      bool clear_qp_samples);
Instructs the QualityScaler whether to clear samples before
checking QP the next time or to increase the frequency of checking
(corresponding to AdaptDown's return value prior to this CL).

QualityScaler no longer using AdaptationObserverInterface, this class
is renamed and moved to overuse_frame_detector.h.

The dependency between CheckQpTasks is made explicit with
CheckQpTask::Result and variables like observed_enough_frames_,
adapt_called_ and adapt_failed_ are moved there and given more
descriptive names.

Bug: webrtc:11521
Change-Id: I7faf795aeee5ded18ce75eb1617f88226e337228
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/173760
Reviewed-by: Evan Shrubsole <eshr@google.com>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Henrik Boström <hbos@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#31140}
This commit is contained in:
Henrik Boström
2020-04-27 17:40:55 +02:00
committed by Commit Bot
parent 9abc6bd8aa
commit 012aa375b1
11 changed files with 482 additions and 256 deletions
Download Patch File Download Diff File Expand all files Collapse all files

276
modules/video_coding/utility/quality_scaler.cc
View File

@ -19,6 +19,8 @@
#include "rtc_base/logging.h"
#include "rtc_base/numerics/exp_filter.h"
#include "rtc_base/task_queue.h"
#include "rtc_base/task_utils/to_queued_task.h"
#include "rtc_base/weak_ptr.h"
// TODO(kthelgason): Some versions of Android have issues with log2.
// See https://code.google.com/p/android/issues/detail?id=212634 for details
@ -69,15 +71,192 @@ class QualityScaler::QpSmoother {
rtc::ExpFilter smoother_;
};
QualityScaler::QualityScaler(AdaptationObserverInterface* observer,
// The QualityScaler checks for QP periodically by queuing CheckQpTasks. The
// task will either run to completion and trigger a new task being queued, or it
// will be destroyed because the QualityScaler is destroyed.
//
// When high or low QP is reported, the task will be pending until a callback is
// invoked. This lets the QualityScalerQpUsageHandlerInterface react to QP usage
// asynchronously and prevents checking for QP until the stream has potentially
// been reconfigured.
class QualityScaler::CheckQpTask {
public:
// The result of one CheckQpTask may influence the delay of the next
// CheckQpTask.
struct Result {
bool observed_enough_frames = false;
bool qp_usage_reported = false;
bool clear_qp_samples = false;
};
CheckQpTask(QualityScaler* quality_scaler, Result previous_task_result)
: quality_scaler_(quality_scaler),
state_(State::kNotStarted),
previous_task_result_(previous_task_result),
weak_ptr_factory_(this) {}
void StartDelayedTask() {
RTC_DCHECK_EQ(state_, State::kNotStarted);
state_ = State::kCheckingQp;
TaskQueueBase::Current()->PostDelayedTask(
ToQueuedTask([this_weak_ptr = weak_ptr_factory_.GetWeakPtr(), this] {
if (!this_weak_ptr) {
// The task has been cancelled through destruction.
return;
}
RTC_DCHECK_EQ(state_, State::kCheckingQp);
RTC_DCHECK_RUN_ON(&quality_scaler_->task_checker_);
switch (quality_scaler_->CheckQp()) {
case QualityScaler::CheckQpResult::kInsufficientSamples: {
result_.observed_enough_frames = false;
// After this line, |this| may be deleted.
DoCompleteTask();
return;
}
case QualityScaler::CheckQpResult::kNormalQp: {
result_.observed_enough_frames = true;
// After this line, |this| may be deleted.
DoCompleteTask();
return;
}
case QualityScaler::CheckQpResult::kHighQp: {
result_.observed_enough_frames = true;
result_.qp_usage_reported = true;
state_ = State::kAwaitingQpUsageHandled;
rtc::scoped_refptr<QualityScalerQpUsageHandlerCallbackInterface>
callback = ConstructCallback();
quality_scaler_->fast_rampup_ = false;
// After this line, |this| may be deleted.
quality_scaler_->handler_->OnReportQpUsageHigh(callback);
return;
}
case QualityScaler::CheckQpResult::kLowQp: {
result_.observed_enough_frames = true;
result_.qp_usage_reported = true;
state_ = State::kAwaitingQpUsageHandled;
rtc::scoped_refptr<QualityScalerQpUsageHandlerCallbackInterface>
callback = ConstructCallback();
// After this line, |this| may be deleted.
quality_scaler_->handler_->OnReportQpUsageLow(callback);
return;
}
}
}),
GetCheckingQpDelayMs());
}
void OnQpUsageHandled(bool clear_qp_samples) {
RTC_DCHECK_EQ(state_, State::kAwaitingQpUsageHandled);
result_.clear_qp_samples = clear_qp_samples;
if (clear_qp_samples)
quality_scaler_->ClearSamples();
DoCompleteTask();
}
bool HasCompletedTask() const { return state_ == State::kCompleted; }
Result result() const {
RTC_DCHECK(HasCompletedTask());
return result_;
}
private:
enum class State {
kNotStarted,
kCheckingQp,
kAwaitingQpUsageHandled,
kCompleted,
};
// Defined after the definition of QualityScaler::CheckQpTaskHandlerCallback.
// Gets around a forward declaration issue.
rtc::scoped_refptr<QualityScaler::CheckQpTaskHandlerCallback>
ConstructCallback();
// Determines the sampling period of CheckQpTasks.
int64_t GetCheckingQpDelayMs() const {
RTC_DCHECK_RUN_ON(&quality_scaler_->task_checker_);
if (quality_scaler_->fast_rampup_) {
return quality_scaler_->sampling_period_ms_;
}
if (quality_scaler_->experiment_enabled_ &&
!previous_task_result_.observed_enough_frames) {
// Use half the interval while waiting for enough frames.
return quality_scaler_->sampling_period_ms_ / 2;
}
if (!previous_task_result_.clear_qp_samples) {
// Check shortly again.
return quality_scaler_->sampling_period_ms_ / 8;
}
if (quality_scaler_->scale_factor_ &&
!previous_task_result_.qp_usage_reported) {
// Last CheckQp did not call AdaptDown/Up, possibly reduce interval.
return quality_scaler_->sampling_period_ms_ *
quality_scaler_->scale_factor_.value();
}
return quality_scaler_->sampling_period_ms_ *
quality_scaler_->initial_scale_factor_;
}
void DoCompleteTask() {
RTC_DCHECK(state_ == State::kCheckingQp ||
state_ == State::kAwaitingQpUsageHandled);
state_ = State::kCompleted;
// Starting the next task deletes the pending task. After this line, |this|
// has been deleted.
quality_scaler_->StartNextCheckQpTask();
}
QualityScaler* const quality_scaler_;
State state_;
const Result previous_task_result_;
Result result_;
rtc::WeakPtrFactory<CheckQpTask> weak_ptr_factory_;
};
class QualityScaler::CheckQpTaskHandlerCallback
: public QualityScalerQpUsageHandlerCallbackInterface {
public:
CheckQpTaskHandlerCallback(
rtc::WeakPtr<QualityScaler::CheckQpTask> check_qp_task)
: QualityScalerQpUsageHandlerCallbackInterface(),
check_qp_task_(std::move(check_qp_task)),
was_handled_(false) {}
~CheckQpTaskHandlerCallback() { RTC_DCHECK(was_handled_); }
void OnQpUsageHandled(bool clear_qp_samples) {
RTC_DCHECK(!was_handled_);
was_handled_ = true;
if (!check_qp_task_) {
// The task has been cancelled through destruction; the result of the
// operation is ignored.
return;
}
check_qp_task_->OnQpUsageHandled(clear_qp_samples);
}
private:
// The callback may outlive the QualityScaler and its task.
rtc::WeakPtr<QualityScaler::CheckQpTask> const check_qp_task_;
bool was_handled_;
};
rtc::scoped_refptr<QualityScaler::CheckQpTaskHandlerCallback>
QualityScaler::CheckQpTask::ConstructCallback() {
return new CheckQpTaskHandlerCallback(weak_ptr_factory_.GetWeakPtr());
}
QualityScaler::QualityScaler(QualityScalerQpUsageHandlerInterface* handler,
VideoEncoder::QpThresholds thresholds)
: QualityScaler(observer, thresholds, kMeasureMs) {}
: QualityScaler(handler, thresholds, kMeasureMs) {}
// Protected ctor, should not be called directly.
QualityScaler::QualityScaler(AdaptationObserverInterface* observer,
QualityScaler::QualityScaler(QualityScalerQpUsageHandlerInterface* handler,
VideoEncoder::QpThresholds thresholds,
int64_t sampling_period_ms)
: observer_(observer),
: handler_(handler),
thresholds_(thresholds),
sampling_period_ms_(sampling_period_ms),
fast_rampup_(true),
@ -86,7 +265,6 @@ QualityScaler::QualityScaler(AdaptationObserverInterface* observer,
framedrop_percent_media_opt_(5 * 30),
framedrop_percent_all_(5 * 30),
experiment_enabled_(QualityScalingExperiment::Enabled()),
observed_enough_frames_(false),
min_frames_needed_(
QualityScalerSettings::ParseFromFieldTrials().MinFrames().value_or(
kMinFramesNeededToScale)),
@ -94,49 +272,33 @@ QualityScaler::QualityScaler(AdaptationObserverInterface* observer,
.InitialScaleFactor()
.value_or(kSamplePeriodScaleFactor)),
scale_factor_(
QualityScalerSettings::ParseFromFieldTrials().ScaleFactor()),
adapt_called_(false),
adapt_failed_(false) {
QualityScalerSettings::ParseFromFieldTrials().ScaleFactor()) {
RTC_DCHECK_RUN_ON(&task_checker_);
if (experiment_enabled_) {
config_ = QualityScalingExperiment::GetConfig();
qp_smoother_high_.reset(new QpSmoother(config_.alpha_high));
qp_smoother_low_.reset(new QpSmoother(config_.alpha_low));
}
RTC_DCHECK(observer_ != nullptr);
check_qp_task_ = RepeatingTaskHandle::DelayedStart(
TaskQueueBase::Current(), TimeDelta::Millis(GetSamplingPeriodMs()),
[this]() {
CheckQp();
return TimeDelta::Millis(GetSamplingPeriodMs());
});
RTC_DCHECK(handler_ != nullptr);
StartNextCheckQpTask();
RTC_LOG(LS_INFO) << "QP thresholds: low: " << thresholds_.low
<< ", high: " << thresholds_.high;
}
QualityScaler::~QualityScaler() {
RTC_DCHECK_RUN_ON(&task_checker_);
check_qp_task_.Stop();
}
int64_t QualityScaler::GetSamplingPeriodMs() const {
void QualityScaler::StartNextCheckQpTask() {
RTC_DCHECK_RUN_ON(&task_checker_);
if (fast_rampup_) {
return sampling_period_ms_;
RTC_DCHECK(!pending_qp_task_ || pending_qp_task_->HasCompletedTask())
<< "A previous CheckQpTask has not completed yet!";
CheckQpTask::Result previous_task_result;
if (pending_qp_task_) {
previous_task_result = pending_qp_task_->result();
}
if (experiment_enabled_ && !observed_enough_frames_) {
// Use half the interval while waiting for enough frames.
return sampling_period_ms_ / 2;
}
if (adapt_failed_) {
// Check shortly again.
return sampling_period_ms_ / 8;
}
if (scale_factor_ && !adapt_called_) {
// Last CheckQp did not call AdaptDown/Up, possibly reduce interval.
return sampling_period_ms_ * scale_factor_.value();
}
return sampling_period_ms_ * initial_scale_factor_;
pending_qp_task_ = std::make_unique<CheckQpTask>(this, previous_task_result);
pending_qp_task_->StartDelayedTask();
}
void QualityScaler::SetQpThresholds(VideoEncoder::QpThresholds thresholds) {
@ -181,12 +343,10 @@ bool QualityScaler::QpFastFilterLow() const {
return (avg_qp_high) ? (avg_qp_high.value() <= thresholds_.low) : false;
}
void QualityScaler::CheckQp() {
QualityScaler::CheckQpResult QualityScaler::CheckQp() const {
RTC_DCHECK_RUN_ON(&task_checker_);
// Should be set through InitEncode -> Should be set by now.
RTC_DCHECK_GE(thresholds_.low, 0);
adapt_failed_ = false;
adapt_called_ = false;
// If we have not observed at least this many frames we can't make a good
// scaling decision.
@ -194,10 +354,8 @@ void QualityScaler::CheckQp() {
? framedrop_percent_all_.Size()
: framedrop_percent_media_opt_.Size();
if (frames < min_frames_needed_) {
observed_enough_frames_ = false;
return;
return CheckQpResult::kInsufficientSamples;
}
observed_enough_frames_ = true;
// Check if we should scale down due to high frame drop.
const absl::optional<int> drop_rate =
@ -206,8 +364,7 @@ void QualityScaler::CheckQp() {
: framedrop_percent_media_opt_.GetAverageRoundedDown();
if (drop_rate && *drop_rate >= kFramedropPercentThreshold) {
RTC_LOG(LS_INFO) << "Reporting high QP, framedrop percent " << *drop_rate;
ReportQpHigh();
return;
return CheckQpResult::kHighQp;
}
// Check if we should scale up or down based on QP.
@ -221,38 +378,14 @@ void QualityScaler::CheckQp() {
RTC_LOG(LS_INFO) << "Checking average QP " << *avg_qp_high << " ("
<< *avg_qp_low << ").";
if (*avg_qp_high > thresholds_.high) {
ReportQpHigh();
return;
return CheckQpResult::kHighQp;
}
if (*avg_qp_low <= thresholds_.low) {
// QP has been low. We want to try a higher resolution.
ReportQpLow();
return;
return CheckQpResult::kLowQp;
}
}
}
void QualityScaler::ReportQpLow() {
RTC_DCHECK_RUN_ON(&task_checker_);
ClearSamples();
observer_->AdaptUp(VideoAdaptationReason::kQuality);
adapt_called_ = true;
}
void QualityScaler::ReportQpHigh() {
RTC_DCHECK_RUN_ON(&task_checker_);
if (observer_->AdaptDown(VideoAdaptationReason::kQuality)) {
ClearSamples();
} else {
adapt_failed_ = true;
}
// If we've scaled down, wait longer before scaling up again.
if (fast_rampup_) {
fast_rampup_ = false;
}
adapt_called_ = true;
return CheckQpResult::kNormalQp;
}
void QualityScaler::ClearSamples() {
@ -265,4 +398,13 @@ void QualityScaler::ClearSamples() {
if (qp_smoother_low_)
qp_smoother_low_->Reset();
}
QualityScalerQpUsageHandlerInterface::~QualityScalerQpUsageHandlerInterface() {}
QualityScalerQpUsageHandlerCallbackInterface::
QualityScalerQpUsageHandlerCallbackInterface() {}
QualityScalerQpUsageHandlerCallbackInterface::
~QualityScalerQpUsageHandlerCallbackInterface() {}
} // namespace webrtc