zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move quality scaler into adaptation module

Browse Source 
This allows for further refactoring, eventually moving
all of quality scaler out of video stream encoder.

Bug: webrtc:11222
Change-Id: Id121608da56f57549a616ccc5f141bb598668b40
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/167728
Commit-Queue: Evan Shrubsole <eshr@google.com>
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30417}
This commit is contained in:
Evan Shrubsole
2020-01-29 17:04:44 +01:00
committed by Commit Bot
parent 1cf15bfe55
commit cf0595234c
4 changed files with 42 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
video/overuse_frame_detector_resource_adaptation_module.cc
View File

@ -359,7 +359,7 @@ OveruseFrameDetectorResourceAdaptationModule::
last_input_frame_size_(absl::nullopt), last_input_frame_size_(absl::nullopt),
target_frame_rate_(absl::nullopt), target_frame_rate_(absl::nullopt),
target_bitrate_bps_(absl::nullopt), target_bitrate_bps_(absl::nullopt),
is_quality_scaler_enabled_(false), quality_scaler_(nullptr),
encoder_settings_(absl::nullopt), encoder_settings_(absl::nullopt),
encoder_stats_observer_(encoder_stats_observer) { encoder_stats_observer_(encoder_stats_observer) {
RTC_DCHECK(adaptation_listener_); RTC_DCHECK(adaptation_listener_);
@ -480,9 +480,14 @@ void OveruseFrameDetectorResourceAdaptationModule::OnEncodeCompleted(
encode_duration_us); encode_duration_us);
} }
void OveruseFrameDetectorResourceAdaptationModule::SetIsQualityScalerEnabled( void OveruseFrameDetectorResourceAdaptationModule::UpdateQualityScalerSettings(
bool is_quality_scaler_enabled) { absl::optional<VideoEncoder::QpThresholds> qp_thresholds) {
is_quality_scaler_enabled_ = is_quality_scaler_enabled; if (qp_thresholds.has_value()) {
quality_scaler_ =
std::make_unique<QualityScaler>(this, qp_thresholds.value());
} else {
quality_scaler_ = nullptr;
}
} }
void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) { void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) {
@ -824,11 +829,11 @@ OveruseFrameDetectorResourceAdaptationModule::GetActiveCounts(
break; break;
case kQuality: case kQuality:
if (!IsFramerateScalingEnabled(degradation_preference_) || if (!IsFramerateScalingEnabled(degradation_preference_) ||
!is_quality_scaler_enabled_) { !quality_scaler_) {
counts.num_framerate_reductions = absl::nullopt; counts.num_framerate_reductions = absl::nullopt;
} }
if (!IsResolutionScalingEnabled(degradation_preference_) || if (!IsResolutionScalingEnabled(degradation_preference_) ||
!is_quality_scaler_enabled_) { !quality_scaler_) {
counts.num_resolution_reductions = absl::nullopt; counts.num_resolution_reductions = absl::nullopt;
} }
break; break;

12
video/overuse_frame_detector_resource_adaptation_module.h
View File

@ -63,6 +63,7 @@ class OveruseFrameDetectorResourceAdaptationModule
DegradationPreference degradation_preference() const { DegradationPreference degradation_preference() const {
return degradation_preference_; return degradation_preference_;
} }
QualityScaler* quality_scaler() const { return quality_scaler_.get(); }
// ResourceAdaptationModuleInterface implementation. // ResourceAdaptationModuleInterface implementation.
void StartResourceAdaptation( void StartResourceAdaptation(
@ -85,12 +86,9 @@ class OveruseFrameDetectorResourceAdaptationModule
int64_t capture_time_us, int64_t capture_time_us,
absl::optional<int> encode_duration_us) override; absl::optional<int> encode_duration_us) override;
// Inform the detector whether or not the quality scaler is enabled. This // Use nullopt to disable quality scaling.
// helps GetActiveCounts() return absl::nullopt when appropriate. void UpdateQualityScalerSettings(
// TODO(hbos): This feels really hacky, can we report the right values without absl::optional<VideoEncoder::QpThresholds> qp_thresholds);
// this boolean? It would be really easy to report the wrong thing if this
// method is called incorrectly.
void SetIsQualityScalerEnabled(bool is_quality_scaler_enabled);
class AdaptCounter final { class AdaptCounter final {
public: public:
@ -209,7 +207,7 @@ class OveruseFrameDetectorResourceAdaptationModule
absl::optional<int> last_input_frame_size_; absl::optional<int> last_input_frame_size_;
absl::optional<double> target_frame_rate_; absl::optional<double> target_frame_rate_;
absl::optional<uint32_t> target_bitrate_bps_; absl::optional<uint32_t> target_bitrate_bps_;
bool is_quality_scaler_enabled_; std::unique_ptr<QualityScaler> quality_scaler_;
absl::optional<EncoderSettings> encoder_settings_; absl::optional<EncoderSettings> encoder_settings_;
VideoStreamEncoderObserver* const encoder_stats_observer_; VideoStreamEncoderObserver* const encoder_stats_observer_;
}; };

47
video/video_stream_encoder.cc
View File

@ -18,6 +18,7 @@
#include <utility> #include <utility>
#include "absl/algorithm/container.h" #include "absl/algorithm/container.h"
#include "absl/types/optional.h"
#include "api/video/encoded_image.h" #include "api/video/encoded_image.h"
#include "api/video/i420_buffer.h" #include "api/video/i420_buffer.h"
#include "api/video/video_bitrate_allocator_factory.h" #include "api/video/video_bitrate_allocator_factory.h"
@ -341,8 +342,7 @@ void VideoStreamEncoder::Stop() {
rate_allocator_ = nullptr; rate_allocator_ = nullptr;
bitrate_observer_ = nullptr; bitrate_observer_ = nullptr;
ReleaseEncoder(); ReleaseEncoder();
quality_scaler_ = nullptr; resource_adaptation_module_->UpdateQualityScalerSettings(absl::nullopt);
resource_adaptation_module_->SetIsQualityScalerEnabled(false);
shutdown_event_.Set(); shutdown_event_.Set();
}); });
@ -786,7 +786,7 @@ void VideoStreamEncoder::ConfigureQualityScaler(
scaling_settings.thresholds; scaling_settings.thresholds;
if (quality_scaling_allowed) { if (quality_scaling_allowed) {
if (quality_scaler_ == nullptr) { if (resource_adaptation_module_->quality_scaler() == nullptr) {
// Quality scaler has not already been configured. // Quality scaler has not already been configured.
// Use experimental thresholds if available. // Use experimental thresholds if available.
@ -795,28 +795,24 @@ void VideoStreamEncoder::ConfigureQualityScaler(
experimental_thresholds = QualityScalingExperiment::GetQpThresholds( experimental_thresholds = QualityScalingExperiment::GetQpThresholds(
encoder_config_.codec_type); encoder_config_.codec_type);
} }
// Since the interface is non-public, std::make_unique can't do this resource_adaptation_module_->UpdateQualityScalerSettings(
// upcast. experimental_thresholds ? *experimental_thresholds
AdaptationObserverInterface* observer = resource_adaptation_module_.get();
quality_scaler_ = std::make_unique<QualityScaler>(
observer, experimental_thresholds ? *experimental_thresholds
: *(scaling_settings.thresholds)); : *(scaling_settings.thresholds));
resource_adaptation_module_->SetIsQualityScalerEnabled(true);
initial_framedrop_ = 0; initial_framedrop_ = 0;
} }
} else { } else {
quality_scaler_.reset(nullptr); resource_adaptation_module_->UpdateQualityScalerSettings(absl::nullopt);
resource_adaptation_module_->SetIsQualityScalerEnabled(false);
initial_framedrop_ = kMaxInitialFramedrop; initial_framedrop_ = kMaxInitialFramedrop;
} }
QualityScaler* quality_scaler = resource_adaptation_module_->quality_scaler();
if (resource_adaptation_module_->degradation_preference() == if (resource_adaptation_module_->degradation_preference() ==
DegradationPreference::BALANCED && DegradationPreference::BALANCED &&
quality_scaler_ && last_frame_info_) { quality_scaler && last_frame_info_) {
absl::optional<VideoEncoder::QpThresholds> thresholds = absl::optional<VideoEncoder::QpThresholds> thresholds =
resource_adaptation_module_->GetQpThresholds(); resource_adaptation_module_->GetQpThresholds();
if (thresholds) { if (thresholds) {
quality_scaler_->SetQpThresholds(*thresholds); quality_scaler->SetQpThresholds(*thresholds);
} }
} }
@ -1553,8 +1549,10 @@ void VideoStreamEncoder::OnDroppedFrame(DropReason reason) {
VideoStreamEncoderObserver::DropReason::kMediaOptimization); VideoStreamEncoderObserver::DropReason::kMediaOptimization);
encoder_queue_.PostTask([this] { encoder_queue_.PostTask([this] {
RTC_DCHECK_RUN_ON(&encoder_queue_); RTC_DCHECK_RUN_ON(&encoder_queue_);
if (quality_scaler_) QualityScaler* quality_scaler =
quality_scaler_->ReportDroppedFrameByMediaOpt(); resource_adaptation_module_->quality_scaler();
if (quality_scaler)
quality_scaler->ReportDroppedFrameByMediaOpt();
}); });
break; break;
case DropReason::kDroppedByEncoder: case DropReason::kDroppedByEncoder:
@ -1562,8 +1560,10 @@ void VideoStreamEncoder::OnDroppedFrame(DropReason reason) {
VideoStreamEncoderObserver::DropReason::kEncoder); VideoStreamEncoderObserver::DropReason::kEncoder);
encoder_queue_.PostTask([this] { encoder_queue_.PostTask([this] {
RTC_DCHECK_RUN_ON(&encoder_queue_); RTC_DCHECK_RUN_ON(&encoder_queue_);
if (quality_scaler_) QualityScaler* quality_scaler =
quality_scaler_->ReportDroppedFrameByEncoder(); resource_adaptation_module_->quality_scaler();
if (quality_scaler)
quality_scaler->ReportDroppedFrameByEncoder();
}); });
break; break;
} }
@ -1607,7 +1607,8 @@ void VideoStreamEncoder::OnBitrateUpdated(DataRate target_bitrate,
<< " rtt " << round_trip_time_ms; << " rtt " << round_trip_time_ms;
if (set_start_bitrate_bps_ > 0 && !has_seen_first_bwe_drop_ && if (set_start_bitrate_bps_ > 0 && !has_seen_first_bwe_drop_ &&
quality_scaler_ && quality_scaler_settings_.InitialBitrateIntervalMs() && resource_adaptation_module_->quality_scaler() &&
quality_scaler_settings_.InitialBitrateIntervalMs() &&
quality_scaler_settings_.InitialBitrateFactor()) { quality_scaler_settings_.InitialBitrateFactor()) {
int64_t diff_ms = clock_->TimeInMilliseconds() - set_start_bitrate_time_ms_; int64_t diff_ms = clock_->TimeInMilliseconds() - set_start_bitrate_time_ms_;
if (diff_ms < quality_scaler_settings_.InitialBitrateIntervalMs().value() && if (diff_ms < quality_scaler_settings_.InitialBitrateIntervalMs().value() &&
@ -1680,7 +1681,8 @@ bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const {
} }
bool VideoStreamEncoder::TryQualityRampup(int64_t now_ms) { bool VideoStreamEncoder::TryQualityRampup(int64_t now_ms) {
if (!quality_scaler_) QualityScaler* quality_scaler = resource_adaptation_module_->quality_scaler();
if (!quality_scaler)
return false; return false;
uint32_t bw_kbps = last_encoder_rate_settings_ uint32_t bw_kbps = last_encoder_rate_settings_
@ -1692,7 +1694,7 @@ bool VideoStreamEncoder::TryQualityRampup(int64_t now_ms) {
// Verify that encoder is at max bitrate and the QP is low. // Verify that encoder is at max bitrate and the QP is low.
if (encoder_target_bitrate_bps_.value_or(0) == if (encoder_target_bitrate_bps_.value_or(0) ==
send_codec_.maxBitrate * 1000 && send_codec_.maxBitrate * 1000 &&
quality_scaler_->QpFastFilterLow()) { quality_scaler->QpFastFilterLow()) {
return true; return true;
} }
} }
@ -1764,8 +1766,9 @@ void VideoStreamEncoder::RunPostEncode(const EncodedImage& encoded_image,
encoded_image.Timestamp(), time_sent_us, encoded_image.Timestamp(), time_sent_us,
encoded_image.capture_time_ms_ * rtc::kNumMicrosecsPerMillisec, encoded_image.capture_time_ms_ * rtc::kNumMicrosecsPerMillisec,
encode_duration_us); encode_duration_us);
if (quality_scaler_ && encoded_image.qp_ >= 0) QualityScaler* quality_scaler = resource_adaptation_module_->quality_scaler();
quality_scaler_->ReportQp(encoded_image.qp_, time_sent_us); if (quality_scaler && encoded_image.qp_ >= 0)
quality_scaler->ReportQp(encoded_image.qp_, time_sent_us);
if (bitrate_adjuster_) { if (bitrate_adjuster_) {
bitrate_adjuster_->OnEncodedFrame(encoded_image, temporal_index); bitrate_adjuster_->OnEncodedFrame(encoded_image, temporal_index);
} }

3
video/video_stream_encoder.h
View File

@ -226,9 +226,6 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
const RateControlSettings rate_control_settings_; const RateControlSettings rate_control_settings_;
const QualityScalerSettings quality_scaler_settings_; const QualityScalerSettings quality_scaler_settings_;
std::unique_ptr<QualityScaler> quality_scaler_ RTC_GUARDED_BY(&encoder_queue_)
RTC_PT_GUARDED_BY(&encoder_queue_);
VideoStreamEncoderObserver* const encoder_stats_observer_; VideoStreamEncoderObserver* const encoder_stats_observer_;
// |thread_checker_| checks that public methods that are related to lifetime // |thread_checker_| checks that public methods that are related to lifetime
// of VideoStreamEncoder are called on the same thread. // of VideoStreamEncoder are called on the same thread.