zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move DegradationPreference logic to the encoder queue.

Browse Source 
This moves SetHasInputVideoAndDegradationPreference() to the encoder
queue. OveruseFrameDetectorResourceAdaptationModule is now entirely
single-threaded, including its inner class VideoSourceRestrictor.

VideoStreamEncoder now protects the module with RTC_GUARDED_BY. This
ensures it is safely used, even without a SequenceChecker inside of the
module. The module's |encoder_queue_| is removed.

The one task queue reference that is needed - passing down the current
task queue to StartCheckForOveruse() - is replaced by a TaskQueueBase*
(instead of rtc::TaskQueue*), enabling obtaining the current queue with
TaskQueueBase::Current(). (There is no rtc::TaskQueue::Current().)

Furthermore, the only uses of VideoSourceSinkController that isn't on
the encoder queue are documented, with a TODO saying if these are moved
the VideoSourceSinkController could also be made single-threaded.
However since this requires introducing a delay to
VideoStreamEncoder::SetSource() and VideoStreamEncoder::Stop(),
arguably a more risky change, if this is to be attempted that should be
in a separate CL.

Bug: webrtc:11222
Change-Id: I448ca5125708d5f66b95b0b180d6d24cc356dfa9
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/165783
Commit-Queue: Henrik Boström <hbos@webrtc.org>
Reviewed-by: Evan Shrubsole <eshr@google.com>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30263}
This commit is contained in:
Henrik Boström
2020-01-15 11:42:12 +01:00
committed by Commit Bot
parent 6ef59d1ced
commit 07b17df771
9 changed files with 122 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
video/BUILD.gn
View File

@ -194,6 +194,7 @@ rtc_library("video_stream_encoder_impl") {
deps = [
"../api:rtp_parameters",
"../api/task_queue:task_queue",
"../api/units:data_rate",
"../api/video:encoded_image",
"../api/video:video_bitrate_allocation",

4
video/overuse_frame_detector.cc
View File

@ -540,7 +540,7 @@ OveruseFrameDetector::OveruseFrameDetector(
OveruseFrameDetector::~OveruseFrameDetector() {}
void OveruseFrameDetector::StartCheckForOveruse(
rtc::TaskQueue* task_queue,
TaskQueueBase* task_queue_base,
const CpuOveruseOptions& options,
AdaptationObserverInterface* overuse_observer) {
RTC_DCHECK_RUN_ON(&task_checker_);
@ -549,7 +549,7 @@ void OveruseFrameDetector::StartCheckForOveruse(
SetOptions(options);
check_overuse_task_ = RepeatingTaskHandle::DelayedStart(
task_queue->Get(), TimeDelta::ms(kTimeToFirstCheckForOveruseMs),
task_queue_base, TimeDelta::ms(kTimeToFirstCheckForOveruseMs),
[this, overuse_observer] {
CheckForOveruse(overuse_observer);
return TimeDelta::ms(kCheckForOveruseIntervalMs);

4
video/overuse_frame_detector.h
View File

@ -15,13 +15,13 @@
#include <memory>
#include "absl/types/optional.h"
#include "api/task_queue/task_queue_base.h"
#include "api/video/video_stream_encoder_observer.h"
#include "modules/video_coding/utility/quality_scaler.h"
#include "rtc_base/constructor_magic.h"
#include "rtc_base/experiments/field_trial_parser.h"
#include "rtc_base/numerics/exp_filter.h"
#include "rtc_base/synchronization/sequence_checker.h"
#include "rtc_base/task_queue.h"
#include "rtc_base/task_utils/repeating_task.h"
#include "rtc_base/thread_annotations.h"
@ -58,7 +58,7 @@ class OveruseFrameDetector {
virtual ~OveruseFrameDetector();
// Start to periodically check for overuse.
void StartCheckForOveruse(rtc::TaskQueue* task_queue,
void StartCheckForOveruse(TaskQueueBase* task_queue_base,
const CpuOveruseOptions& options,
AdaptationObserverInterface* overuse_observer);

174
video/overuse_frame_detector_resource_adaptation_module.cc
View File

@ -17,6 +17,7 @@
#include <utility>
#include "absl/algorithm/container.h"
#include "api/task_queue/task_queue_base.h"
#include "api/video/video_source_interface.h"
#include "call/adaptation/video_source_restrictions.h"
#include "rtc_base/logging.h"
@ -73,10 +74,6 @@ VideoSourceRestrictions ApplyDegradationPreference(
// AdaptDown() - only how to modify the source/sink restrictions when this
// happens. Note that it is also not responsible for reconfigruring the
// source/sink, it is only a keeper of desired restrictions.
//
// Thread safety is ensured between SetHasInputVideoAndDegradationPreference()
// calls on the worker thread and adaptation logic on the encoder task queue
// using a lock.
class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor {
public:
VideoSourceRestrictor()
@ -84,25 +81,19 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor {
degradation_preference_(DegradationPreference::DISABLED) {}
VideoSourceRestrictions source_restrictions() {
rtc::CritScope lock(&crit_);
return source_restrictions_;
}
// Inform the restrictor of new source status and degradation preference.
// TODO(hbos): Can this be moved to the encoder queue? If so, the |crit_| lock
// can be removed and we only need a sequence checker.
void SetHasInputVideoAndDegradationPreference(
bool has_input_video,
DegradationPreference degradation_preference) {
// Called on libjingle's worker thread.
RTC_DCHECK_RUN_ON(&main_checker_);
rtc::CritScope lock(&crit_);
has_input_video_ = has_input_video;
degradation_preference_ = degradation_preference;
}
// Updates the source_restrictions(). The source/sink has to be informed of
// this separately.
void ClearRestrictions() {
rtc::CritScope lock(&crit_);
source_restrictions_ = VideoSourceRestrictions();
}
@ -111,14 +102,9 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor {
bool RequestResolutionLowerThan(int pixel_count,
int min_pixels_per_frame,
bool* min_pixels_reached) {
// Called on the encoder task queue.
rtc::CritScope lock(&crit_);
if (!has_input_video_ ||
!IsResolutionScalingEnabled(degradation_preference_)) {
// This can happen since |degradation_preference_| is set on libjingle's
// worker thread but the adaptation is done on the encoder task queue.
RTC_DCHECK(IsResolutionScalingEnabled(degradation_preference_));
if (!has_input_video_)
return false;
}
// The input video frame size will have a resolution less than or equal to
// |max_pixel_count| depending on how the source can scale the frame size.
const int pixels_wanted = (pixel_count * 3) / 5;
@ -145,7 +131,6 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor {
// Updates the source_restrictions(). The source/sink has to be informed of
// this separately.
int RequestFramerateLowerThan(int fps) {
// Called on the encoder task queue.
// The input video frame rate will be scaled down to 2/3, rounding down.
int framerate_wanted = (fps * 2) / 3;
return RestrictFramerate(framerate_wanted) ? framerate_wanted : -1;
@ -164,14 +149,9 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor {
// Updates the source_restrictions(). The source/sink has to be informed of
// this separately.
bool RequestHigherResolutionThan(int pixel_count) {
// Called on the encoder task queue.
rtc::CritScope lock(&crit_);
if (!has_input_video_ ||
!IsResolutionScalingEnabled(degradation_preference_)) {
// This can happen since |degradation_preference_| is set on libjingle's
// worker thread but the adaptation is done on the encoder task queue.
RTC_DCHECK(IsResolutionScalingEnabled(degradation_preference_));
if (!has_input_video_)
return false;
}
int max_pixels_wanted = pixel_count;
if (max_pixels_wanted != std::numeric_limits<int>::max())
max_pixels_wanted = pixel_count * 4;
@ -203,7 +183,6 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor {
// adaptation requests are removed completely. In that case, consider
// |max_framerate_| to be the current limit (assuming the capturer complies).
int RequestHigherFramerateThan(int fps) {
// Called on the encoder task queue.
// The input frame rate will be scaled up to the last step, with rounding.
int framerate_wanted = fps;
if (fps != std::numeric_limits<int>::max())
@ -215,10 +194,8 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor {
// Updates the source_restrictions(). The source/sink has to be informed of
// this separately.
bool RestrictFramerate(int fps) {
// Called on the encoder task queue.
rtc::CritScope lock(&crit_);
if (!has_input_video_ ||
!IsFramerateScalingEnabled(degradation_preference_))
RTC_DCHECK(IsFramerateScalingEnabled(degradation_preference_));
if (!has_input_video_)
return false;
const int fps_wanted = std::max(kMinFramerateFps, fps);
@ -238,10 +215,8 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor {
// Updates the source_restrictions(). The source/sink has to be informed of
// this separately.
bool IncreaseFramerate(int fps) {
// Called on the encoder task queue.
rtc::CritScope lock(&crit_);
if (!has_input_video_ ||
!IsFramerateScalingEnabled(degradation_preference_))
RTC_DCHECK(IsFramerateScalingEnabled(degradation_preference_));
if (!has_input_video_)
return false;
const int fps_wanted = std::max(kMinFramerateFps, fps);
@ -259,11 +234,9 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor {
}
private:
rtc::CriticalSection crit_;
SequenceChecker main_checker_;
VideoSourceRestrictions source_restrictions_ RTC_GUARDED_BY(&crit_);
bool has_input_video_ RTC_GUARDED_BY(&crit_);
DegradationPreference degradation_preference_ RTC_GUARDED_BY(&crit_);
VideoSourceRestrictions source_restrictions_;
bool has_input_video_;
DegradationPreference degradation_preference_;
RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceRestrictor);
};
@ -397,8 +370,7 @@ OveruseFrameDetectorResourceAdaptationModule::
std::unique_ptr<OveruseFrameDetector> overuse_detector,
VideoStreamEncoderObserver* encoder_stats_observer,
ResourceAdaptationModuleListener* adaptation_listener)
: encoder_queue_(nullptr),
adaptation_listener_(adaptation_listener),
: adaptation_listener_(adaptation_listener),
video_stream_encoder_(video_stream_encoder),
degradation_preference_(DegradationPreference::DISABLED),
adapt_counters_(),
@ -422,24 +394,13 @@ OveruseFrameDetectorResourceAdaptationModule::
OveruseFrameDetectorResourceAdaptationModule::
~OveruseFrameDetectorResourceAdaptationModule() {}
void OveruseFrameDetectorResourceAdaptationModule::Initialize(
rtc::TaskQueue* encoder_queue) {
RTC_DCHECK(!encoder_queue_);
encoder_queue_ = encoder_queue;
RTC_DCHECK(encoder_queue_);
}
void OveruseFrameDetectorResourceAdaptationModule::SetEncoder(
VideoEncoder* encoder) {
RTC_DCHECK(encoder_queue_);
RTC_DCHECK_RUN_ON(encoder_queue_);
encoder_ = encoder;
}
void OveruseFrameDetectorResourceAdaptationModule::StartCheckForOveruse(
ResourceAdaptationModuleListener* adaptation_listener) {
RTC_DCHECK(encoder_queue_);
RTC_DCHECK_RUN_ON(encoder_queue_);
RTC_DCHECK(encoder_);
// TODO(hbos): When AdaptUp() and AdaptDown() are no longer invoked outside
// the interval between StartCheckForOveruse() and StopCheckForOveruse(),
@ -449,20 +410,17 @@ void OveruseFrameDetectorResourceAdaptationModule::StartCheckForOveruse(
// AdaptUp() and AdaptDown() even when the OveruseDetector is inactive.
RTC_DCHECK_EQ(adaptation_listener, adaptation_listener_);
overuse_detector_->StartCheckForOveruse(
encoder_queue_, video_stream_encoder_->GetCpuOveruseOptions(), this);
TaskQueueBase::Current(), video_stream_encoder_->GetCpuOveruseOptions(),
this);
}
void OveruseFrameDetectorResourceAdaptationModule::StopCheckForOveruse() {
RTC_DCHECK(encoder_queue_);
RTC_DCHECK_RUN_ON(encoder_queue_);
overuse_detector_->StopCheckForOveruse();
}
void OveruseFrameDetectorResourceAdaptationModule::FrameCaptured(
const VideoFrame& frame,
int64_t time_when_first_seen_us) {
RTC_DCHECK(encoder_queue_);
RTC_DCHECK_RUN_ON(encoder_queue_);
overuse_detector_->FrameCaptured(frame, time_when_first_seen_us);
}
@ -471,44 +429,32 @@ void OveruseFrameDetectorResourceAdaptationModule::FrameSent(
int64_t time_sent_in_us,
int64_t capture_time_us,
absl::optional<int> encode_duration_us) {
RTC_DCHECK(encoder_queue_);
RTC_DCHECK_RUN_ON(encoder_queue_);
overuse_detector_->FrameSent(timestamp, time_sent_in_us, capture_time_us,
encode_duration_us);
}
void OveruseFrameDetectorResourceAdaptationModule::SetLastFramePixelCount(
absl::optional<int> last_frame_pixel_count) {
RTC_DCHECK(encoder_queue_);
RTC_DCHECK_RUN_ON(encoder_queue_);
last_frame_pixel_count_ = last_frame_pixel_count;
}
void OveruseFrameDetectorResourceAdaptationModule::SetEncoderConfig(
VideoEncoderConfig encoder_config) {
RTC_DCHECK(encoder_queue_);
RTC_DCHECK_RUN_ON(encoder_queue_);
encoder_config_ = std::move(encoder_config);
}
void OveruseFrameDetectorResourceAdaptationModule::SetCodecMaxFramerate(
int codec_max_framerate) {
RTC_DCHECK(encoder_queue_);
RTC_DCHECK_RUN_ON(encoder_queue_);
codec_max_framerate_ = codec_max_framerate;
}
void OveruseFrameDetectorResourceAdaptationModule::SetEncoderStartBitrateBps(
uint32_t encoder_start_bitrate_bps) {
RTC_DCHECK(encoder_queue_);
RTC_DCHECK_RUN_ON(encoder_queue_);
encoder_start_bitrate_bps_ = encoder_start_bitrate_bps;
}
void OveruseFrameDetectorResourceAdaptationModule::SetIsQualityScalerEnabled(
bool is_quality_scaler_enabled) {
RTC_DCHECK(encoder_queue_);
RTC_DCHECK_RUN_ON(encoder_queue_);
is_quality_scaler_enabled_ = is_quality_scaler_enabled;
}
@ -516,43 +462,25 @@ void OveruseFrameDetectorResourceAdaptationModule::
SetHasInputVideoAndDegradationPreference(
bool has_input_video,
DegradationPreference degradation_preference) {
// TODO(https://crbug.com/webrtc/11222): Move this call to the encoder queue,
// making VideoSourceRestrictor single-threaded and removing the only call to
// MaybeUpdateVideoSourceRestrictions() that isn't on the |encoder_queue_|.
source_restrictor_->SetHasInputVideoAndDegradationPreference(
has_input_video, degradation_preference);
MaybeUpdateVideoSourceRestrictions(degradation_preference);
encoder_queue_->PostTask([this, degradation_preference] {
RTC_DCHECK_RUN_ON(encoder_queue_);
if (degradation_preference_ != degradation_preference) {
// Reset adaptation state, so that we're not tricked into thinking there's
// an already pending request of the same type.
last_adaptation_request_.reset();
if (degradation_preference == DegradationPreference::BALANCED ||
degradation_preference_ == DegradationPreference::BALANCED) {
// TODO(asapersson): Consider removing |adapt_counters_| map and use one
// AdaptCounter for all modes.
source_restrictor_->ClearRestrictions();
adapt_counters_.clear();
}
if (degradation_preference_ != degradation_preference) {
// Reset adaptation state, so that we're not tricked into thinking there's
// an already pending request of the same type.
last_adaptation_request_.reset();
if (degradation_preference == DegradationPreference::BALANCED ||
degradation_preference_ == DegradationPreference::BALANCED) {
// TODO(asapersson): Consider removing |adapt_counters_| map and use one
// AdaptCounter for all modes.
source_restrictor_->ClearRestrictions();
adapt_counters_.clear();
}
degradation_preference_ = degradation_preference;
// This is the second time we're invoking
// MaybeUpdateVideoSourceRestrictions() in this method. This is because
// current tests expect the changes to the source restrictions to be
// immediate (outside of the encoder queue) while it is possible that they
// change again after ClearRestrictions() on the encoder queue.
// TODO(https://crbug.com/webrtc/11222): Change the expectations to allow
// source restrictions only to change on the encoder queue. This unblocks
// making OveruseFrameDetectorResourceAdaptationModule and
// VideoSourceRestrictor single-threaded.
MaybeUpdateVideoSourceRestrictions(degradation_preference_);
});
}
degradation_preference_ = degradation_preference;
source_restrictor_->SetHasInputVideoAndDegradationPreference(
has_input_video, degradation_preference_);
MaybeUpdateVideoSourceRestrictions();
}
void OveruseFrameDetectorResourceAdaptationModule::RefreshTargetFramerate() {
RTC_DCHECK(encoder_queue_);
RTC_DCHECK_RUN_ON(encoder_queue_);
absl::optional<double> restricted_frame_rate =
ApplyDegradationPreference(source_restrictor_->source_restrictions(),
degradation_preference_)
@ -571,17 +499,13 @@ void OveruseFrameDetectorResourceAdaptationModule::RefreshTargetFramerate() {
}
void OveruseFrameDetectorResourceAdaptationModule::ResetAdaptationCounters() {
RTC_DCHECK(encoder_queue_);
RTC_DCHECK_RUN_ON(encoder_queue_);
last_adaptation_request_.reset();
source_restrictor_->ClearRestrictions();
adapt_counters_.clear();
MaybeUpdateVideoSourceRestrictions(degradation_preference_);
MaybeUpdateVideoSourceRestrictions();
}
void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) {
RTC_DCHECK(encoder_queue_);
RTC_DCHECK_RUN_ON(encoder_queue_);
const AdaptCounter& adapt_counter = GetConstAdaptCounter();
int num_downgrades = adapt_counter.TotalCount(reason);
if (num_downgrades == 0)
@ -685,7 +609,7 @@ void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) {
// Tell the adaptation listener to reconfigure the source for us according to
// the latest adaptation.
MaybeUpdateVideoSourceRestrictions(degradation_preference_);
MaybeUpdateVideoSourceRestrictions();
last_adaptation_request_.emplace(adaptation_request);
@ -696,8 +620,6 @@ void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) {
bool OveruseFrameDetectorResourceAdaptationModule::AdaptDown(
AdaptReason reason) {
RTC_DCHECK(encoder_queue_);
RTC_DCHECK_RUN_ON(encoder_queue_);
AdaptationRequest adaptation_request = {
*last_frame_pixel_count_, encoder_stats_observer_->GetInputFrameRate(),
AdaptationRequest::Mode::kAdaptDown};
@ -791,7 +713,7 @@ bool OveruseFrameDetectorResourceAdaptationModule::AdaptDown(
// Tell the adaptation listener to reconfigure the source for us according to
// the latest adaptation.
MaybeUpdateVideoSourceRestrictions(degradation_preference_);
MaybeUpdateVideoSourceRestrictions();
last_adaptation_request_.emplace(adaptation_request);
@ -802,21 +724,13 @@ bool OveruseFrameDetectorResourceAdaptationModule::AdaptDown(
}
void OveruseFrameDetectorResourceAdaptationModule::
MaybeUpdateVideoSourceRestrictions(
DegradationPreference degradation_preference) {
absl::optional<VideoSourceRestrictions> updated_restrictions;
{
rtc::CritScope lock(&video_source_restrictions_crit_);
VideoSourceRestrictions new_restrictions = ApplyDegradationPreference(
source_restrictor_->source_restrictions(), degradation_preference);
if (video_source_restrictions_ != new_restrictions) {
video_source_restrictions_ = std::move(new_restrictions);
updated_restrictions = video_source_restrictions_;
}
}
if (updated_restrictions.has_value()) {
MaybeUpdateVideoSourceRestrictions() {
VideoSourceRestrictions new_restrictions = ApplyDegradationPreference(
source_restrictor_->source_restrictions(), degradation_preference_);
if (video_source_restrictions_ != new_restrictions) {
video_source_restrictions_ = std::move(new_restrictions);
adaptation_listener_->OnVideoSourceRestrictionsUpdated(
updated_restrictions.value());
video_source_restrictions_);
}
}
@ -840,8 +754,6 @@ void OveruseFrameDetectorResourceAdaptationModule::UpdateAdaptationStats(
VideoStreamEncoderObserver::AdaptationSteps
OveruseFrameDetectorResourceAdaptationModule::GetActiveCounts(
AdaptReason reason) {
RTC_DCHECK(encoder_queue_);
RTC_DCHECK_RUN_ON(encoder_queue_);
VideoStreamEncoderObserver::AdaptationSteps counts =
GetConstAdaptCounter().Counts(reason);
switch (reason) {
@ -885,15 +797,11 @@ OveruseFrameDetectorResourceAdaptationModule::GetAdaptCounter() {
const OveruseFrameDetectorResourceAdaptationModule::AdaptCounter&
OveruseFrameDetectorResourceAdaptationModule::GetConstAdaptCounter() {
RTC_DCHECK(encoder_queue_);
RTC_DCHECK_RUN_ON(encoder_queue_);
return adapt_counters_[degradation_preference_];
}
absl::optional<VideoEncoder::QpThresholds>
OveruseFrameDetectorResourceAdaptationModule::GetQpThresholds() const {
RTC_DCHECK(encoder_queue_);
RTC_DCHECK_RUN_ON(encoder_queue_);
RTC_DCHECK(last_frame_pixel_count_.has_value());
return balanced_settings_.GetQpThresholds(encoder_config_.codec_type,
last_frame_pixel_count_.value());

67
video/overuse_frame_detector_resource_adaptation_module.h
View File

@ -36,6 +36,9 @@ class VideoStreamEncoder;
// resolution up or down based on encode usage percent. It keeps track of video
// source settings, adaptation counters and may get influenced by
// VideoStreamEncoder's quality scaler through AdaptUp() and AdaptDown() calls.
//
// This class is single-threaded. The caller is responsible for ensuring safe
// usage.
// TODO(hbos): Reduce the coupling with VideoStreamEncoder.
// TODO(hbos): Add unittests specific to this class, it is currently only tested
// indirectly in video_stream_encoder_unittest.cc and other tests exercising
@ -48,6 +51,8 @@ class OveruseFrameDetectorResourceAdaptationModule
: public ResourceAdaptationModuleInterface,
public AdaptationObserverInterface {
public:
// The module can be constructed on any sequence, but must be initialized and
// used on a single sequence, e.g. the encoder queue.
OveruseFrameDetectorResourceAdaptationModule(
VideoStreamEncoder* video_stream_encoder,
std::unique_ptr<OveruseFrameDetector> overuse_detector,
@ -55,7 +60,6 @@ class OveruseFrameDetectorResourceAdaptationModule
ResourceAdaptationModuleListener* adaptation_listener);
~OveruseFrameDetectorResourceAdaptationModule() override;
void Initialize(rtc::TaskQueue* encoder_queue);
// Sets the encoder to reconfigure based on overuse.
// TODO(hbos): Don't reconfigure the encoder directly. Instead, define the
// output of a resource adaptation module as a struct and let the
@ -63,8 +67,6 @@ class OveruseFrameDetectorResourceAdaptationModule
void SetEncoder(VideoEncoder* encoder);
DegradationPreference degradation_preference() const {
RTC_DCHECK(encoder_queue_);
RTC_DCHECK_RUN_ON(encoder_queue_);
return degradation_preference_;
}
@ -187,60 +189,39 @@ class OveruseFrameDetectorResourceAdaptationModule
// Makes |video_source_restrictions_| up-to-date and informs the
// |adaptation_listener_| if restrictions are changed, allowing the listener
// to reconfigure the source accordingly.
// TODO(https://crbug.com/webrtc/11222): When
// SetHasInputVideoAndDegradationPreference() stops calling this method prior
// to updating |degradation_preference_| on the encoder queue, remove its
// argument in favor of using |degradation_preference_| directly.
void MaybeUpdateVideoSourceRestrictions(
DegradationPreference degradation_preference);
void MaybeUpdateVideoSourceRestrictions();
void UpdateAdaptationStats(AdaptReason reason) RTC_RUN_ON(encoder_queue_);
DegradationPreference EffectiveDegradataionPreference()
RTC_RUN_ON(encoder_queue_);
AdaptCounter& GetAdaptCounter() RTC_RUN_ON(encoder_queue_);
bool CanAdaptUpResolution(int pixels, uint32_t bitrate_bps) const
RTC_RUN_ON(encoder_queue_);
void UpdateAdaptationStats(AdaptReason reason);
DegradationPreference EffectiveDegradataionPreference();
AdaptCounter& GetAdaptCounter();
bool CanAdaptUpResolution(int pixels, uint32_t bitrate_bps) const;
rtc::TaskQueue* encoder_queue_;
// TODO(https://crbug.com/webrtc/11222): Update
// SetHasInputVideoAndDegradationPreference() to do all work on the encoder
// queue (including |source_restrictor_| and |adaptation_listener_| usage).
// When this is the case, remove |VideoSourceRestrictor::crit_| and
// |video_source_restrictions_crit_| and replace |encoder_queue_| with a
// sequence checker.
rtc::CriticalSection video_source_restrictions_crit_;
ResourceAdaptationModuleListener* const adaptation_listener_;
// The restrictions that |adaptation_listener_| is informed of.
VideoSourceRestrictions video_source_restrictions_
RTC_GUARDED_BY(&video_source_restrictions_crit_);
VideoSourceRestrictions video_source_restrictions_;
// Used to query CpuOveruseOptions at StartCheckForOveruse().
VideoStreamEncoder* video_stream_encoder_ RTC_GUARDED_BY(encoder_queue_);
DegradationPreference degradation_preference_ RTC_GUARDED_BY(encoder_queue_);
VideoStreamEncoder* video_stream_encoder_;
DegradationPreference degradation_preference_;
// Counters used for deciding if the video resolution or framerate is
// currently restricted, and if so, why, on a per degradation preference
// basis.
// TODO(sprang): Replace this with a state holding a relative overuse measure
// instead, that can be translated into suitable down-scale or fps limit.
std::map<const DegradationPreference, AdaptCounter> adapt_counters_
RTC_GUARDED_BY(encoder_queue_);
const BalancedDegradationSettings balanced_settings_
RTC_GUARDED_BY(encoder_queue_);
std::map<const DegradationPreference, AdaptCounter> adapt_counters_;
const BalancedDegradationSettings balanced_settings_;
// Stores a snapshot of the last adaptation request triggered by an AdaptUp
// or AdaptDown signal.
absl::optional<AdaptationRequest> last_adaptation_request_
RTC_GUARDED_BY(encoder_queue_);
absl::optional<int> last_frame_pixel_count_ RTC_GUARDED_BY(encoder_queue_);
absl::optional<AdaptationRequest> last_adaptation_request_;
absl::optional<int> last_frame_pixel_count_;
// Keeps track of source restrictions that this adaptation module outputs.
const std::unique_ptr<VideoSourceRestrictor> source_restrictor_;
const std::unique_ptr<OveruseFrameDetector> overuse_detector_
RTC_PT_GUARDED_BY(encoder_queue_);
int codec_max_framerate_ RTC_GUARDED_BY(encoder_queue_);
uint32_t encoder_start_bitrate_bps_ RTC_GUARDED_BY(encoder_queue_);
bool is_quality_scaler_enabled_ RTC_GUARDED_BY(encoder_queue_);
VideoEncoderConfig encoder_config_ RTC_GUARDED_BY(encoder_queue_);
VideoEncoder* encoder_ RTC_GUARDED_BY(encoder_queue_);
VideoStreamEncoderObserver* const encoder_stats_observer_
RTC_GUARDED_BY(encoder_queue_);
const std::unique_ptr<OveruseFrameDetector> overuse_detector_;
int codec_max_framerate_;
uint32_t encoder_start_bitrate_bps_;
bool is_quality_scaler_enabled_;
VideoEncoderConfig encoder_config_;
VideoEncoder* encoder_;
VideoStreamEncoderObserver* const encoder_stats_observer_;
};
} // namespace webrtc

6
video/overuse_frame_detector_unittest.cc
View File

@ -433,7 +433,8 @@ TEST_F(OveruseFrameDetectorTest, RunOnTqNormalUsage) {
queue.SendTask(
[&] {
overuse_detector_->StartCheckForOveruse(&queue, options_, observer_);
overuse_detector_->StartCheckForOveruse(queue.Get(), options_,
observer_);
},
RTC_FROM_HERE);
@ -914,7 +915,8 @@ TEST_F(OveruseFrameDetectorTest2, RunOnTqNormalUsage) {
queue.SendTask(
[&] {
overuse_detector_->StartCheckForOveruse(&queue, options_, observer_);
overuse_detector_->StartCheckForOveruse(queue.Get(), options_,
observer_);
},
RTC_FROM_HERE);

13
video/video_stream_encoder.cc
View File

@ -332,7 +332,6 @@ VideoStreamEncoder::VideoStreamEncoder(
TaskQueueFactory::Priority::NORMAL)) {
RTC_DCHECK(encoder_stats_observer);
RTC_DCHECK_GE(number_of_cores, 1);
resource_adaptation_module_->Initialize(encoder_queue());
for (auto& state : encoder_buffer_state_)
state.fill(std::numeric_limits<int64_t>::max());
@ -388,10 +387,10 @@ void VideoStreamEncoder::SetSource(
const DegradationPreference& degradation_preference) {
RTC_DCHECK_RUN_ON(&thread_checker_);
video_source_sink_controller_->SetSource(source);
resource_adaptation_module_->SetHasInputVideoAndDegradationPreference(
source, degradation_preference);
encoder_queue_.PostTask([this, degradation_preference] {
encoder_queue_.PostTask([this, source, degradation_preference] {
RTC_DCHECK_RUN_ON(&encoder_queue_);
resource_adaptation_module_->SetHasInputVideoAndDegradationPreference(
source, degradation_preference);
if (encoder_)
ConfigureQualityScaler(encoder_->GetEncoderInfo());
@ -1731,19 +1730,19 @@ CpuOveruseOptions VideoStreamEncoder::GetCpuOveruseOptions() const {
bool VideoStreamEncoder::TriggerAdaptDown(
AdaptationObserverInterface::AdaptReason reason) {
RTC_DCHECK_RUN_ON(&encoder_queue_);
return resource_adaptation_module_->AdaptDown(reason);
}
void VideoStreamEncoder::TriggerAdaptUp(
AdaptationObserverInterface::AdaptReason reason) {
RTC_DCHECK_RUN_ON(&encoder_queue_);
resource_adaptation_module_->AdaptUp(reason);
}
void VideoStreamEncoder::OnVideoSourceRestrictionsUpdated(
VideoSourceRestrictions restrictions) {
// TODO(https://crbug.com/webrtc/11222): DCHECK that we are using the
// |encoder_queue_| when OnVideoSourceRestrictionsUpdated() is no longer
// invoked off this thread due to VideoStreamEncoder::SetSource() stuff.
RTC_DCHECK_RUN_ON(&encoder_queue_);
video_source_sink_controller_->SetRestrictions(std::move(restrictions));
video_source_sink_controller_->PushSourceSinkSettings();
}

13
video/video_stream_encoder.h
View File

@ -408,9 +408,20 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
// track of whether a request has been made or not.
bool encoder_switch_requested_ RTC_GUARDED_BY(&encoder_queue_);
// The controller updates the sink wants based on restrictions that come from
// the resource adaptation module or adaptation due to bandwidth adaptation.
//
// This is used on the encoder queue, with a few exceptions:
// - VideoStreamEncoder::SetSource() invokes SetSource().
// - VideoStreamEncoder::SetSink() invokes SetRotationApplied() and
// PushSourceSinkSettings().
// - VideoStreamEncoder::Stop() invokes SetSource().
// TODO(hbos): If these can be moved to the encoder queue,
// VideoSourceSinkController can be made single-threaded, and its lock can be
// replaced with a sequence checker.
std::unique_ptr<VideoSourceSinkController> video_source_sink_controller_;
std::unique_ptr<OveruseFrameDetectorResourceAdaptationModule>
resource_adaptation_module_;
resource_adaptation_module_ RTC_GUARDED_BY(&encoder_queue_);
// All public methods are proxied to |encoder_queue_|. It must must be
// destroyed first to make sure no tasks are run that use other members.

33
video/video_stream_encoder_unittest.cc
View File

@ -1805,7 +1805,12 @@ TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
test::FrameForwarder new_video_source;
video_stream_encoder_->SetSource(
&new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
// Give the encoder queue time to process the change in degradation preference
// by waiting for an encoded frame.
new_video_source.IncomingCapturedFrame(
CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
sink_.WaitForEncodedFrame(frame_timestamp);
frame_timestamp += kFrameIntervalMs;
// Initially no degradation registered.
VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
@ -1831,6 +1836,12 @@ TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
// Turn off degradation completely.
video_stream_encoder_->SetSource(&new_video_source,
webrtc::DegradationPreference::DISABLED);
// Give the encoder queue time to process the change in degradation preference
// by waiting for an encoded frame.
new_video_source.IncomingCapturedFrame(
CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
sink_.WaitForEncodedFrame(frame_timestamp);
frame_timestamp += kFrameIntervalMs;
VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
video_stream_encoder_->TriggerCpuOveruse();
@ -1845,6 +1856,12 @@ TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
// Calling SetSource with resolution scaling enabled apply the old SinkWants.
video_stream_encoder_->SetSource(
&new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
// Give the encoder queue time to process the change in degradation preference
// by waiting for an encoded frame.
new_video_source.IncomingCapturedFrame(
CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
sink_.WaitForEncodedFrame(frame_timestamp);
frame_timestamp += kFrameIntervalMs;
EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
kFrameWidth * kFrameHeight);
EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
@ -1853,6 +1870,12 @@ TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
// Calling SetSource with framerate scaling enabled apply the old SinkWants.
video_stream_encoder_->SetSource(
&new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
// Give the encoder queue time to process the change in degradation preference
// by waiting for an encoded frame.
new_video_source.IncomingCapturedFrame(
CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
sink_.WaitForEncodedFrame(frame_timestamp);
frame_timestamp += kFrameIntervalMs;
EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
EXPECT_EQ(std::numeric_limits<int>::max(),
new_video_source.sink_wants().max_pixel_count);
@ -2561,12 +2584,16 @@ TEST_F(VideoStreamEncoderTest,
test::FrameForwarder new_video_source;
video_stream_encoder_->SetSource(
&new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
// Give the encoder queue time to process the change in degradation preference
// by waiting for an encoded frame.
new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sink_.WaitForEncodedFrame(3);
VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
// Trigger adapt down, expect reduced framerate.
video_stream_encoder_->TriggerQualityLow();
new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sink_.WaitForEncodedFrame(3);
new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sink_.WaitForEncodedFrame(4);
VerifyFpsLtResolutionMax(new_video_source.sink_wants(), kInputFps);
// Trigger adapt up, expect no restriction.