diff --git a/video/BUILD.gn b/video/BUILD.gn index c77973ef44..a048a2b186 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -194,6 +194,7 @@ rtc_library("video_stream_encoder_impl") { deps = [ "../api:rtp_parameters", + "../api/task_queue:task_queue", "../api/units:data_rate", "../api/video:encoded_image", "../api/video:video_bitrate_allocation", diff --git a/video/overuse_frame_detector.cc b/video/overuse_frame_detector.cc index ade9303ae6..1046c1ea3f 100644 --- a/video/overuse_frame_detector.cc +++ b/video/overuse_frame_detector.cc @@ -540,7 +540,7 @@ OveruseFrameDetector::OveruseFrameDetector( OveruseFrameDetector::~OveruseFrameDetector() {} void OveruseFrameDetector::StartCheckForOveruse( - rtc::TaskQueue* task_queue, + TaskQueueBase* task_queue_base, const CpuOveruseOptions& options, AdaptationObserverInterface* overuse_observer) { RTC_DCHECK_RUN_ON(&task_checker_); @@ -549,7 +549,7 @@ void OveruseFrameDetector::StartCheckForOveruse( SetOptions(options); check_overuse_task_ = RepeatingTaskHandle::DelayedStart( - task_queue->Get(), TimeDelta::ms(kTimeToFirstCheckForOveruseMs), + task_queue_base, TimeDelta::ms(kTimeToFirstCheckForOveruseMs), [this, overuse_observer] { CheckForOveruse(overuse_observer); return TimeDelta::ms(kCheckForOveruseIntervalMs); diff --git a/video/overuse_frame_detector.h b/video/overuse_frame_detector.h index 20e097111b..4f64734944 100644 --- a/video/overuse_frame_detector.h +++ b/video/overuse_frame_detector.h @@ -15,13 +15,13 @@ #include #include "absl/types/optional.h" +#include "api/task_queue/task_queue_base.h" #include "api/video/video_stream_encoder_observer.h" #include "modules/video_coding/utility/quality_scaler.h" #include "rtc_base/constructor_magic.h" #include "rtc_base/experiments/field_trial_parser.h" #include "rtc_base/numerics/exp_filter.h" #include "rtc_base/synchronization/sequence_checker.h" -#include "rtc_base/task_queue.h" #include "rtc_base/task_utils/repeating_task.h" #include "rtc_base/thread_annotations.h" @@ -58,7 +58,7 @@ class OveruseFrameDetector { virtual ~OveruseFrameDetector(); // Start to periodically check for overuse. - void StartCheckForOveruse(rtc::TaskQueue* task_queue, + void StartCheckForOveruse(TaskQueueBase* task_queue_base, const CpuOveruseOptions& options, AdaptationObserverInterface* overuse_observer); diff --git a/video/overuse_frame_detector_resource_adaptation_module.cc b/video/overuse_frame_detector_resource_adaptation_module.cc index 1e449b99be..2bd937baa8 100644 --- a/video/overuse_frame_detector_resource_adaptation_module.cc +++ b/video/overuse_frame_detector_resource_adaptation_module.cc @@ -17,6 +17,7 @@ #include #include "absl/algorithm/container.h" +#include "api/task_queue/task_queue_base.h" #include "api/video/video_source_interface.h" #include "call/adaptation/video_source_restrictions.h" #include "rtc_base/logging.h" @@ -73,10 +74,6 @@ VideoSourceRestrictions ApplyDegradationPreference( // AdaptDown() - only how to modify the source/sink restrictions when this // happens. Note that it is also not responsible for reconfigruring the // source/sink, it is only a keeper of desired restrictions. -// -// Thread safety is ensured between SetHasInputVideoAndDegradationPreference() -// calls on the worker thread and adaptation logic on the encoder task queue -// using a lock. class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor { public: VideoSourceRestrictor() @@ -84,25 +81,19 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor { degradation_preference_(DegradationPreference::DISABLED) {} VideoSourceRestrictions source_restrictions() { - rtc::CritScope lock(&crit_); return source_restrictions_; } - // Inform the restrictor of new source status and degradation preference. - // TODO(hbos): Can this be moved to the encoder queue? If so, the |crit_| lock - // can be removed and we only need a sequence checker. void SetHasInputVideoAndDegradationPreference( bool has_input_video, DegradationPreference degradation_preference) { - // Called on libjingle's worker thread. - RTC_DCHECK_RUN_ON(&main_checker_); - rtc::CritScope lock(&crit_); has_input_video_ = has_input_video; degradation_preference_ = degradation_preference; } + // Updates the source_restrictions(). The source/sink has to be informed of + // this separately. void ClearRestrictions() { - rtc::CritScope lock(&crit_); source_restrictions_ = VideoSourceRestrictions(); } @@ -111,14 +102,9 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor { bool RequestResolutionLowerThan(int pixel_count, int min_pixels_per_frame, bool* min_pixels_reached) { - // Called on the encoder task queue. - rtc::CritScope lock(&crit_); - if (!has_input_video_ || - !IsResolutionScalingEnabled(degradation_preference_)) { - // This can happen since |degradation_preference_| is set on libjingle's - // worker thread but the adaptation is done on the encoder task queue. + RTC_DCHECK(IsResolutionScalingEnabled(degradation_preference_)); + if (!has_input_video_) return false; - } // The input video frame size will have a resolution less than or equal to // |max_pixel_count| depending on how the source can scale the frame size. const int pixels_wanted = (pixel_count * 3) / 5; @@ -145,7 +131,6 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor { // Updates the source_restrictions(). The source/sink has to be informed of // this separately. int RequestFramerateLowerThan(int fps) { - // Called on the encoder task queue. // The input video frame rate will be scaled down to 2/3, rounding down. int framerate_wanted = (fps * 2) / 3; return RestrictFramerate(framerate_wanted) ? framerate_wanted : -1; @@ -164,14 +149,9 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor { // Updates the source_restrictions(). The source/sink has to be informed of // this separately. bool RequestHigherResolutionThan(int pixel_count) { - // Called on the encoder task queue. - rtc::CritScope lock(&crit_); - if (!has_input_video_ || - !IsResolutionScalingEnabled(degradation_preference_)) { - // This can happen since |degradation_preference_| is set on libjingle's - // worker thread but the adaptation is done on the encoder task queue. + RTC_DCHECK(IsResolutionScalingEnabled(degradation_preference_)); + if (!has_input_video_) return false; - } int max_pixels_wanted = pixel_count; if (max_pixels_wanted != std::numeric_limits::max()) max_pixels_wanted = pixel_count * 4; @@ -203,7 +183,6 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor { // adaptation requests are removed completely. In that case, consider // |max_framerate_| to be the current limit (assuming the capturer complies). int RequestHigherFramerateThan(int fps) { - // Called on the encoder task queue. // The input frame rate will be scaled up to the last step, with rounding. int framerate_wanted = fps; if (fps != std::numeric_limits::max()) @@ -215,10 +194,8 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor { // Updates the source_restrictions(). The source/sink has to be informed of // this separately. bool RestrictFramerate(int fps) { - // Called on the encoder task queue. - rtc::CritScope lock(&crit_); - if (!has_input_video_ || - !IsFramerateScalingEnabled(degradation_preference_)) + RTC_DCHECK(IsFramerateScalingEnabled(degradation_preference_)); + if (!has_input_video_) return false; const int fps_wanted = std::max(kMinFramerateFps, fps); @@ -238,10 +215,8 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor { // Updates the source_restrictions(). The source/sink has to be informed of // this separately. bool IncreaseFramerate(int fps) { - // Called on the encoder task queue. - rtc::CritScope lock(&crit_); - if (!has_input_video_ || - !IsFramerateScalingEnabled(degradation_preference_)) + RTC_DCHECK(IsFramerateScalingEnabled(degradation_preference_)); + if (!has_input_video_) return false; const int fps_wanted = std::max(kMinFramerateFps, fps); @@ -259,11 +234,9 @@ class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor { } private: - rtc::CriticalSection crit_; - SequenceChecker main_checker_; - VideoSourceRestrictions source_restrictions_ RTC_GUARDED_BY(&crit_); - bool has_input_video_ RTC_GUARDED_BY(&crit_); - DegradationPreference degradation_preference_ RTC_GUARDED_BY(&crit_); + VideoSourceRestrictions source_restrictions_; + bool has_input_video_; + DegradationPreference degradation_preference_; RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceRestrictor); }; @@ -397,8 +370,7 @@ OveruseFrameDetectorResourceAdaptationModule:: std::unique_ptr overuse_detector, VideoStreamEncoderObserver* encoder_stats_observer, ResourceAdaptationModuleListener* adaptation_listener) - : encoder_queue_(nullptr), - adaptation_listener_(adaptation_listener), + : adaptation_listener_(adaptation_listener), video_stream_encoder_(video_stream_encoder), degradation_preference_(DegradationPreference::DISABLED), adapt_counters_(), @@ -422,24 +394,13 @@ OveruseFrameDetectorResourceAdaptationModule:: OveruseFrameDetectorResourceAdaptationModule:: ~OveruseFrameDetectorResourceAdaptationModule() {} -void OveruseFrameDetectorResourceAdaptationModule::Initialize( - rtc::TaskQueue* encoder_queue) { - RTC_DCHECK(!encoder_queue_); - encoder_queue_ = encoder_queue; - RTC_DCHECK(encoder_queue_); -} - void OveruseFrameDetectorResourceAdaptationModule::SetEncoder( VideoEncoder* encoder) { - RTC_DCHECK(encoder_queue_); - RTC_DCHECK_RUN_ON(encoder_queue_); encoder_ = encoder; } void OveruseFrameDetectorResourceAdaptationModule::StartCheckForOveruse( ResourceAdaptationModuleListener* adaptation_listener) { - RTC_DCHECK(encoder_queue_); - RTC_DCHECK_RUN_ON(encoder_queue_); RTC_DCHECK(encoder_); // TODO(hbos): When AdaptUp() and AdaptDown() are no longer invoked outside // the interval between StartCheckForOveruse() and StopCheckForOveruse(), @@ -449,20 +410,17 @@ void OveruseFrameDetectorResourceAdaptationModule::StartCheckForOveruse( // AdaptUp() and AdaptDown() even when the OveruseDetector is inactive. RTC_DCHECK_EQ(adaptation_listener, adaptation_listener_); overuse_detector_->StartCheckForOveruse( - encoder_queue_, video_stream_encoder_->GetCpuOveruseOptions(), this); + TaskQueueBase::Current(), video_stream_encoder_->GetCpuOveruseOptions(), + this); } void OveruseFrameDetectorResourceAdaptationModule::StopCheckForOveruse() { - RTC_DCHECK(encoder_queue_); - RTC_DCHECK_RUN_ON(encoder_queue_); overuse_detector_->StopCheckForOveruse(); } void OveruseFrameDetectorResourceAdaptationModule::FrameCaptured( const VideoFrame& frame, int64_t time_when_first_seen_us) { - RTC_DCHECK(encoder_queue_); - RTC_DCHECK_RUN_ON(encoder_queue_); overuse_detector_->FrameCaptured(frame, time_when_first_seen_us); } @@ -471,44 +429,32 @@ void OveruseFrameDetectorResourceAdaptationModule::FrameSent( int64_t time_sent_in_us, int64_t capture_time_us, absl::optional encode_duration_us) { - RTC_DCHECK(encoder_queue_); - RTC_DCHECK_RUN_ON(encoder_queue_); overuse_detector_->FrameSent(timestamp, time_sent_in_us, capture_time_us, encode_duration_us); } void OveruseFrameDetectorResourceAdaptationModule::SetLastFramePixelCount( absl::optional last_frame_pixel_count) { - RTC_DCHECK(encoder_queue_); - RTC_DCHECK_RUN_ON(encoder_queue_); last_frame_pixel_count_ = last_frame_pixel_count; } void OveruseFrameDetectorResourceAdaptationModule::SetEncoderConfig( VideoEncoderConfig encoder_config) { - RTC_DCHECK(encoder_queue_); - RTC_DCHECK_RUN_ON(encoder_queue_); encoder_config_ = std::move(encoder_config); } void OveruseFrameDetectorResourceAdaptationModule::SetCodecMaxFramerate( int codec_max_framerate) { - RTC_DCHECK(encoder_queue_); - RTC_DCHECK_RUN_ON(encoder_queue_); codec_max_framerate_ = codec_max_framerate; } void OveruseFrameDetectorResourceAdaptationModule::SetEncoderStartBitrateBps( uint32_t encoder_start_bitrate_bps) { - RTC_DCHECK(encoder_queue_); - RTC_DCHECK_RUN_ON(encoder_queue_); encoder_start_bitrate_bps_ = encoder_start_bitrate_bps; } void OveruseFrameDetectorResourceAdaptationModule::SetIsQualityScalerEnabled( bool is_quality_scaler_enabled) { - RTC_DCHECK(encoder_queue_); - RTC_DCHECK_RUN_ON(encoder_queue_); is_quality_scaler_enabled_ = is_quality_scaler_enabled; } @@ -516,43 +462,25 @@ void OveruseFrameDetectorResourceAdaptationModule:: SetHasInputVideoAndDegradationPreference( bool has_input_video, DegradationPreference degradation_preference) { - // TODO(https://crbug.com/webrtc/11222): Move this call to the encoder queue, - // making VideoSourceRestrictor single-threaded and removing the only call to - // MaybeUpdateVideoSourceRestrictions() that isn't on the |encoder_queue_|. - source_restrictor_->SetHasInputVideoAndDegradationPreference( - has_input_video, degradation_preference); - MaybeUpdateVideoSourceRestrictions(degradation_preference); - encoder_queue_->PostTask([this, degradation_preference] { - RTC_DCHECK_RUN_ON(encoder_queue_); - if (degradation_preference_ != degradation_preference) { - // Reset adaptation state, so that we're not tricked into thinking there's - // an already pending request of the same type. - last_adaptation_request_.reset(); - if (degradation_preference == DegradationPreference::BALANCED || - degradation_preference_ == DegradationPreference::BALANCED) { - // TODO(asapersson): Consider removing |adapt_counters_| map and use one - // AdaptCounter for all modes. - source_restrictor_->ClearRestrictions(); - adapt_counters_.clear(); - } + if (degradation_preference_ != degradation_preference) { + // Reset adaptation state, so that we're not tricked into thinking there's + // an already pending request of the same type. + last_adaptation_request_.reset(); + if (degradation_preference == DegradationPreference::BALANCED || + degradation_preference_ == DegradationPreference::BALANCED) { + // TODO(asapersson): Consider removing |adapt_counters_| map and use one + // AdaptCounter for all modes. + source_restrictor_->ClearRestrictions(); + adapt_counters_.clear(); } - degradation_preference_ = degradation_preference; - // This is the second time we're invoking - // MaybeUpdateVideoSourceRestrictions() in this method. This is because - // current tests expect the changes to the source restrictions to be - // immediate (outside of the encoder queue) while it is possible that they - // change again after ClearRestrictions() on the encoder queue. - // TODO(https://crbug.com/webrtc/11222): Change the expectations to allow - // source restrictions only to change on the encoder queue. This unblocks - // making OveruseFrameDetectorResourceAdaptationModule and - // VideoSourceRestrictor single-threaded. - MaybeUpdateVideoSourceRestrictions(degradation_preference_); - }); + } + degradation_preference_ = degradation_preference; + source_restrictor_->SetHasInputVideoAndDegradationPreference( + has_input_video, degradation_preference_); + MaybeUpdateVideoSourceRestrictions(); } void OveruseFrameDetectorResourceAdaptationModule::RefreshTargetFramerate() { - RTC_DCHECK(encoder_queue_); - RTC_DCHECK_RUN_ON(encoder_queue_); absl::optional restricted_frame_rate = ApplyDegradationPreference(source_restrictor_->source_restrictions(), degradation_preference_) @@ -571,17 +499,13 @@ void OveruseFrameDetectorResourceAdaptationModule::RefreshTargetFramerate() { } void OveruseFrameDetectorResourceAdaptationModule::ResetAdaptationCounters() { - RTC_DCHECK(encoder_queue_); - RTC_DCHECK_RUN_ON(encoder_queue_); last_adaptation_request_.reset(); source_restrictor_->ClearRestrictions(); adapt_counters_.clear(); - MaybeUpdateVideoSourceRestrictions(degradation_preference_); + MaybeUpdateVideoSourceRestrictions(); } void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) { - RTC_DCHECK(encoder_queue_); - RTC_DCHECK_RUN_ON(encoder_queue_); const AdaptCounter& adapt_counter = GetConstAdaptCounter(); int num_downgrades = adapt_counter.TotalCount(reason); if (num_downgrades == 0) @@ -685,7 +609,7 @@ void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) { // Tell the adaptation listener to reconfigure the source for us according to // the latest adaptation. - MaybeUpdateVideoSourceRestrictions(degradation_preference_); + MaybeUpdateVideoSourceRestrictions(); last_adaptation_request_.emplace(adaptation_request); @@ -696,8 +620,6 @@ void OveruseFrameDetectorResourceAdaptationModule::AdaptUp(AdaptReason reason) { bool OveruseFrameDetectorResourceAdaptationModule::AdaptDown( AdaptReason reason) { - RTC_DCHECK(encoder_queue_); - RTC_DCHECK_RUN_ON(encoder_queue_); AdaptationRequest adaptation_request = { *last_frame_pixel_count_, encoder_stats_observer_->GetInputFrameRate(), AdaptationRequest::Mode::kAdaptDown}; @@ -791,7 +713,7 @@ bool OveruseFrameDetectorResourceAdaptationModule::AdaptDown( // Tell the adaptation listener to reconfigure the source for us according to // the latest adaptation. - MaybeUpdateVideoSourceRestrictions(degradation_preference_); + MaybeUpdateVideoSourceRestrictions(); last_adaptation_request_.emplace(adaptation_request); @@ -802,21 +724,13 @@ bool OveruseFrameDetectorResourceAdaptationModule::AdaptDown( } void OveruseFrameDetectorResourceAdaptationModule:: - MaybeUpdateVideoSourceRestrictions( - DegradationPreference degradation_preference) { - absl::optional updated_restrictions; - { - rtc::CritScope lock(&video_source_restrictions_crit_); - VideoSourceRestrictions new_restrictions = ApplyDegradationPreference( - source_restrictor_->source_restrictions(), degradation_preference); - if (video_source_restrictions_ != new_restrictions) { - video_source_restrictions_ = std::move(new_restrictions); - updated_restrictions = video_source_restrictions_; - } - } - if (updated_restrictions.has_value()) { + MaybeUpdateVideoSourceRestrictions() { + VideoSourceRestrictions new_restrictions = ApplyDegradationPreference( + source_restrictor_->source_restrictions(), degradation_preference_); + if (video_source_restrictions_ != new_restrictions) { + video_source_restrictions_ = std::move(new_restrictions); adaptation_listener_->OnVideoSourceRestrictionsUpdated( - updated_restrictions.value()); + video_source_restrictions_); } } @@ -840,8 +754,6 @@ void OveruseFrameDetectorResourceAdaptationModule::UpdateAdaptationStats( VideoStreamEncoderObserver::AdaptationSteps OveruseFrameDetectorResourceAdaptationModule::GetActiveCounts( AdaptReason reason) { - RTC_DCHECK(encoder_queue_); - RTC_DCHECK_RUN_ON(encoder_queue_); VideoStreamEncoderObserver::AdaptationSteps counts = GetConstAdaptCounter().Counts(reason); switch (reason) { @@ -885,15 +797,11 @@ OveruseFrameDetectorResourceAdaptationModule::GetAdaptCounter() { const OveruseFrameDetectorResourceAdaptationModule::AdaptCounter& OveruseFrameDetectorResourceAdaptationModule::GetConstAdaptCounter() { - RTC_DCHECK(encoder_queue_); - RTC_DCHECK_RUN_ON(encoder_queue_); return adapt_counters_[degradation_preference_]; } absl::optional OveruseFrameDetectorResourceAdaptationModule::GetQpThresholds() const { - RTC_DCHECK(encoder_queue_); - RTC_DCHECK_RUN_ON(encoder_queue_); RTC_DCHECK(last_frame_pixel_count_.has_value()); return balanced_settings_.GetQpThresholds(encoder_config_.codec_type, last_frame_pixel_count_.value()); diff --git a/video/overuse_frame_detector_resource_adaptation_module.h b/video/overuse_frame_detector_resource_adaptation_module.h index dcd78e83d9..28902cf2b9 100644 --- a/video/overuse_frame_detector_resource_adaptation_module.h +++ b/video/overuse_frame_detector_resource_adaptation_module.h @@ -36,6 +36,9 @@ class VideoStreamEncoder; // resolution up or down based on encode usage percent. It keeps track of video // source settings, adaptation counters and may get influenced by // VideoStreamEncoder's quality scaler through AdaptUp() and AdaptDown() calls. +// +// This class is single-threaded. The caller is responsible for ensuring safe +// usage. // TODO(hbos): Reduce the coupling with VideoStreamEncoder. // TODO(hbos): Add unittests specific to this class, it is currently only tested // indirectly in video_stream_encoder_unittest.cc and other tests exercising @@ -48,6 +51,8 @@ class OveruseFrameDetectorResourceAdaptationModule : public ResourceAdaptationModuleInterface, public AdaptationObserverInterface { public: + // The module can be constructed on any sequence, but must be initialized and + // used on a single sequence, e.g. the encoder queue. OveruseFrameDetectorResourceAdaptationModule( VideoStreamEncoder* video_stream_encoder, std::unique_ptr overuse_detector, @@ -55,7 +60,6 @@ class OveruseFrameDetectorResourceAdaptationModule ResourceAdaptationModuleListener* adaptation_listener); ~OveruseFrameDetectorResourceAdaptationModule() override; - void Initialize(rtc::TaskQueue* encoder_queue); // Sets the encoder to reconfigure based on overuse. // TODO(hbos): Don't reconfigure the encoder directly. Instead, define the // output of a resource adaptation module as a struct and let the @@ -63,8 +67,6 @@ class OveruseFrameDetectorResourceAdaptationModule void SetEncoder(VideoEncoder* encoder); DegradationPreference degradation_preference() const { - RTC_DCHECK(encoder_queue_); - RTC_DCHECK_RUN_ON(encoder_queue_); return degradation_preference_; } @@ -187,60 +189,39 @@ class OveruseFrameDetectorResourceAdaptationModule // Makes |video_source_restrictions_| up-to-date and informs the // |adaptation_listener_| if restrictions are changed, allowing the listener // to reconfigure the source accordingly. - // TODO(https://crbug.com/webrtc/11222): When - // SetHasInputVideoAndDegradationPreference() stops calling this method prior - // to updating |degradation_preference_| on the encoder queue, remove its - // argument in favor of using |degradation_preference_| directly. - void MaybeUpdateVideoSourceRestrictions( - DegradationPreference degradation_preference); + void MaybeUpdateVideoSourceRestrictions(); - void UpdateAdaptationStats(AdaptReason reason) RTC_RUN_ON(encoder_queue_); - DegradationPreference EffectiveDegradataionPreference() - RTC_RUN_ON(encoder_queue_); - AdaptCounter& GetAdaptCounter() RTC_RUN_ON(encoder_queue_); - bool CanAdaptUpResolution(int pixels, uint32_t bitrate_bps) const - RTC_RUN_ON(encoder_queue_); + void UpdateAdaptationStats(AdaptReason reason); + DegradationPreference EffectiveDegradataionPreference(); + AdaptCounter& GetAdaptCounter(); + bool CanAdaptUpResolution(int pixels, uint32_t bitrate_bps) const; - rtc::TaskQueue* encoder_queue_; - // TODO(https://crbug.com/webrtc/11222): Update - // SetHasInputVideoAndDegradationPreference() to do all work on the encoder - // queue (including |source_restrictor_| and |adaptation_listener_| usage). - // When this is the case, remove |VideoSourceRestrictor::crit_| and - // |video_source_restrictions_crit_| and replace |encoder_queue_| with a - // sequence checker. - rtc::CriticalSection video_source_restrictions_crit_; ResourceAdaptationModuleListener* const adaptation_listener_; // The restrictions that |adaptation_listener_| is informed of. - VideoSourceRestrictions video_source_restrictions_ - RTC_GUARDED_BY(&video_source_restrictions_crit_); + VideoSourceRestrictions video_source_restrictions_; // Used to query CpuOveruseOptions at StartCheckForOveruse(). - VideoStreamEncoder* video_stream_encoder_ RTC_GUARDED_BY(encoder_queue_); - DegradationPreference degradation_preference_ RTC_GUARDED_BY(encoder_queue_); + VideoStreamEncoder* video_stream_encoder_; + DegradationPreference degradation_preference_; // Counters used for deciding if the video resolution or framerate is // currently restricted, and if so, why, on a per degradation preference // basis. // TODO(sprang): Replace this with a state holding a relative overuse measure // instead, that can be translated into suitable down-scale or fps limit. - std::map adapt_counters_ - RTC_GUARDED_BY(encoder_queue_); - const BalancedDegradationSettings balanced_settings_ - RTC_GUARDED_BY(encoder_queue_); + std::map adapt_counters_; + const BalancedDegradationSettings balanced_settings_; // Stores a snapshot of the last adaptation request triggered by an AdaptUp // or AdaptDown signal. - absl::optional last_adaptation_request_ - RTC_GUARDED_BY(encoder_queue_); - absl::optional last_frame_pixel_count_ RTC_GUARDED_BY(encoder_queue_); + absl::optional last_adaptation_request_; + absl::optional last_frame_pixel_count_; // Keeps track of source restrictions that this adaptation module outputs. const std::unique_ptr source_restrictor_; - const std::unique_ptr overuse_detector_ - RTC_PT_GUARDED_BY(encoder_queue_); - int codec_max_framerate_ RTC_GUARDED_BY(encoder_queue_); - uint32_t encoder_start_bitrate_bps_ RTC_GUARDED_BY(encoder_queue_); - bool is_quality_scaler_enabled_ RTC_GUARDED_BY(encoder_queue_); - VideoEncoderConfig encoder_config_ RTC_GUARDED_BY(encoder_queue_); - VideoEncoder* encoder_ RTC_GUARDED_BY(encoder_queue_); - VideoStreamEncoderObserver* const encoder_stats_observer_ - RTC_GUARDED_BY(encoder_queue_); + const std::unique_ptr overuse_detector_; + int codec_max_framerate_; + uint32_t encoder_start_bitrate_bps_; + bool is_quality_scaler_enabled_; + VideoEncoderConfig encoder_config_; + VideoEncoder* encoder_; + VideoStreamEncoderObserver* const encoder_stats_observer_; }; } // namespace webrtc diff --git a/video/overuse_frame_detector_unittest.cc b/video/overuse_frame_detector_unittest.cc index 7d8217ccf9..432cc271bd 100644 --- a/video/overuse_frame_detector_unittest.cc +++ b/video/overuse_frame_detector_unittest.cc @@ -433,7 +433,8 @@ TEST_F(OveruseFrameDetectorTest, RunOnTqNormalUsage) { queue.SendTask( [&] { - overuse_detector_->StartCheckForOveruse(&queue, options_, observer_); + overuse_detector_->StartCheckForOveruse(queue.Get(), options_, + observer_); }, RTC_FROM_HERE); @@ -914,7 +915,8 @@ TEST_F(OveruseFrameDetectorTest2, RunOnTqNormalUsage) { queue.SendTask( [&] { - overuse_detector_->StartCheckForOveruse(&queue, options_, observer_); + overuse_detector_->StartCheckForOveruse(queue.Get(), options_, + observer_); }, RTC_FROM_HERE); diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 3ddbf58677..eecd7de9da 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -332,7 +332,6 @@ VideoStreamEncoder::VideoStreamEncoder( TaskQueueFactory::Priority::NORMAL)) { RTC_DCHECK(encoder_stats_observer); RTC_DCHECK_GE(number_of_cores, 1); - resource_adaptation_module_->Initialize(encoder_queue()); for (auto& state : encoder_buffer_state_) state.fill(std::numeric_limits::max()); @@ -388,10 +387,10 @@ void VideoStreamEncoder::SetSource( const DegradationPreference& degradation_preference) { RTC_DCHECK_RUN_ON(&thread_checker_); video_source_sink_controller_->SetSource(source); - resource_adaptation_module_->SetHasInputVideoAndDegradationPreference( - source, degradation_preference); - encoder_queue_.PostTask([this, degradation_preference] { + encoder_queue_.PostTask([this, source, degradation_preference] { RTC_DCHECK_RUN_ON(&encoder_queue_); + resource_adaptation_module_->SetHasInputVideoAndDegradationPreference( + source, degradation_preference); if (encoder_) ConfigureQualityScaler(encoder_->GetEncoderInfo()); @@ -1731,19 +1730,19 @@ CpuOveruseOptions VideoStreamEncoder::GetCpuOveruseOptions() const { bool VideoStreamEncoder::TriggerAdaptDown( AdaptationObserverInterface::AdaptReason reason) { + RTC_DCHECK_RUN_ON(&encoder_queue_); return resource_adaptation_module_->AdaptDown(reason); } void VideoStreamEncoder::TriggerAdaptUp( AdaptationObserverInterface::AdaptReason reason) { + RTC_DCHECK_RUN_ON(&encoder_queue_); resource_adaptation_module_->AdaptUp(reason); } void VideoStreamEncoder::OnVideoSourceRestrictionsUpdated( VideoSourceRestrictions restrictions) { - // TODO(https://crbug.com/webrtc/11222): DCHECK that we are using the - // |encoder_queue_| when OnVideoSourceRestrictionsUpdated() is no longer - // invoked off this thread due to VideoStreamEncoder::SetSource() stuff. + RTC_DCHECK_RUN_ON(&encoder_queue_); video_source_sink_controller_->SetRestrictions(std::move(restrictions)); video_source_sink_controller_->PushSourceSinkSettings(); } diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 6db3d53da0..bd76ee5f9c 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -408,9 +408,20 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, // track of whether a request has been made or not. bool encoder_switch_requested_ RTC_GUARDED_BY(&encoder_queue_); + // The controller updates the sink wants based on restrictions that come from + // the resource adaptation module or adaptation due to bandwidth adaptation. + // + // This is used on the encoder queue, with a few exceptions: + // - VideoStreamEncoder::SetSource() invokes SetSource(). + // - VideoStreamEncoder::SetSink() invokes SetRotationApplied() and + // PushSourceSinkSettings(). + // - VideoStreamEncoder::Stop() invokes SetSource(). + // TODO(hbos): If these can be moved to the encoder queue, + // VideoSourceSinkController can be made single-threaded, and its lock can be + // replaced with a sequence checker. std::unique_ptr video_source_sink_controller_; std::unique_ptr - resource_adaptation_module_; + resource_adaptation_module_ RTC_GUARDED_BY(&encoder_queue_); // All public methods are proxied to |encoder_queue_|. It must must be // destroyed first to make sure no tasks are run that use other members. diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index 4b4cd2d67f..e5439f33d3 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -1805,7 +1805,12 @@ TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) { test::FrameForwarder new_video_source; video_stream_encoder_->SetSource( &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION); - + // Give the encoder queue time to process the change in degradation preference + // by waiting for an encoded frame. + new_video_source.IncomingCapturedFrame( + CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth)); + sink_.WaitForEncodedFrame(frame_timestamp); + frame_timestamp += kFrameIntervalMs; // Initially no degradation registered. VerifyFpsMaxResolutionMax(new_video_source.sink_wants()); @@ -1831,6 +1836,12 @@ TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) { // Turn off degradation completely. video_stream_encoder_->SetSource(&new_video_source, webrtc::DegradationPreference::DISABLED); + // Give the encoder queue time to process the change in degradation preference + // by waiting for an encoded frame. + new_video_source.IncomingCapturedFrame( + CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth)); + sink_.WaitForEncodedFrame(frame_timestamp); + frame_timestamp += kFrameIntervalMs; VerifyFpsMaxResolutionMax(new_video_source.sink_wants()); video_stream_encoder_->TriggerCpuOveruse(); @@ -1845,6 +1856,12 @@ TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) { // Calling SetSource with resolution scaling enabled apply the old SinkWants. video_stream_encoder_->SetSource( &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE); + // Give the encoder queue time to process the change in degradation preference + // by waiting for an encoded frame. + new_video_source.IncomingCapturedFrame( + CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth)); + sink_.WaitForEncodedFrame(frame_timestamp); + frame_timestamp += kFrameIntervalMs; EXPECT_LT(new_video_source.sink_wants().max_pixel_count, kFrameWidth * kFrameHeight); EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count); @@ -1853,6 +1870,12 @@ TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) { // Calling SetSource with framerate scaling enabled apply the old SinkWants. video_stream_encoder_->SetSource( &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION); + // Give the encoder queue time to process the change in degradation preference + // by waiting for an encoded frame. + new_video_source.IncomingCapturedFrame( + CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth)); + sink_.WaitForEncodedFrame(frame_timestamp); + frame_timestamp += kFrameIntervalMs; EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count); EXPECT_EQ(std::numeric_limits::max(), new_video_source.sink_wants().max_pixel_count); @@ -2561,12 +2584,16 @@ TEST_F(VideoStreamEncoderTest, test::FrameForwarder new_video_source; video_stream_encoder_->SetSource( &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION); + // Give the encoder queue time to process the change in degradation preference + // by waiting for an encoded frame. + new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight)); + sink_.WaitForEncodedFrame(3); VerifyFpsMaxResolutionMax(new_video_source.sink_wants()); // Trigger adapt down, expect reduced framerate. video_stream_encoder_->TriggerQualityLow(); - new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight)); - sink_.WaitForEncodedFrame(3); + new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight)); + sink_.WaitForEncodedFrame(4); VerifyFpsLtResolutionMax(new_video_source.sink_wants(), kInputFps); // Trigger adapt up, expect no restriction.