[Adaptation] Adaptation holds restrictions
This refactoring enables us to do multi-step adaptations instead of the normal single step adaptations. This work is required pre-requisite work to remove the DropDueToSize methods from the ResourceAdaptationProcessorInterface. This work also gives a path to fixing webrtc:11719, since double adaptations can be done in a controlled manner with this API. Bug: webrtc:11700,webrtc:11719 Change-Id: Ic7d27a49c82d0cf7ebaf79d7bdf0bc6f382770ed Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/178200 Commit-Queue: Evan Shrubsole <eshr@google.com> Reviewed-by: Henrik Boström <hbos@webrtc.org> Cr-Commit-Position: refs/heads/master@{#31631}
This commit is contained in:
Evan Shrubsole
committed by
Commit Bot
Commit Bot
parent
9b79ad33af
commit
d7d2f27b1b
@ -55,6 +55,7 @@ rtc_library("resource_adaptation") {
|
||||
absl_deps = [
|
||||
"//third_party/abseil-cpp/absl/algorithm:container",
|
||||
"//third_party/abseil-cpp/absl/types:optional",
|
||||
"//third_party/abseil-cpp/absl/types:variant",
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
@ -281,9 +281,7 @@ ResourceAdaptationProcessor::OnResourceUnderuse(
|
||||
}
|
||||
VideoSourceRestrictions restrictions_before =
|
||||
stream_adapter_->source_restrictions();
|
||||
VideoStreamAdapter::RestrictionsWithCounters peek_restrictions =
|
||||
stream_adapter_->PeekNextRestrictions(adaptation);
|
||||
VideoSourceRestrictions restrictions_after = peek_restrictions.restrictions;
|
||||
VideoSourceRestrictions restrictions_after = adaptation.restrictions();
|
||||
// Check that resource is most limited...
|
||||
std::vector<rtc::scoped_refptr<Resource>> most_limited_resources;
|
||||
VideoStreamAdapter::RestrictionsWithCounters most_limited_restrictions;
|
||||
@ -305,7 +303,7 @@ ResourceAdaptationProcessor::OnResourceUnderuse(
|
||||
// If the most restricted resource is less limited than current restrictions
|
||||
// then proceed with adapting up.
|
||||
if (!most_limited_resources.empty() &&
|
||||
most_limited_restrictions.adaptation_counters.Total() >=
|
||||
most_limited_restrictions.counters.Total() >=
|
||||
stream_adapter_->adaptation_counters().Total()) {
|
||||
// If |reason_resource| is not one of the most limiting resources then abort
|
||||
// adaptation.
|
||||
@ -322,7 +320,8 @@ ResourceAdaptationProcessor::OnResourceUnderuse(
|
||||
if (most_limited_resources.size() > 1) {
|
||||
// If there are multiple most limited resources, all must signal underuse
|
||||
// before the adaptation is applied.
|
||||
UpdateResourceLimitations(reason_resource, peek_restrictions);
|
||||
UpdateResourceLimitations(reason_resource, adaptation.restrictions(),
|
||||
adaptation.counters());
|
||||
processing_in_progress_ = false;
|
||||
rtc::StringBuilder message;
|
||||
message << "Resource \"" << reason_resource->Name()
|
||||
@ -374,11 +373,10 @@ ResourceAdaptationProcessor::OnResourceOveruse(
|
||||
// Apply adaptation.
|
||||
VideoSourceRestrictions restrictions_before =
|
||||
stream_adapter_->source_restrictions();
|
||||
VideoStreamAdapter::RestrictionsWithCounters peek_next_restrictions =
|
||||
stream_adapter_->PeekNextRestrictions(adaptation);
|
||||
VideoSourceRestrictions restrictions_after =
|
||||
peek_next_restrictions.restrictions;
|
||||
stream_adapter_->ApplyAdaptation(adaptation, reason_resource);
|
||||
VideoSourceRestrictions restrictions_after = adaptation.restrictions();
|
||||
UpdateResourceLimitations(reason_resource, adaptation.restrictions(),
|
||||
adaptation.counters());
|
||||
stream_adapter_->ApplyAdaptation(adaptation, nullptr);
|
||||
for (auto* adaptation_listener : adaptation_listeners_) {
|
||||
adaptation_listener->OnAdaptationApplied(
|
||||
adaptation.input_state(), restrictions_before, restrictions_after,
|
||||
@ -423,13 +421,13 @@ ResourceAdaptationProcessor::FindMostLimitedResources() const {
|
||||
adaptation_limits_by_resources_) {
|
||||
const auto& restrictions_with_counters =
|
||||
resource_and_adaptation_limit_.second;
|
||||
if (restrictions_with_counters.adaptation_counters.Total() >
|
||||
most_limited_restrictions.adaptation_counters.Total()) {
|
||||
if (restrictions_with_counters.counters.Total() >
|
||||
most_limited_restrictions.counters.Total()) {
|
||||
most_limited_restrictions = restrictions_with_counters;
|
||||
most_limited_resources.clear();
|
||||
most_limited_resources.push_back(resource_and_adaptation_limit_.first);
|
||||
} else if (most_limited_restrictions.adaptation_counters ==
|
||||
restrictions_with_counters.adaptation_counters) {
|
||||
} else if (most_limited_restrictions.counters ==
|
||||
restrictions_with_counters.counters) {
|
||||
most_limited_resources.push_back(resource_and_adaptation_limit_.first);
|
||||
}
|
||||
}
|
||||
@ -439,16 +437,18 @@ ResourceAdaptationProcessor::FindMostLimitedResources() const {
|
||||
|
||||
void ResourceAdaptationProcessor::UpdateResourceLimitations(
|
||||
rtc::scoped_refptr<Resource> reason_resource,
|
||||
const VideoStreamAdapter::RestrictionsWithCounters& restrictions) {
|
||||
const VideoSourceRestrictions& restrictions,
|
||||
const VideoAdaptationCounters& counters) {
|
||||
auto& adaptation_limits = adaptation_limits_by_resources_[reason_resource];
|
||||
if (adaptation_limits == restrictions) {
|
||||
if (adaptation_limits.restrictions == restrictions &&
|
||||
adaptation_limits.counters == counters) {
|
||||
return;
|
||||
}
|
||||
adaptation_limits = restrictions;
|
||||
adaptation_limits = {restrictions, counters};
|
||||
|
||||
std::map<rtc::scoped_refptr<Resource>, VideoAdaptationCounters> limitations;
|
||||
for (const auto& p : adaptation_limits_by_resources_) {
|
||||
limitations.insert(std::make_pair(p.first, p.second.adaptation_counters));
|
||||
limitations.insert(std::make_pair(p.first, p.second.counters));
|
||||
}
|
||||
for (auto limitations_listener : resource_limitations_listeners_) {
|
||||
limitations_listener->OnResourceLimitationChanged(reason_resource,
|
||||
@ -468,8 +468,7 @@ void ResourceAdaptationProcessor::
|
||||
VideoStreamAdapter::RestrictionsWithCounters most_limited =
|
||||
FindMostLimitedResources().second;
|
||||
|
||||
if (removed_limitations.adaptation_counters.Total() <=
|
||||
most_limited.adaptation_counters.Total()) {
|
||||
if (removed_limitations.counters.Total() <= most_limited.counters.Total()) {
|
||||
// The removed limitations were less limited than the most limited resource.
|
||||
// Don't change the current restrictions.
|
||||
return;
|
||||
@ -477,7 +476,7 @@ void ResourceAdaptationProcessor::
|
||||
|
||||
// Apply the new most limited resource as the next restrictions.
|
||||
Adaptation adapt_to = stream_adapter_->GetAdaptationTo(
|
||||
most_limited.adaptation_counters, most_limited.restrictions);
|
||||
most_limited.counters, most_limited.restrictions);
|
||||
RTC_DCHECK_EQ(adapt_to.status(), Adaptation::Status::kValid);
|
||||
stream_adapter_->ApplyAdaptation(adapt_to, nullptr);
|
||||
for (auto* adaptation_listener : adaptation_listeners_) {
|
||||
@ -489,7 +488,7 @@ void ResourceAdaptationProcessor::
|
||||
RTC_LOG(INFO) << "Most limited resource removed. Restoring restrictions to "
|
||||
"next most limited restrictions: "
|
||||
<< most_limited.restrictions.ToString() << " with counters "
|
||||
<< most_limited.adaptation_counters.ToString();
|
||||
<< most_limited.counters.ToString();
|
||||
}
|
||||
|
||||
void ResourceAdaptationProcessor::OnVideoSourceRestrictionsUpdated(
|
||||
@ -499,8 +498,8 @@ void ResourceAdaptationProcessor::OnVideoSourceRestrictionsUpdated(
|
||||
const VideoSourceRestrictions& unfiltered_restrictions) {
|
||||
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
|
||||
if (reason) {
|
||||
UpdateResourceLimitations(reason,
|
||||
{unfiltered_restrictions, adaptation_counters});
|
||||
UpdateResourceLimitations(reason, unfiltered_restrictions,
|
||||
adaptation_counters);
|
||||
} else if (adaptation_counters.Total() == 0) {
|
||||
// Adaptations are cleared.
|
||||
adaptation_limits_by_resources_.clear();
|
||||
|
||||
@ -22,6 +22,7 @@
|
||||
#include "api/rtp_parameters.h"
|
||||
#include "api/scoped_refptr.h"
|
||||
#include "api/task_queue/task_queue_base.h"
|
||||
#include "api/video/video_adaptation_counters.h"
|
||||
#include "api/video/video_frame.h"
|
||||
#include "api/video/video_stream_encoder_observer.h"
|
||||
#include "call/adaptation/adaptation_constraint.h"
|
||||
@ -151,10 +152,10 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface,
|
||||
// changes to ensure |effective_degradation_preference_| is up-to-date.
|
||||
void MaybeUpdateEffectiveDegradationPreference();
|
||||
|
||||
void UpdateResourceLimitations(
|
||||
rtc::scoped_refptr<Resource> reason_resource,
|
||||
const VideoStreamAdapter::RestrictionsWithCounters&
|
||||
peek_next_restrictions) RTC_RUN_ON(resource_adaptation_queue_);
|
||||
void UpdateResourceLimitations(rtc::scoped_refptr<Resource> reason_resource,
|
||||
const VideoSourceRestrictions& restrictions,
|
||||
const VideoAdaptationCounters& counters)
|
||||
RTC_RUN_ON(resource_adaptation_queue_);
|
||||
|
||||
// Searches |adaptation_limits_by_resources_| for each resource with the
|
||||
// highest total adaptation counts. Adaptation up may only occur if the
|
||||
|
||||
@ -15,11 +15,17 @@
|
||||
#include <utility>
|
||||
|
||||
#include "absl/types/optional.h"
|
||||
#include "absl/types/variant.h"
|
||||
#include "api/video/video_adaptation_counters.h"
|
||||
#include "api/video/video_adaptation_reason.h"
|
||||
#include "api/video_codecs/video_encoder.h"
|
||||
#include "call/adaptation/video_source_restrictions.h"
|
||||
#include "call/adaptation/video_stream_input_state.h"
|
||||
#include "rtc_base/checks.h"
|
||||
#include "rtc_base/constructor_magic.h"
|
||||
#include "rtc_base/logging.h"
|
||||
#include "rtc_base/numerics/safe_conversions.h"
|
||||
#include "rtc_base/synchronization/sequence_checker.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
@ -27,13 +33,6 @@ const int kMinFrameRateFps = 2;
|
||||
|
||||
namespace {
|
||||
|
||||
// Generate suggested higher and lower frame rates and resolutions, to be
|
||||
// applied to the VideoSourceRestrictor. These are used in "maintain-resolution"
|
||||
// and "maintain-framerate". The "balanced" degradation preference also makes
|
||||
// use of BalancedDegradationPreference when generating suggestions. The
|
||||
// VideoSourceRestrictor decidedes whether or not a proposed adaptation is
|
||||
// valid.
|
||||
|
||||
// For frame rate, the steps we take are 2/3 (down) and 3/2 (up).
|
||||
int GetLowerFrameRateThan(int fps) {
|
||||
RTC_DCHECK(fps != std::numeric_limits<int>::max());
|
||||
@ -59,6 +58,56 @@ int GetLowerResolutionThan(int pixel_count) {
|
||||
return (pixel_count * 3) / 5;
|
||||
}
|
||||
|
||||
int GetIncreasedMaxPixelsWanted(int target_pixels) {
|
||||
if (target_pixels == std::numeric_limits<int>::max())
|
||||
return std::numeric_limits<int>::max();
|
||||
// When we decrease resolution, we go down to at most 3/5 of current pixels.
|
||||
// Thus to increase resolution, we need 3/5 to get back to where we started.
|
||||
// When going up, the desired max_pixels_per_frame() has to be significantly
|
||||
// higher than the target because the source's native resolutions might not
|
||||
// match the target. We pick 12/5 of the target.
|
||||
//
|
||||
// (This value was historically 4 times the old target, which is (3/5)*4 of
|
||||
// the new target - or 12/5 - assuming the target is adjusted according to
|
||||
// the above steps.)
|
||||
RTC_DCHECK(target_pixels != std::numeric_limits<int>::max());
|
||||
return (target_pixels * 12) / 5;
|
||||
}
|
||||
|
||||
bool CanDecreaseResolutionTo(int target_pixels,
|
||||
const VideoStreamInputState& input_state,
|
||||
const VideoSourceRestrictions& restrictions) {
|
||||
int max_pixels_per_frame =
|
||||
rtc::dchecked_cast<int>(restrictions.max_pixels_per_frame().value_or(
|
||||
std::numeric_limits<int>::max()));
|
||||
return target_pixels < max_pixels_per_frame &&
|
||||
target_pixels >= input_state.min_pixels_per_frame();
|
||||
}
|
||||
|
||||
bool CanIncreaseResolutionTo(int target_pixels,
|
||||
const VideoSourceRestrictions& restrictions) {
|
||||
int max_pixels_wanted = GetIncreasedMaxPixelsWanted(target_pixels);
|
||||
int max_pixels_per_frame =
|
||||
rtc::dchecked_cast<int>(restrictions.max_pixels_per_frame().value_or(
|
||||
std::numeric_limits<int>::max()));
|
||||
return max_pixels_wanted > max_pixels_per_frame;
|
||||
}
|
||||
|
||||
bool CanDecreaseFrameRateTo(int max_frame_rate,
|
||||
const VideoSourceRestrictions& restrictions) {
|
||||
const int fps_wanted = std::max(kMinFrameRateFps, max_frame_rate);
|
||||
return fps_wanted <
|
||||
rtc::dchecked_cast<int>(restrictions.max_frame_rate().value_or(
|
||||
std::numeric_limits<int>::max()));
|
||||
}
|
||||
|
||||
bool CanIncreaseFrameRateTo(int max_frame_rate,
|
||||
const VideoSourceRestrictions& restrictions) {
|
||||
return max_frame_rate >
|
||||
rtc::dchecked_cast<int>(restrictions.max_frame_rate().value_or(
|
||||
std::numeric_limits<int>::max()));
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
VideoSourceRestrictionsListener::~VideoSourceRestrictionsListener() = default;
|
||||
@ -105,45 +154,17 @@ const char* Adaptation::StatusToString(Adaptation::Status status) {
|
||||
}
|
||||
}
|
||||
|
||||
Adaptation::Step::Step(StepType type, int target) : type(type), target(target) {
|
||||
RTC_DCHECK_NE(type, Adaptation::StepType::kForce);
|
||||
}
|
||||
|
||||
Adaptation::Step::Step(VideoSourceRestrictions restrictions,
|
||||
VideoAdaptationCounters counters)
|
||||
: type(Adaptation::StepType::kForce),
|
||||
restrictions(restrictions),
|
||||
counters(counters) {}
|
||||
|
||||
Adaptation::Adaptation(int validation_id,
|
||||
Step step,
|
||||
VideoStreamInputState input_state)
|
||||
: validation_id_(validation_id),
|
||||
status_(Status::kValid),
|
||||
step_(std::move(step)),
|
||||
min_pixel_limit_reached_(false),
|
||||
input_state_(input_state) {}
|
||||
|
||||
Adaptation::Adaptation(int validation_id,
|
||||
Step step,
|
||||
VideoSourceRestrictions restrictions,
|
||||
VideoAdaptationCounters counters,
|
||||
VideoStreamInputState input_state,
|
||||
bool min_pixel_limit_reached)
|
||||
: validation_id_(validation_id),
|
||||
status_(Status::kValid),
|
||||
step_(std::move(step)),
|
||||
min_pixel_limit_reached_(min_pixel_limit_reached),
|
||||
input_state_(input_state) {}
|
||||
|
||||
Adaptation::Adaptation(int validation_id,
|
||||
Status invalid_status,
|
||||
VideoStreamInputState input_state)
|
||||
: validation_id_(validation_id),
|
||||
status_(invalid_status),
|
||||
step_(absl::nullopt),
|
||||
min_pixel_limit_reached_(false),
|
||||
input_state_(input_state) {
|
||||
RTC_DCHECK_NE(status_, Status::kValid);
|
||||
}
|
||||
input_state_(std::move(input_state)),
|
||||
restrictions_(std::move(restrictions)),
|
||||
counters_(std::move(counters)) {}
|
||||
|
||||
Adaptation::Adaptation(int validation_id,
|
||||
Status invalid_status,
|
||||
@ -151,9 +172,8 @@ Adaptation::Adaptation(int validation_id,
|
||||
bool min_pixel_limit_reached)
|
||||
: validation_id_(validation_id),
|
||||
status_(invalid_status),
|
||||
step_(absl::nullopt),
|
||||
min_pixel_limit_reached_(min_pixel_limit_reached),
|
||||
input_state_(input_state) {
|
||||
input_state_(std::move(input_state)) {
|
||||
RTC_DCHECK_NE(status_, Status::kValid);
|
||||
}
|
||||
|
||||
@ -165,193 +185,25 @@ bool Adaptation::min_pixel_limit_reached() const {
|
||||
return min_pixel_limit_reached_;
|
||||
}
|
||||
|
||||
const Adaptation::Step& Adaptation::step() const {
|
||||
RTC_DCHECK_EQ(status_, Status::kValid);
|
||||
return step_.value();
|
||||
}
|
||||
const VideoStreamInputState& Adaptation::input_state() const {
|
||||
return input_state_;
|
||||
}
|
||||
|
||||
// VideoSourceRestrictor is responsible for keeping track of current
|
||||
// VideoSourceRestrictions.
|
||||
class VideoStreamAdapter::VideoSourceRestrictor {
|
||||
public:
|
||||
VideoSourceRestrictor() {}
|
||||
const VideoSourceRestrictions& Adaptation::restrictions() const {
|
||||
return restrictions_;
|
||||
}
|
||||
|
||||
VideoSourceRestrictions source_restrictions() const {
|
||||
return source_restrictions_;
|
||||
}
|
||||
const VideoAdaptationCounters& adaptation_counters() const {
|
||||
return adaptations_;
|
||||
}
|
||||
void ClearRestrictions() {
|
||||
source_restrictions_ = VideoSourceRestrictions();
|
||||
adaptations_ = VideoAdaptationCounters();
|
||||
}
|
||||
|
||||
void ForceRestrictions(const VideoSourceRestrictions& restrictions,
|
||||
const VideoAdaptationCounters& counters) {
|
||||
source_restrictions_ = restrictions;
|
||||
adaptations_ = counters;
|
||||
}
|
||||
|
||||
void set_min_pixels_per_frame(int min_pixels_per_frame) {
|
||||
min_pixels_per_frame_ = min_pixels_per_frame;
|
||||
}
|
||||
|
||||
int min_pixels_per_frame() const { return min_pixels_per_frame_; }
|
||||
|
||||
bool CanDecreaseResolutionTo(int target_pixels) const {
|
||||
int max_pixels_per_frame = rtc::dchecked_cast<int>(
|
||||
source_restrictions_.max_pixels_per_frame().value_or(
|
||||
std::numeric_limits<int>::max()));
|
||||
return target_pixels < max_pixels_per_frame &&
|
||||
target_pixels >= min_pixels_per_frame_;
|
||||
}
|
||||
|
||||
bool CanIncreaseResolutionTo(int target_pixels) const {
|
||||
int max_pixels_wanted = GetIncreasedMaxPixelsWanted(target_pixels);
|
||||
int max_pixels_per_frame = rtc::dchecked_cast<int>(
|
||||
source_restrictions_.max_pixels_per_frame().value_or(
|
||||
std::numeric_limits<int>::max()));
|
||||
return max_pixels_wanted > max_pixels_per_frame;
|
||||
}
|
||||
|
||||
bool CanDecreaseFrameRateTo(int max_frame_rate) const {
|
||||
const int fps_wanted = std::max(kMinFrameRateFps, max_frame_rate);
|
||||
return fps_wanted < rtc::dchecked_cast<int>(
|
||||
source_restrictions_.max_frame_rate().value_or(
|
||||
std::numeric_limits<int>::max()));
|
||||
}
|
||||
|
||||
bool CanIncreaseFrameRateTo(int max_frame_rate) const {
|
||||
return max_frame_rate > rtc::dchecked_cast<int>(
|
||||
source_restrictions_.max_frame_rate().value_or(
|
||||
std::numeric_limits<int>::max()));
|
||||
}
|
||||
|
||||
void ApplyAdaptationStep(const Adaptation::Step& step,
|
||||
DegradationPreference degradation_preference) {
|
||||
switch (step.type) {
|
||||
case Adaptation::StepType::kIncreaseResolution:
|
||||
RTC_DCHECK(step.target);
|
||||
IncreaseResolutionTo(step.target.value());
|
||||
break;
|
||||
case Adaptation::StepType::kDecreaseResolution:
|
||||
RTC_DCHECK(step.target);
|
||||
DecreaseResolutionTo(step.target.value());
|
||||
break;
|
||||
case Adaptation::StepType::kIncreaseFrameRate:
|
||||
RTC_DCHECK(step.target);
|
||||
IncreaseFrameRateTo(step.target.value());
|
||||
// TODO(https://crbug.com/webrtc/11222): Don't adapt in two steps.
|
||||
// GetAdaptationUp() should tell us the correct value, but BALANCED
|
||||
// logic in DecrementFramerate() makes it hard to predict whether this
|
||||
// will be the last step. Remove the dependency on
|
||||
// adaptation_counters().
|
||||
if (degradation_preference == DegradationPreference::BALANCED &&
|
||||
adaptation_counters().fps_adaptations == 0 &&
|
||||
step.target != std::numeric_limits<int>::max()) {
|
||||
RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting.";
|
||||
IncreaseFrameRateTo(std::numeric_limits<int>::max());
|
||||
}
|
||||
break;
|
||||
case Adaptation::StepType::kDecreaseFrameRate:
|
||||
RTC_DCHECK(step.target);
|
||||
DecreaseFrameRateTo(step.target.value());
|
||||
break;
|
||||
case Adaptation::StepType::kForce:
|
||||
RTC_DCHECK(step.restrictions);
|
||||
RTC_DCHECK(step.counters);
|
||||
ForceRestrictions(step.restrictions.value(), step.counters.value());
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
static int GetIncreasedMaxPixelsWanted(int target_pixels) {
|
||||
if (target_pixels == std::numeric_limits<int>::max())
|
||||
return std::numeric_limits<int>::max();
|
||||
// When we decrease resolution, we go down to at most 3/5 of current pixels.
|
||||
// Thus to increase resolution, we need 3/5 to get back to where we started.
|
||||
// When going up, the desired max_pixels_per_frame() has to be significantly
|
||||
// higher than the target because the source's native resolutions might not
|
||||
// match the target. We pick 12/5 of the target.
|
||||
//
|
||||
// (This value was historically 4 times the old target, which is (3/5)*4 of
|
||||
// the new target - or 12/5 - assuming the target is adjusted according to
|
||||
// the above steps.)
|
||||
RTC_DCHECK(target_pixels != std::numeric_limits<int>::max());
|
||||
return (target_pixels * 12) / 5;
|
||||
}
|
||||
|
||||
void DecreaseResolutionTo(int target_pixels) {
|
||||
RTC_DCHECK(CanDecreaseResolutionTo(target_pixels));
|
||||
RTC_LOG(LS_INFO) << "Scaling down resolution, max pixels: "
|
||||
<< target_pixels;
|
||||
source_restrictions_.set_max_pixels_per_frame(
|
||||
target_pixels != std::numeric_limits<int>::max()
|
||||
? absl::optional<size_t>(target_pixels)
|
||||
: absl::nullopt);
|
||||
source_restrictions_.set_target_pixels_per_frame(absl::nullopt);
|
||||
++adaptations_.resolution_adaptations;
|
||||
}
|
||||
|
||||
void IncreaseResolutionTo(int target_pixels) {
|
||||
RTC_DCHECK(CanIncreaseResolutionTo(target_pixels));
|
||||
int max_pixels_wanted = GetIncreasedMaxPixelsWanted(target_pixels);
|
||||
RTC_LOG(LS_INFO) << "Scaling up resolution, max pixels: "
|
||||
<< max_pixels_wanted;
|
||||
source_restrictions_.set_max_pixels_per_frame(
|
||||
max_pixels_wanted != std::numeric_limits<int>::max()
|
||||
? absl::optional<size_t>(max_pixels_wanted)
|
||||
: absl::nullopt);
|
||||
source_restrictions_.set_target_pixels_per_frame(
|
||||
max_pixels_wanted != std::numeric_limits<int>::max()
|
||||
? absl::optional<size_t>(target_pixels)
|
||||
: absl::nullopt);
|
||||
--adaptations_.resolution_adaptations;
|
||||
RTC_DCHECK_GE(adaptations_.resolution_adaptations, 0);
|
||||
}
|
||||
|
||||
void DecreaseFrameRateTo(int max_frame_rate) {
|
||||
RTC_DCHECK(CanDecreaseFrameRateTo(max_frame_rate));
|
||||
max_frame_rate = std::max(kMinFrameRateFps, max_frame_rate);
|
||||
RTC_LOG(LS_INFO) << "Scaling down framerate: " << max_frame_rate;
|
||||
source_restrictions_.set_max_frame_rate(
|
||||
max_frame_rate != std::numeric_limits<int>::max()
|
||||
? absl::optional<double>(max_frame_rate)
|
||||
: absl::nullopt);
|
||||
++adaptations_.fps_adaptations;
|
||||
}
|
||||
|
||||
void IncreaseFrameRateTo(int max_frame_rate) {
|
||||
RTC_DCHECK(CanIncreaseFrameRateTo(max_frame_rate));
|
||||
RTC_LOG(LS_INFO) << "Scaling up framerate: " << max_frame_rate;
|
||||
source_restrictions_.set_max_frame_rate(
|
||||
max_frame_rate != std::numeric_limits<int>::max()
|
||||
? absl::optional<double>(max_frame_rate)
|
||||
: absl::nullopt);
|
||||
--adaptations_.fps_adaptations;
|
||||
RTC_DCHECK_GE(adaptations_.fps_adaptations, 0);
|
||||
}
|
||||
|
||||
// Needed by CanDecreaseResolutionTo().
|
||||
int min_pixels_per_frame_ = 0;
|
||||
// Current State.
|
||||
VideoSourceRestrictions source_restrictions_;
|
||||
VideoAdaptationCounters adaptations_;
|
||||
};
|
||||
const VideoAdaptationCounters& Adaptation::counters() const {
|
||||
return counters_;
|
||||
}
|
||||
|
||||
VideoStreamAdapter::VideoStreamAdapter(
|
||||
VideoStreamInputStateProvider* input_state_provider)
|
||||
: source_restrictor_(std::make_unique<VideoSourceRestrictor>()),
|
||||
input_state_provider_(input_state_provider),
|
||||
: input_state_provider_(input_state_provider),
|
||||
balanced_settings_(),
|
||||
adaptation_validation_id_(0),
|
||||
degradation_preference_(DegradationPreference::DISABLED),
|
||||
last_adaptation_request_(absl::nullopt),
|
||||
awaiting_frame_size_change_(absl::nullopt),
|
||||
last_video_source_restrictions_() {
|
||||
sequence_checker_.Detach();
|
||||
}
|
||||
@ -360,12 +212,12 @@ VideoStreamAdapter::~VideoStreamAdapter() {}
|
||||
|
||||
VideoSourceRestrictions VideoStreamAdapter::source_restrictions() const {
|
||||
RTC_DCHECK_RUN_ON(&sequence_checker_);
|
||||
return source_restrictor_->source_restrictions();
|
||||
return current_restrictions_.restrictions;
|
||||
}
|
||||
|
||||
const VideoAdaptationCounters& VideoStreamAdapter::adaptation_counters() const {
|
||||
RTC_DCHECK_RUN_ON(&sequence_checker_);
|
||||
return source_restrictor_->adaptation_counters();
|
||||
return current_restrictions_.counters;
|
||||
}
|
||||
|
||||
void VideoStreamAdapter::ClearRestrictions() {
|
||||
@ -373,8 +225,9 @@ void VideoStreamAdapter::ClearRestrictions() {
|
||||
// Invalidate any previously returned Adaptation.
|
||||
RTC_LOG(INFO) << "Resetting restrictions";
|
||||
++adaptation_validation_id_;
|
||||
source_restrictor_->ClearRestrictions();
|
||||
last_adaptation_request_.reset();
|
||||
current_restrictions_ = {VideoSourceRestrictions(),
|
||||
VideoAdaptationCounters()};
|
||||
awaiting_frame_size_change_ = absl::nullopt;
|
||||
BroadcastVideoRestrictionsUpdate(nullptr);
|
||||
}
|
||||
|
||||
@ -415,88 +268,88 @@ void VideoStreamAdapter::SetDegradationPreference(
|
||||
}
|
||||
}
|
||||
|
||||
struct VideoStreamAdapter::RestrictionsOrStateVisitor {
|
||||
Adaptation operator()(const RestrictionsWithCounters& r) const {
|
||||
return Adaptation(adaptation_validation_id, r.restrictions, r.counters,
|
||||
input_state, min_pixel_limit_reached());
|
||||
}
|
||||
Adaptation operator()(const Adaptation::Status& status) const {
|
||||
RTC_DCHECK_NE(status, Adaptation::Status::kValid);
|
||||
return Adaptation(adaptation_validation_id, status, input_state,
|
||||
min_pixel_limit_reached());
|
||||
}
|
||||
bool min_pixel_limit_reached() const {
|
||||
return input_state.frame_size_pixels().has_value() &&
|
||||
GetLowerResolutionThan(input_state.frame_size_pixels().value()) <
|
||||
input_state.min_pixels_per_frame();
|
||||
}
|
||||
|
||||
const int adaptation_validation_id;
|
||||
const VideoStreamInputState& input_state;
|
||||
};
|
||||
|
||||
Adaptation VideoStreamAdapter::RestrictionsOrStateToAdaptation(
|
||||
VideoStreamAdapter::RestrictionsOrState step_or_state,
|
||||
const VideoStreamInputState& input_state) const {
|
||||
RTC_DCHECK(!step_or_state.valueless_by_exception());
|
||||
return absl::visit(
|
||||
RestrictionsOrStateVisitor{adaptation_validation_id_, input_state},
|
||||
step_or_state);
|
||||
}
|
||||
|
||||
Adaptation VideoStreamAdapter::GetAdaptationUp(
|
||||
const VideoStreamInputState& input_state) const {
|
||||
return RestrictionsOrStateToAdaptation(GetAdaptationUpStep(input_state),
|
||||
input_state);
|
||||
}
|
||||
|
||||
Adaptation VideoStreamAdapter::GetAdaptationUp() {
|
||||
RTC_DCHECK_RUN_ON(&sequence_checker_);
|
||||
RTC_DCHECK_NE(degradation_preference_, DegradationPreference::DISABLED);
|
||||
VideoStreamInputState input_state = input_state_provider_->InputState();
|
||||
++adaptation_validation_id_;
|
||||
Adaptation adaptation = GetAdaptationUp(input_state);
|
||||
return adaptation;
|
||||
}
|
||||
|
||||
VideoStreamAdapter::RestrictionsOrState VideoStreamAdapter::GetAdaptationUpStep(
|
||||
const VideoStreamInputState& input_state) const {
|
||||
if (!HasSufficientInputForAdaptation(input_state)) {
|
||||
return Adaptation(adaptation_validation_id_,
|
||||
Adaptation::Status::kInsufficientInput, input_state);
|
||||
return Adaptation::Status::kInsufficientInput;
|
||||
}
|
||||
source_restrictor_->set_min_pixels_per_frame(
|
||||
input_state.min_pixels_per_frame());
|
||||
// Don't adapt if we're awaiting a previous adaptation to have an effect.
|
||||
bool last_request_increased_resolution =
|
||||
last_adaptation_request_ && last_adaptation_request_->step_type_ ==
|
||||
Adaptation::StepType::kIncreaseResolution;
|
||||
if (last_request_increased_resolution &&
|
||||
if (awaiting_frame_size_change_ &&
|
||||
awaiting_frame_size_change_->pixels_increased &&
|
||||
degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE &&
|
||||
input_state.frame_size_pixels().value() <=
|
||||
last_adaptation_request_->input_pixel_count_) {
|
||||
return Adaptation(adaptation_validation_id_,
|
||||
Adaptation::Status::kAwaitingPreviousAdaptation,
|
||||
input_state);
|
||||
awaiting_frame_size_change_->frame_size_pixels) {
|
||||
return Adaptation::Status::kAwaitingPreviousAdaptation;
|
||||
}
|
||||
|
||||
// Maybe propose targets based on degradation preference.
|
||||
switch (degradation_preference_) {
|
||||
case DegradationPreference::BALANCED: {
|
||||
// Attempt to increase target frame rate.
|
||||
int target_fps =
|
||||
balanced_settings_.MaxFps(input_state.video_codec_type(),
|
||||
input_state.frame_size_pixels().value());
|
||||
if (source_restrictor_->CanIncreaseFrameRateTo(target_fps)) {
|
||||
return Adaptation(
|
||||
adaptation_validation_id_,
|
||||
Adaptation::Step(Adaptation::StepType::kIncreaseFrameRate,
|
||||
target_fps),
|
||||
input_state);
|
||||
RestrictionsOrState increase_frame_rate =
|
||||
IncreaseFramerate(input_state, current_restrictions_);
|
||||
if (absl::holds_alternative<RestrictionsWithCounters>(
|
||||
increase_frame_rate)) {
|
||||
return increase_frame_rate;
|
||||
}
|
||||
// Scale up resolution.
|
||||
// else, increase resolution.
|
||||
ABSL_FALLTHROUGH_INTENDED;
|
||||
}
|
||||
case DegradationPreference::MAINTAIN_FRAMERATE: {
|
||||
// Attempt to increase pixel count.
|
||||
int target_pixels = input_state.frame_size_pixels().value();
|
||||
if (source_restrictor_->adaptation_counters().resolution_adaptations ==
|
||||
1) {
|
||||
RTC_LOG(LS_INFO) << "Removing resolution down-scaling setting.";
|
||||
target_pixels = std::numeric_limits<int>::max();
|
||||
}
|
||||
target_pixels = GetHigherResolutionThan(target_pixels);
|
||||
if (!source_restrictor_->CanIncreaseResolutionTo(target_pixels)) {
|
||||
return Adaptation(adaptation_validation_id_,
|
||||
Adaptation::Status::kLimitReached, input_state);
|
||||
}
|
||||
return Adaptation(
|
||||
adaptation_validation_id_,
|
||||
Adaptation::Step(Adaptation::StepType::kIncreaseResolution,
|
||||
target_pixels),
|
||||
input_state);
|
||||
return IncreaseResolution(input_state, current_restrictions_);
|
||||
}
|
||||
case DegradationPreference::MAINTAIN_RESOLUTION: {
|
||||
// Scale up framerate.
|
||||
int target_fps = input_state.frames_per_second();
|
||||
if (source_restrictor_->adaptation_counters().fps_adaptations == 1) {
|
||||
RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting.";
|
||||
target_fps = std::numeric_limits<int>::max();
|
||||
}
|
||||
target_fps = GetHigherFrameRateThan(target_fps);
|
||||
if (!source_restrictor_->CanIncreaseFrameRateTo(target_fps)) {
|
||||
return Adaptation(adaptation_validation_id_,
|
||||
Adaptation::Status::kLimitReached, input_state);
|
||||
}
|
||||
return Adaptation(
|
||||
adaptation_validation_id_,
|
||||
Adaptation::Step(Adaptation::StepType::kIncreaseFrameRate,
|
||||
target_fps),
|
||||
input_state);
|
||||
return IncreaseFramerate(input_state, current_restrictions_);
|
||||
}
|
||||
case DegradationPreference::DISABLED:
|
||||
RTC_NOTREACHED();
|
||||
return Adaptation(adaptation_validation_id_,
|
||||
Adaptation::Status::kLimitReached, input_state);
|
||||
return Adaptation::Status::kLimitReached;
|
||||
}
|
||||
}
|
||||
|
||||
@ -505,93 +358,167 @@ Adaptation VideoStreamAdapter::GetAdaptationDown() {
|
||||
RTC_DCHECK_NE(degradation_preference_, DegradationPreference::DISABLED);
|
||||
VideoStreamInputState input_state = input_state_provider_->InputState();
|
||||
++adaptation_validation_id_;
|
||||
return RestrictionsOrStateToAdaptation(GetAdaptationDownStep(input_state),
|
||||
input_state);
|
||||
}
|
||||
|
||||
VideoStreamAdapter::RestrictionsOrState
|
||||
VideoStreamAdapter::GetAdaptationDownStep(
|
||||
const VideoStreamInputState& input_state) const {
|
||||
if (!HasSufficientInputForAdaptation(input_state)) {
|
||||
return Adaptation(adaptation_validation_id_,
|
||||
Adaptation::Status::kInsufficientInput, input_state);
|
||||
return Adaptation::Status::kInsufficientInput;
|
||||
}
|
||||
source_restrictor_->set_min_pixels_per_frame(
|
||||
input_state.min_pixels_per_frame());
|
||||
// Don't adapt if we're awaiting a previous adaptation to have an effect or
|
||||
// if we switched degradation preference.
|
||||
bool last_request_decreased_resolution =
|
||||
last_adaptation_request_ && last_adaptation_request_->step_type_ ==
|
||||
Adaptation::StepType::kDecreaseResolution;
|
||||
if (last_request_decreased_resolution &&
|
||||
if (awaiting_frame_size_change_ &&
|
||||
!awaiting_frame_size_change_->pixels_increased &&
|
||||
degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE &&
|
||||
input_state.frame_size_pixels().value() >=
|
||||
last_adaptation_request_->input_pixel_count_) {
|
||||
return Adaptation(adaptation_validation_id_,
|
||||
Adaptation::Status::kAwaitingPreviousAdaptation,
|
||||
input_state);
|
||||
awaiting_frame_size_change_->frame_size_pixels) {
|
||||
return Adaptation::Status::kAwaitingPreviousAdaptation;
|
||||
}
|
||||
|
||||
// Maybe propose targets based on degradation preference.
|
||||
switch (degradation_preference_) {
|
||||
case DegradationPreference::BALANCED: {
|
||||
// Try scale down framerate, if lower.
|
||||
int target_fps =
|
||||
balanced_settings_.MinFps(input_state.video_codec_type(),
|
||||
input_state.frame_size_pixels().value());
|
||||
if (source_restrictor_->CanDecreaseFrameRateTo(target_fps)) {
|
||||
return Adaptation(
|
||||
adaptation_validation_id_,
|
||||
Adaptation::Step(Adaptation::StepType::kDecreaseFrameRate,
|
||||
target_fps),
|
||||
input_state);
|
||||
RestrictionsOrState decrease_frame_rate =
|
||||
DecreaseFramerate(input_state, current_restrictions_);
|
||||
if (absl::holds_alternative<RestrictionsWithCounters>(
|
||||
decrease_frame_rate)) {
|
||||
return decrease_frame_rate;
|
||||
}
|
||||
// Scale down resolution.
|
||||
// else, decrease resolution.
|
||||
ABSL_FALLTHROUGH_INTENDED;
|
||||
}
|
||||
case DegradationPreference::MAINTAIN_FRAMERATE: {
|
||||
// Scale down resolution.
|
||||
int target_pixels =
|
||||
GetLowerResolutionThan(input_state.frame_size_pixels().value());
|
||||
bool min_pixel_limit_reached =
|
||||
target_pixels < source_restrictor_->min_pixels_per_frame();
|
||||
if (!source_restrictor_->CanDecreaseResolutionTo(target_pixels)) {
|
||||
return Adaptation(adaptation_validation_id_,
|
||||
Adaptation::Status::kLimitReached, input_state,
|
||||
min_pixel_limit_reached);
|
||||
}
|
||||
return Adaptation(
|
||||
adaptation_validation_id_,
|
||||
Adaptation::Step(Adaptation::StepType::kDecreaseResolution,
|
||||
target_pixels),
|
||||
input_state, min_pixel_limit_reached);
|
||||
return DecreaseResolution(input_state, current_restrictions_);
|
||||
}
|
||||
case DegradationPreference::MAINTAIN_RESOLUTION: {
|
||||
int target_fps = GetLowerFrameRateThan(input_state.frames_per_second());
|
||||
if (!source_restrictor_->CanDecreaseFrameRateTo(target_fps)) {
|
||||
return Adaptation(adaptation_validation_id_,
|
||||
Adaptation::Status::kLimitReached, input_state);
|
||||
}
|
||||
return Adaptation(
|
||||
adaptation_validation_id_,
|
||||
Adaptation::Step(Adaptation::StepType::kDecreaseFrameRate,
|
||||
target_fps),
|
||||
input_state);
|
||||
return DecreaseFramerate(input_state, current_restrictions_);
|
||||
}
|
||||
case DegradationPreference::DISABLED:
|
||||
RTC_NOTREACHED();
|
||||
return Adaptation(adaptation_validation_id_,
|
||||
Adaptation::Status::kLimitReached, input_state);
|
||||
return Adaptation::Status::kLimitReached;
|
||||
}
|
||||
}
|
||||
|
||||
VideoStreamAdapter::RestrictionsWithCounters
|
||||
VideoStreamAdapter::PeekNextRestrictions(const Adaptation& adaptation) const {
|
||||
RTC_DCHECK_RUN_ON(&sequence_checker_);
|
||||
RTC_DCHECK_EQ(adaptation.validation_id_, adaptation_validation_id_);
|
||||
RTC_LOG(LS_INFO) << "PeekNextRestrictions called";
|
||||
if (adaptation.status() != Adaptation::Status::kValid) {
|
||||
return {source_restrictor_->source_restrictions(),
|
||||
source_restrictor_->adaptation_counters()};
|
||||
VideoStreamAdapter::RestrictionsOrState VideoStreamAdapter::DecreaseResolution(
|
||||
const VideoStreamInputState& input_state,
|
||||
const RestrictionsWithCounters& current_restrictions) {
|
||||
int target_pixels =
|
||||
GetLowerResolutionThan(input_state.frame_size_pixels().value());
|
||||
if (!CanDecreaseResolutionTo(target_pixels, input_state,
|
||||
current_restrictions.restrictions)) {
|
||||
return Adaptation::Status::kLimitReached;
|
||||
}
|
||||
VideoSourceRestrictor restrictor_copy = *source_restrictor_;
|
||||
restrictor_copy.ApplyAdaptationStep(adaptation.step(),
|
||||
degradation_preference_);
|
||||
return {restrictor_copy.source_restrictions(),
|
||||
restrictor_copy.adaptation_counters()};
|
||||
RestrictionsWithCounters new_restrictions = current_restrictions;
|
||||
RTC_LOG(LS_INFO) << "Scaling down resolution, max pixels: " << target_pixels;
|
||||
new_restrictions.restrictions.set_max_pixels_per_frame(
|
||||
target_pixels != std::numeric_limits<int>::max()
|
||||
? absl::optional<size_t>(target_pixels)
|
||||
: absl::nullopt);
|
||||
new_restrictions.restrictions.set_target_pixels_per_frame(absl::nullopt);
|
||||
++new_restrictions.counters.resolution_adaptations;
|
||||
return new_restrictions;
|
||||
}
|
||||
|
||||
VideoStreamAdapter::RestrictionsOrState VideoStreamAdapter::DecreaseFramerate(
|
||||
const VideoStreamInputState& input_state,
|
||||
const RestrictionsWithCounters& current_restrictions) const {
|
||||
int max_frame_rate;
|
||||
if (degradation_preference_ == DegradationPreference::MAINTAIN_RESOLUTION) {
|
||||
max_frame_rate = GetLowerFrameRateThan(input_state.frames_per_second());
|
||||
} else if (degradation_preference_ == DegradationPreference::BALANCED) {
|
||||
max_frame_rate =
|
||||
balanced_settings_.MinFps(input_state.video_codec_type(),
|
||||
input_state.frame_size_pixels().value());
|
||||
} else {
|
||||
RTC_NOTREACHED();
|
||||
max_frame_rate = GetLowerFrameRateThan(input_state.frames_per_second());
|
||||
}
|
||||
if (!CanDecreaseFrameRateTo(max_frame_rate,
|
||||
current_restrictions.restrictions)) {
|
||||
return Adaptation::Status::kLimitReached;
|
||||
}
|
||||
RestrictionsWithCounters new_restrictions = current_restrictions;
|
||||
max_frame_rate = std::max(kMinFrameRateFps, max_frame_rate);
|
||||
RTC_LOG(LS_INFO) << "Scaling down framerate: " << max_frame_rate;
|
||||
new_restrictions.restrictions.set_max_frame_rate(
|
||||
max_frame_rate != std::numeric_limits<int>::max()
|
||||
? absl::optional<double>(max_frame_rate)
|
||||
: absl::nullopt);
|
||||
++new_restrictions.counters.fps_adaptations;
|
||||
return new_restrictions;
|
||||
}
|
||||
|
||||
VideoStreamAdapter::RestrictionsOrState VideoStreamAdapter::IncreaseResolution(
|
||||
const VideoStreamInputState& input_state,
|
||||
const RestrictionsWithCounters& current_restrictions) {
|
||||
int target_pixels = input_state.frame_size_pixels().value();
|
||||
if (current_restrictions.counters.resolution_adaptations == 1) {
|
||||
RTC_LOG(LS_INFO) << "Removing resolution down-scaling setting.";
|
||||
target_pixels = std::numeric_limits<int>::max();
|
||||
}
|
||||
target_pixels = GetHigherResolutionThan(target_pixels);
|
||||
if (!CanIncreaseResolutionTo(target_pixels,
|
||||
current_restrictions.restrictions)) {
|
||||
return Adaptation::Status::kLimitReached;
|
||||
}
|
||||
int max_pixels_wanted = GetIncreasedMaxPixelsWanted(target_pixels);
|
||||
RestrictionsWithCounters new_restrictions = current_restrictions;
|
||||
RTC_LOG(LS_INFO) << "Scaling up resolution, max pixels: "
|
||||
<< max_pixels_wanted;
|
||||
new_restrictions.restrictions.set_max_pixels_per_frame(
|
||||
max_pixels_wanted != std::numeric_limits<int>::max()
|
||||
? absl::optional<size_t>(max_pixels_wanted)
|
||||
: absl::nullopt);
|
||||
new_restrictions.restrictions.set_target_pixels_per_frame(
|
||||
max_pixels_wanted != std::numeric_limits<int>::max()
|
||||
? absl::optional<size_t>(target_pixels)
|
||||
: absl::nullopt);
|
||||
--new_restrictions.counters.resolution_adaptations;
|
||||
RTC_DCHECK_GE(new_restrictions.counters.resolution_adaptations, 0);
|
||||
return new_restrictions;
|
||||
}
|
||||
|
||||
VideoStreamAdapter::RestrictionsOrState VideoStreamAdapter::IncreaseFramerate(
|
||||
const VideoStreamInputState& input_state,
|
||||
const RestrictionsWithCounters& current_restrictions) const {
|
||||
int max_frame_rate;
|
||||
if (degradation_preference_ == DegradationPreference::MAINTAIN_RESOLUTION) {
|
||||
max_frame_rate = GetHigherFrameRateThan(input_state.frames_per_second());
|
||||
} else if (degradation_preference_ == DegradationPreference::BALANCED) {
|
||||
max_frame_rate =
|
||||
balanced_settings_.MaxFps(input_state.video_codec_type(),
|
||||
input_state.frame_size_pixels().value());
|
||||
// In BALANCED, the max_frame_rate must be checked before proceeding. This
|
||||
// is because the MaxFps might be the current Fps and so the balanced
|
||||
// settings may want to scale up the resolution.=
|
||||
if (!CanIncreaseFrameRateTo(max_frame_rate,
|
||||
current_restrictions.restrictions)) {
|
||||
return Adaptation::Status::kLimitReached;
|
||||
}
|
||||
} else {
|
||||
RTC_NOTREACHED();
|
||||
max_frame_rate = GetHigherFrameRateThan(input_state.frames_per_second());
|
||||
}
|
||||
if (current_restrictions.counters.fps_adaptations == 1) {
|
||||
RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting.";
|
||||
max_frame_rate = std::numeric_limits<int>::max();
|
||||
}
|
||||
if (!CanIncreaseFrameRateTo(max_frame_rate,
|
||||
current_restrictions.restrictions)) {
|
||||
return Adaptation::Status::kLimitReached;
|
||||
}
|
||||
RTC_LOG(LS_INFO) << "Scaling up framerate: " << max_frame_rate;
|
||||
RestrictionsWithCounters new_restrictions = current_restrictions;
|
||||
new_restrictions.restrictions.set_max_frame_rate(
|
||||
max_frame_rate != std::numeric_limits<int>::max()
|
||||
? absl::optional<double>(max_frame_rate)
|
||||
: absl::nullopt);
|
||||
--new_restrictions.counters.fps_adaptations;
|
||||
RTC_DCHECK_GE(new_restrictions.counters.fps_adaptations, 0);
|
||||
return new_restrictions;
|
||||
}
|
||||
|
||||
void VideoStreamAdapter::ApplyAdaptation(
|
||||
@ -599,17 +526,22 @@ void VideoStreamAdapter::ApplyAdaptation(
|
||||
rtc::scoped_refptr<Resource> resource) {
|
||||
RTC_DCHECK_RUN_ON(&sequence_checker_);
|
||||
RTC_DCHECK_EQ(adaptation.validation_id_, adaptation_validation_id_);
|
||||
RTC_LOG(LS_INFO) << "ApplyAdaptation called";
|
||||
if (adaptation.status() != Adaptation::Status::kValid)
|
||||
return;
|
||||
// Remember the input pixels and fps of this adaptation. Used to avoid
|
||||
// adapting again before this adaptation has had an effect.
|
||||
last_adaptation_request_.emplace(AdaptationRequest{
|
||||
adaptation.input_state_.frame_size_pixels().value(),
|
||||
adaptation.input_state_.frames_per_second(), adaptation.step().type});
|
||||
// Adapt!
|
||||
source_restrictor_->ApplyAdaptationStep(adaptation.step(),
|
||||
degradation_preference_);
|
||||
if (DidIncreaseResolution(current_restrictions_.restrictions,
|
||||
adaptation.restrictions())) {
|
||||
awaiting_frame_size_change_.emplace(
|
||||
true, adaptation.input_state().frame_size_pixels().value());
|
||||
} else if (DidDecreaseResolution(current_restrictions_.restrictions,
|
||||
adaptation.restrictions())) {
|
||||
awaiting_frame_size_change_.emplace(
|
||||
false, adaptation.input_state().frame_size_pixels().value());
|
||||
} else {
|
||||
awaiting_frame_size_change_ = absl::nullopt;
|
||||
}
|
||||
current_restrictions_ = {adaptation.restrictions(), adaptation.counters()};
|
||||
BroadcastVideoRestrictionsUpdate(resource);
|
||||
}
|
||||
|
||||
@ -619,8 +551,8 @@ Adaptation VideoStreamAdapter::GetAdaptationTo(
|
||||
// Adapts up/down from the current levels so counters are equal.
|
||||
RTC_DCHECK_RUN_ON(&sequence_checker_);
|
||||
VideoStreamInputState input_state = input_state_provider_->InputState();
|
||||
return Adaptation(adaptation_validation_id_,
|
||||
Adaptation::Step(restrictions, counters), input_state);
|
||||
return Adaptation(adaptation_validation_id_, restrictions, counters,
|
||||
input_state, false);
|
||||
}
|
||||
|
||||
void VideoStreamAdapter::BroadcastVideoRestrictionsUpdate(
|
||||
@ -633,10 +565,10 @@ void VideoStreamAdapter::BroadcastVideoRestrictionsUpdate(
|
||||
}
|
||||
for (auto* restrictions_listener : restrictions_listeners_) {
|
||||
restrictions_listener->OnVideoSourceRestrictionsUpdated(
|
||||
filtered, source_restrictor_->adaptation_counters(), resource,
|
||||
filtered, current_restrictions_.counters, resource,
|
||||
source_restrictions());
|
||||
}
|
||||
last_video_source_restrictions_ = source_restrictor_->source_restrictions();
|
||||
last_video_source_restrictions_ = current_restrictions_.restrictions;
|
||||
last_filtered_restrictions_ = filtered;
|
||||
}
|
||||
|
||||
@ -648,4 +580,10 @@ bool VideoStreamAdapter::HasSufficientInputForAdaptation(
|
||||
input_state.frames_per_second() >= kMinFrameRateFps);
|
||||
}
|
||||
|
||||
VideoStreamAdapter::AwaitingFrameSizeChange::AwaitingFrameSizeChange(
|
||||
bool pixels_increased,
|
||||
int frame_size_pixels)
|
||||
: pixels_increased(pixels_increased),
|
||||
frame_size_pixels(frame_size_pixels) {}
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
@ -12,9 +12,11 @@
|
||||
#define CALL_ADAPTATION_VIDEO_STREAM_ADAPTER_H_
|
||||
|
||||
#include <memory>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/types/optional.h"
|
||||
#include "absl/types/variant.h"
|
||||
#include "api/adaptation/resource.h"
|
||||
#include "api/rtp_parameters.h"
|
||||
#include "api/video/video_adaptation_counters.h"
|
||||
@ -24,6 +26,7 @@
|
||||
#include "call/adaptation/video_stream_input_state_provider.h"
|
||||
#include "modules/video_coding/utility/quality_scaler.h"
|
||||
#include "rtc_base/experiments/balanced_degradation_settings.h"
|
||||
#include "rtc_base/thread_annotations.h"
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
@ -53,9 +56,9 @@ VideoSourceRestrictions FilterRestrictionsByDegradationPreference(
|
||||
|
||||
int GetHigherResolutionThan(int pixel_count);
|
||||
|
||||
// Represents one step that the VideoStreamAdapter can take when adapting the
|
||||
// VideoSourceRestrictions up or down. Or, if adaptation is not valid, provides
|
||||
// a Status code indicating the reason for not adapting.
|
||||
// Either represents the next VideoSourceRestrictions the VideoStreamAdapter
|
||||
// will take, or provides a Status code indicating the reason for not adapting
|
||||
// if the adaptation is not valid.
|
||||
class Adaptation final {
|
||||
public:
|
||||
enum class Status {
|
||||
@ -75,69 +78,39 @@ class Adaptation final {
|
||||
|
||||
static const char* StatusToString(Status status);
|
||||
|
||||
// The status of this Adaptation. To find out how this Adaptation affects
|
||||
// VideoSourceRestrictions, see VideoStreamAdapter::PeekNextRestrictions().
|
||||
Status status() const;
|
||||
const VideoStreamInputState& input_state() const;
|
||||
const VideoSourceRestrictions& restrictions() const;
|
||||
const VideoAdaptationCounters& counters() const;
|
||||
// Used for stats reporting.
|
||||
bool min_pixel_limit_reached() const;
|
||||
|
||||
const VideoStreamInputState& input_state() const;
|
||||
|
||||
private:
|
||||
// The adapter needs to know about step type and step target in order to
|
||||
// construct and perform an Adaptation, which is a detail we do not want to
|
||||
// expose to the public interface.
|
||||
friend class VideoStreamAdapter;
|
||||
|
||||
enum class StepType {
|
||||
kIncreaseResolution,
|
||||
kDecreaseResolution,
|
||||
kIncreaseFrameRate,
|
||||
kDecreaseFrameRate,
|
||||
kForce
|
||||
};
|
||||
|
||||
struct Step {
|
||||
Step(StepType type, int target);
|
||||
// StepType is kForce
|
||||
Step(VideoSourceRestrictions restrictions,
|
||||
VideoAdaptationCounters counters);
|
||||
const StepType type;
|
||||
// Pixel or frame rate depending on |type|.
|
||||
// Only set when |type| is not kForce.
|
||||
const absl::optional<int> target;
|
||||
// Only set when |type| is kForce.
|
||||
const absl::optional<VideoSourceRestrictions> restrictions;
|
||||
// Only set when |type| is kForce.
|
||||
const absl::optional<VideoAdaptationCounters> counters;
|
||||
};
|
||||
|
||||
// Constructs with a valid adaptation Step. Status is kValid.
|
||||
Adaptation(int validation_id, Step step, VideoStreamInputState input_state);
|
||||
// Constructs with a valid adaptation. Status is kValid.
|
||||
Adaptation(int validation_id,
|
||||
Step step,
|
||||
VideoSourceRestrictions restrictions,
|
||||
VideoAdaptationCounters counters,
|
||||
VideoStreamInputState input_state,
|
||||
bool min_pixel_limit_reached);
|
||||
// Constructor when adaptation is not valid. Status MUST NOT be kValid.
|
||||
Adaptation(int validation_id,
|
||||
Status invalid_status,
|
||||
VideoStreamInputState input_state);
|
||||
Adaptation(int validation_id,
|
||||
Status invalid_status,
|
||||
VideoStreamInputState input_state,
|
||||
bool min_pixel_limit_reached);
|
||||
|
||||
const Step& step() const; // Only callable if |status_| is kValid.
|
||||
|
||||
// An Adaptation can become invalidated if the state of VideoStreamAdapter is
|
||||
// modified before the Adaptation is applied. To guard against this, this ID
|
||||
// has to match VideoStreamAdapter::adaptation_validation_id_ when applied.
|
||||
// TODO(https://crbug.com/webrtc/11700): Remove the validation_id_.
|
||||
const int validation_id_;
|
||||
const Status status_;
|
||||
const absl::optional<Step> step_; // Only present if |status_| is kValid.
|
||||
const bool min_pixel_limit_reached_;
|
||||
// Input state when adaptation was made.
|
||||
const VideoStreamInputState input_state_;
|
||||
const VideoSourceRestrictions restrictions_;
|
||||
const VideoAdaptationCounters counters_;
|
||||
};
|
||||
|
||||
// Owns the VideoSourceRestriction for a single stream and is responsible for
|
||||
@ -173,58 +146,60 @@ class VideoStreamAdapter {
|
||||
Adaptation GetAdaptationTo(const VideoAdaptationCounters& counters,
|
||||
const VideoSourceRestrictions& restrictions);
|
||||
|
||||
struct RestrictionsWithCounters {
|
||||
VideoSourceRestrictions restrictions;
|
||||
VideoAdaptationCounters adaptation_counters;
|
||||
|
||||
bool operator==(const RestrictionsWithCounters& other) {
|
||||
return restrictions == other.restrictions &&
|
||||
adaptation_counters == other.adaptation_counters;
|
||||
}
|
||||
|
||||
bool operator!=(const RestrictionsWithCounters& other) {
|
||||
return !(*this == other);
|
||||
}
|
||||
};
|
||||
|
||||
// Returns the restrictions that result from applying the adaptation, without
|
||||
// actually applying it. If the adaptation is not valid, current restrictions
|
||||
// are returned.
|
||||
RestrictionsWithCounters PeekNextRestrictions(
|
||||
const Adaptation& adaptation) const;
|
||||
// Updates source_restrictions() based according to the Adaptation. These
|
||||
// adaptations will be attributed to the Resource |resource| if the |resource|
|
||||
// is non-null. If |resource| is null the adaptation will be changed in
|
||||
// general, and thus could be adapted up in the future from other resources.
|
||||
// Updates source_restrictions() the Adaptation.
|
||||
void ApplyAdaptation(const Adaptation& adaptation,
|
||||
rtc::scoped_refptr<Resource> resource);
|
||||
|
||||
private:
|
||||
class VideoSourceRestrictor;
|
||||
struct RestrictionsWithCounters {
|
||||
VideoSourceRestrictions restrictions;
|
||||
VideoAdaptationCounters counters;
|
||||
};
|
||||
|
||||
private:
|
||||
void BroadcastVideoRestrictionsUpdate(
|
||||
const rtc::scoped_refptr<Resource>& resource);
|
||||
|
||||
bool HasSufficientInputForAdaptation(const VideoStreamInputState& input_state)
|
||||
const RTC_RUN_ON(&sequence_checker_);
|
||||
|
||||
// The input frame rate and resolution at the time of an adaptation in the
|
||||
// direction described by |mode_| (up or down).
|
||||
// TODO(https://crbug.com/webrtc/11393): Can this be renamed? Can this be
|
||||
// merged with AdaptationTarget?
|
||||
struct AdaptationRequest {
|
||||
// The pixel count produced by the source at the time of the adaptation.
|
||||
int input_pixel_count_;
|
||||
// Framerate received from the source at the time of the adaptation.
|
||||
int framerate_fps_;
|
||||
// Degradation preference for the request.
|
||||
Adaptation::StepType step_type_;
|
||||
};
|
||||
using RestrictionsOrState =
|
||||
absl::variant<RestrictionsWithCounters, Adaptation::Status>;
|
||||
RestrictionsOrState GetAdaptationUpStep(
|
||||
const VideoStreamInputState& input_state) const
|
||||
RTC_RUN_ON(&sequence_checker_);
|
||||
RestrictionsOrState GetAdaptationDownStep(
|
||||
const VideoStreamInputState& input_state) const
|
||||
RTC_RUN_ON(&sequence_checker_);
|
||||
|
||||
Adaptation GetAdaptationUp(const VideoStreamInputState& input_state) const
|
||||
RTC_RUN_ON(&sequence_checker_);
|
||||
Adaptation GetAdaptationDown(const VideoStreamInputState& input_state) const
|
||||
RTC_RUN_ON(&sequence_checker_);
|
||||
|
||||
static RestrictionsOrState DecreaseResolution(
|
||||
const VideoStreamInputState& input_state,
|
||||
const RestrictionsWithCounters& current_restrictions);
|
||||
static RestrictionsOrState IncreaseResolution(
|
||||
const VideoStreamInputState& input_state,
|
||||
const RestrictionsWithCounters& current_restrictions);
|
||||
// Framerate methods are member functions because they need internal state
|
||||
// if the degradation preference is BALANCED.
|
||||
RestrictionsOrState DecreaseFramerate(
|
||||
const VideoStreamInputState& input_state,
|
||||
const RestrictionsWithCounters& current_restrictions) const
|
||||
RTC_RUN_ON(&sequence_checker_);
|
||||
RestrictionsOrState IncreaseFramerate(
|
||||
const VideoStreamInputState& input_state,
|
||||
const RestrictionsWithCounters& current_restrictions) const
|
||||
RTC_RUN_ON(&sequence_checker_);
|
||||
|
||||
struct RestrictionsOrStateVisitor;
|
||||
Adaptation RestrictionsOrStateToAdaptation(
|
||||
RestrictionsOrState step_or_state,
|
||||
const VideoStreamInputState& input_state) const
|
||||
RTC_RUN_ON(&sequence_checker_);
|
||||
|
||||
SequenceChecker sequence_checker_ RTC_GUARDED_BY(&sequence_checker_);
|
||||
// Owner and modifier of the VideoSourceRestriction of this stream adaptor.
|
||||
const std::unique_ptr<VideoSourceRestrictor> source_restrictor_
|
||||
RTC_GUARDED_BY(&sequence_checker_);
|
||||
// Gets the input state which is the basis of all adaptations.
|
||||
// Thread safe.
|
||||
VideoStreamInputStateProvider* input_state_provider_;
|
||||
@ -238,14 +213,18 @@ class VideoStreamAdapter {
|
||||
// https://w3c.github.io/mst-content-hint/#dom-rtcdegradationpreference
|
||||
DegradationPreference degradation_preference_
|
||||
RTC_GUARDED_BY(&sequence_checker_);
|
||||
// The input frame rate, resolution and adaptation direction of the last
|
||||
// ApplyAdaptationTarget(). Used to avoid adapting twice if a recent
|
||||
// adaptation has not had an effect on the input frame rate or resolution yet.
|
||||
// Used to avoid adapting twice. Stores the resolution at the time of the last
|
||||
// adaptation.
|
||||
// TODO(hbos): Can we implement a more general "cooldown" mechanism of
|
||||
// resources intead? If we already have adapted it seems like we should wait
|
||||
// a while before adapting again, so that we are not acting on usage
|
||||
// measurements that are made obsolete/unreliable by an "ongoing" adaptation.
|
||||
absl::optional<AdaptationRequest> last_adaptation_request_
|
||||
struct AwaitingFrameSizeChange {
|
||||
AwaitingFrameSizeChange(bool pixels_increased, int frame_size);
|
||||
const bool pixels_increased;
|
||||
const int frame_size_pixels;
|
||||
};
|
||||
absl::optional<AwaitingFrameSizeChange> awaiting_frame_size_change_
|
||||
RTC_GUARDED_BY(&sequence_checker_);
|
||||
// The previous restrictions value. Starts as unrestricted.
|
||||
VideoSourceRestrictions last_video_source_restrictions_
|
||||
@ -255,6 +234,9 @@ class VideoStreamAdapter {
|
||||
|
||||
std::vector<VideoSourceRestrictionsListener*> restrictions_listeners_
|
||||
RTC_GUARDED_BY(&sequence_checker_);
|
||||
|
||||
RestrictionsWithCounters current_restrictions_
|
||||
RTC_GUARDED_BY(&sequence_checker_);
|
||||
};
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
@ -719,11 +719,9 @@ TEST_F(VideoStreamAdapterTest, RestrictionBroadcasted) {
|
||||
// Broadcast on ApplyAdaptation.
|
||||
{
|
||||
Adaptation adaptation = adapter_.GetAdaptationDown();
|
||||
VideoStreamAdapter::RestrictionsWithCounters peek =
|
||||
adapter_.PeekNextRestrictions(adaptation);
|
||||
fake_stream.ApplyAdaptation(adaptation);
|
||||
EXPECT_EQ(1, listener.calls());
|
||||
EXPECT_EQ(peek.restrictions, listener.last_restrictions());
|
||||
EXPECT_EQ(adaptation.restrictions(), listener.last_restrictions());
|
||||
}
|
||||
|
||||
// Broadcast on ClearRestrictions().
|
||||
@ -732,7 +730,7 @@ TEST_F(VideoStreamAdapterTest, RestrictionBroadcasted) {
|
||||
EXPECT_EQ(VideoSourceRestrictions(), listener.last_restrictions());
|
||||
}
|
||||
|
||||
TEST_F(VideoStreamAdapterTest, PeekNextRestrictions) {
|
||||
TEST_F(VideoStreamAdapterTest, AdaptationHasNextRestrcitions) {
|
||||
// Any non-disabled DegradationPreference will do.
|
||||
adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE);
|
||||
FakeVideoStream fake_stream(&adapter_, &input_state_provider_, 1280 * 720, 30,
|
||||
@ -741,49 +739,27 @@ TEST_F(VideoStreamAdapterTest, PeekNextRestrictions) {
|
||||
{
|
||||
Adaptation adaptation = adapter_.GetAdaptationUp();
|
||||
EXPECT_EQ(Adaptation::Status::kLimitReached, adaptation.status());
|
||||
VideoStreamAdapter::RestrictionsWithCounters restrictions_with_counters =
|
||||
adapter_.PeekNextRestrictions(adaptation);
|
||||
EXPECT_EQ(restrictions_with_counters.restrictions,
|
||||
adapter_.source_restrictions());
|
||||
EXPECT_EQ(0, restrictions_with_counters.adaptation_counters.Total());
|
||||
EXPECT_EQ(adaptation.restrictions(), adapter_.source_restrictions());
|
||||
EXPECT_EQ(0, adaptation.counters().Total());
|
||||
}
|
||||
// When we adapt down.
|
||||
{
|
||||
Adaptation adaptation = adapter_.GetAdaptationDown();
|
||||
EXPECT_EQ(Adaptation::Status::kValid, adaptation.status());
|
||||
VideoStreamAdapter::RestrictionsWithCounters restrictions_with_counters =
|
||||
adapter_.PeekNextRestrictions(adaptation);
|
||||
fake_stream.ApplyAdaptation(adaptation);
|
||||
EXPECT_EQ(restrictions_with_counters.restrictions,
|
||||
adapter_.source_restrictions());
|
||||
EXPECT_EQ(restrictions_with_counters.adaptation_counters,
|
||||
adapter_.adaptation_counters());
|
||||
EXPECT_EQ(adaptation.restrictions(), adapter_.source_restrictions());
|
||||
EXPECT_EQ(adaptation.counters(), adapter_.adaptation_counters());
|
||||
}
|
||||
// When we adapt up.
|
||||
{
|
||||
Adaptation adaptation = adapter_.GetAdaptationUp();
|
||||
EXPECT_EQ(Adaptation::Status::kValid, adaptation.status());
|
||||
VideoStreamAdapter::RestrictionsWithCounters restrictions_with_counters =
|
||||
adapter_.PeekNextRestrictions(adaptation);
|
||||
fake_stream.ApplyAdaptation(adaptation);
|
||||
EXPECT_EQ(restrictions_with_counters.restrictions,
|
||||
adapter_.source_restrictions());
|
||||
EXPECT_EQ(restrictions_with_counters.adaptation_counters,
|
||||
adapter_.adaptation_counters());
|
||||
EXPECT_EQ(adaptation.restrictions(), adapter_.source_restrictions());
|
||||
EXPECT_EQ(adaptation.counters(), adapter_.adaptation_counters());
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(VideoStreamAdapterTest, PeekRestrictionsDoesNotBroadcast) {
|
||||
FakeVideoStreamAdapterListner listener;
|
||||
adapter_.AddRestrictionsListener(&listener);
|
||||
adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE);
|
||||
FakeVideoStream fake_stream(&adapter_, &input_state_provider_, 1280 * 720, 30,
|
||||
kDefaultMinPixelsPerFrame);
|
||||
Adaptation adaptation = adapter_.GetAdaptationDown();
|
||||
adapter_.PeekNextRestrictions(adaptation);
|
||||
EXPECT_EQ(0, listener.calls());
|
||||
}
|
||||
|
||||
TEST_F(VideoStreamAdapterTest,
|
||||
SetDegradationPreferenceToOrFromBalancedClearsRestrictions) {
|
||||
adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE);
|
||||
@ -829,7 +805,7 @@ TEST(VideoStreamAdapterDeathTest, AdaptDownInvalidatesAdaptations) {
|
||||
input_state_provider.SetInputState(1280 * 720, 30, kDefaultMinPixelsPerFrame);
|
||||
Adaptation adaptation = adapter.GetAdaptationDown();
|
||||
adapter.GetAdaptationDown();
|
||||
EXPECT_DEATH(adapter.PeekNextRestrictions(adaptation), "");
|
||||
EXPECT_DEATH(adapter.ApplyAdaptation(adaptation, nullptr), "");
|
||||
}
|
||||
|
||||
#endif // RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID)
|
||||
|
||||
Reference in New Issue
Block a user