zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Overuse] Initial version of VideoStreamAdapter (Restrictor moved).

Browse Source 
This CL simply moves the VideoSourceRestrictor from being an inner class
of OveruseFrameDetectorResourceAdaptationModule to a new class,
VideoStreamAdapter.

In follow-up CLs, the responsibility of determining what the next step
for adapting up or down should also be moved to the VideoStreamAdapter.

The end-goal is that the VideoStreamAdapter takes care of "can adapt?"
and "do adapt!" type of logic so that a multi-stream aware adaptation
module can decide which stream (adapter) to adapt, and the adapter can
take care of the nitty gritty details of doing so.

In this CL the "can?"/"do!" part is realized but not the logic for
determining what the next step up or down is, and the class interface
needs improvement.

This CL also sets up the video/adaptation/ subdirectory and moves the
AdaptationCounters class here. Other adaptation-related classes (e.g.
the module and its resources) should move into this directory as well
in the future.

Bug: webrtc:11393
Change-Id: I2c12c1281eca854c62791abb65f0aca47a119726
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/169542
Reviewed-by: Evan Shrubsole <eshr@google.com>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Henrik Andreassson <henrika@webrtc.org>
Commit-Queue: Henrik Boström <hbos@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30705}
This commit is contained in:
Henrik Boström
2020-03-06 10:41:25 +01:00
committed by Commit Bot
parent f351cfffe2
commit efbec9a304
15 changed files with 540 additions and 283 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
BUILD.gn
View File

@ -608,6 +608,7 @@ if (rtc_include_tests) {
"test:test_main",
"test:video_test_common",
"video:video_tests",
"video/adaptation:video_adaptation_tests",
]
data = video_engine_tests_resources
if (is_android) {

2
call/adaptation/OWNERS
View File

@ -1,2 +1,2 @@
eshr@webrtc.org
hbos@webrtc.org
sprang@webrtc.org

2
video/BUILD.gn
View File

@ -238,6 +238,7 @@ rtc_library("video_stream_encoder_impl") {
"../rtc_base/task_utils:repeating_task",
"../system_wrappers",
"../system_wrappers:field_trial",
"adaptation:video_adaptation",
"//third_party/abseil-cpp/absl/algorithm:container",
"//third_party/abseil-cpp/absl/base:core_headers",
"//third_party/abseil-cpp/absl/types:optional",
@ -621,6 +622,7 @@ if (rtc_include_tests) {
"../test:test_support",
"../test:video_test_common",
"../test/time_controller",
"adaptation:video_adaptation",
"//testing/gtest",
"//third_party/abseil-cpp/absl/algorithm:container",
"//third_party/abseil-cpp/absl/memory",

48
video/adaptation/BUILD.gn Normal file
View File

@ -0,0 +1,48 @@
# Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
#
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
# tree. An additional intellectual property rights grant can be found
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
import("../../webrtc.gni")
rtc_library("video_adaptation") {
sources = [
"adaptation_counters.cc",
"adaptation_counters.h",
"video_stream_adapter.cc",
"video_stream_adapter.h",
]
deps = [
"../../call/adaptation:resource_adaptation",
"../../rtc_base:checks",
"../../rtc_base:logging",
"../../rtc_base:rtc_base_approved",
"../../rtc_base:rtc_event",
"../../rtc_base:rtc_numerics",
"//third_party/abseil-cpp/absl/types:optional",
]
}
if (rtc_include_tests) {
rtc_library("video_adaptation_tests") {
testonly = true
defines = []
sources = [ "adaptation_counters_unittest.cc" ]
deps = [
":video_adaptation",
"../../rtc_base:checks",
"../../rtc_base:logging",
"../../rtc_base:rtc_base_approved",
"../../rtc_base:rtc_event",
"../../rtc_base:rtc_numerics",
"//test:test_support",
"//testing/gtest",
"//third_party/abseil-cpp/absl/types:optional",
]
}
}

2
video/adaptation/OWNERS Normal file
View File

@ -0,0 +1,2 @@
eshr@webrtc.org
hbos@webrtc.org

38
video/adaptation/adaptation_counters.cc Normal file
View File

@ -0,0 +1,38 @@
/*
* Copyright 2020 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "video/adaptation/adaptation_counters.h"
namespace webrtc {
bool AdaptationCounters::operator==(const AdaptationCounters& rhs) const {
return fps_adaptations == rhs.fps_adaptations &&
resolution_adaptations == rhs.resolution_adaptations;
}
bool AdaptationCounters::operator!=(const AdaptationCounters& rhs) const {
return !(rhs == *this);
}
AdaptationCounters AdaptationCounters::operator+(
const AdaptationCounters& other) const {
return AdaptationCounters(
resolution_adaptations + other.resolution_adaptations,
fps_adaptations + other.fps_adaptations);
}
AdaptationCounters AdaptationCounters::operator-(
const AdaptationCounters& other) const {
return AdaptationCounters(
resolution_adaptations - other.resolution_adaptations,
fps_adaptations - other.fps_adaptations);
}
} // namespace webrtc

38
video/adaptation/adaptation_counters.h Normal file
View File

@ -0,0 +1,38 @@
/*
* Copyright 2020 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef VIDEO_ADAPTATION_ADAPTATION_COUNTERS_H_
#define VIDEO_ADAPTATION_ADAPTATION_COUNTERS_H_
namespace webrtc {
// Counts the number of adaptations have resulted due to resource overuse.
// Today we can adapt resolution and fps.
struct AdaptationCounters {
AdaptationCounters() : resolution_adaptations(0), fps_adaptations(0) {}
AdaptationCounters(int resolution_adaptations, int fps_adaptations)
: resolution_adaptations(resolution_adaptations),
fps_adaptations(fps_adaptations) {}
int Total() const { return fps_adaptations + resolution_adaptations; }
bool operator==(const AdaptationCounters& rhs) const;
bool operator!=(const AdaptationCounters& rhs) const;
AdaptationCounters operator+(const AdaptationCounters& other) const;
AdaptationCounters operator-(const AdaptationCounters& other) const;
int resolution_adaptations;
int fps_adaptations;
};
} // namespace webrtc
#endif // VIDEO_ADAPTATION_ADAPTATION_COUNTERS_H_

50
video/adaptation/adaptation_counters_unittest.cc Normal file
View File

@ -0,0 +1,50 @@
/*
* Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "video/adaptation/adaptation_counters.h"
#include "test/gtest.h"
namespace webrtc {
TEST(AdaptationCountersTest, Addition) {
AdaptationCounters a{0, 0};
AdaptationCounters b{1, 2};
AdaptationCounters total = a + b;
EXPECT_EQ(1, total.resolution_adaptations);
EXPECT_EQ(2, total.fps_adaptations);
}
TEST(AdaptationCountersTest, Subtraction) {
AdaptationCounters a{0, 1};
AdaptationCounters b{2, 1};
AdaptationCounters diff = a - b;
EXPECT_EQ(-2, diff.resolution_adaptations);
EXPECT_EQ(0, diff.fps_adaptations);
}
TEST(AdaptationCountersTest, Equality) {
AdaptationCounters a{1, 2};
AdaptationCounters b{2, 1};
EXPECT_EQ(a, a);
EXPECT_NE(a, b);
}
TEST(AdaptationCountersTest, SelfAdditionSubtraction) {
AdaptationCounters a{1, 0};
AdaptationCounters b{0, 1};
EXPECT_EQ(a, a + b - b);
EXPECT_EQ(a, b + a - b);
EXPECT_EQ(a, a - b + b);
EXPECT_EQ(a, b - b + a);
}
} // namespace webrtc

245
video/adaptation/video_stream_adapter.cc Normal file
View File

@ -0,0 +1,245 @@
/*
* Copyright 2020 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "video/adaptation/video_stream_adapter.h"
#include <algorithm>
#include <limits>
#include "absl/types/optional.h"
#include "rtc_base/constructor_magic.h"
#include "rtc_base/logging.h"
#include "rtc_base/numerics/safe_conversions.h"
namespace webrtc {
// VideoSourceRestrictor is responsible for keeping track of current
// VideoSourceRestrictions. It suggests higher and lower frame rates and
// resolutions (used by "maintain-resolution" and "maintain-framerate"), but is
// ultimately not reponsible for determining when or how we should adapt up or
// down (e.g. "balanced" mode also uses BalancedDegradationPreference).
class VideoStreamAdapter::VideoSourceRestrictor {
public:
// For frame rate, the steps we take are 2/3 (down) and 3/2 (up).
static int GetLowerFrameRateThan(int fps) {
RTC_DCHECK(fps != std::numeric_limits<int>::max());
return (fps * 2) / 3;
}
// TODO(hbos): Use absl::optional<> instead?
static int GetHigherFrameRateThan(int fps) {
return fps != std::numeric_limits<int>::max()
? (fps * 3) / 2
: std::numeric_limits<int>::max();
}
// For resolution, the steps we take are 3/5 (down) and 5/3 (up).
// Notice the asymmetry of which restriction property is set depending on if
// we are adapting up or down:
// - DecreaseResolution() sets the max_pixels_per_frame() to the desired
// target and target_pixels_per_frame() to null.
// - IncreaseResolutionTo() sets the target_pixels_per_frame() to the desired
// target, and max_pixels_per_frame() is set according to
// GetIncreasedMaxPixelsWanted().
static int GetLowerResolutionThan(int pixel_count) {
RTC_DCHECK(pixel_count != std::numeric_limits<int>::max());
return (pixel_count * 3) / 5;
}
// TODO(hbos): Use absl::optional<> instead?
static int GetHigherResolutionThan(int pixel_count) {
return pixel_count != std::numeric_limits<int>::max()
? (pixel_count * 5) / 3
: std::numeric_limits<int>::max();
}
VideoSourceRestrictor() {}
VideoSourceRestrictions source_restrictions() const {
return source_restrictions_;
}
const AdaptationCounters& adaptation_counters() const { return adaptations_; }
void ClearRestrictions() {
source_restrictions_ = VideoSourceRestrictions();
adaptations_ = AdaptationCounters();
}
bool CanDecreaseResolutionTo(int target_pixels, int min_pixels_per_frame) {
int max_pixels_per_frame = rtc::dchecked_cast<int>(
source_restrictions_.max_pixels_per_frame().value_or(
std::numeric_limits<int>::max()));
return target_pixels < max_pixels_per_frame &&
target_pixels >= min_pixels_per_frame;
}
void DecreaseResolutionTo(int target_pixels, int min_pixels_per_frame) {
RTC_DCHECK(CanDecreaseResolutionTo(target_pixels, min_pixels_per_frame));
RTC_LOG(LS_INFO) << "Scaling down resolution, max pixels: "
<< target_pixels;
source_restrictions_.set_max_pixels_per_frame(
target_pixels != std::numeric_limits<int>::max()
? absl::optional<size_t>(target_pixels)
: absl::nullopt);
source_restrictions_.set_target_pixels_per_frame(absl::nullopt);
++adaptations_.resolution_adaptations;
}
bool CanIncreaseResolutionTo(int target_pixels) {
int max_pixels_wanted = GetIncreasedMaxPixelsWanted(target_pixels);
int max_pixels_per_frame = rtc::dchecked_cast<int>(
source_restrictions_.max_pixels_per_frame().value_or(
std::numeric_limits<int>::max()));
return max_pixels_wanted > max_pixels_per_frame;
}
void IncreaseResolutionTo(int target_pixels) {
RTC_DCHECK(CanIncreaseResolutionTo(target_pixels));
int max_pixels_wanted = GetIncreasedMaxPixelsWanted(target_pixels);
RTC_LOG(LS_INFO) << "Scaling up resolution, max pixels: "
<< max_pixels_wanted;
source_restrictions_.set_max_pixels_per_frame(
max_pixels_wanted != std::numeric_limits<int>::max()
? absl::optional<size_t>(max_pixels_wanted)
: absl::nullopt);
source_restrictions_.set_target_pixels_per_frame(
max_pixels_wanted != std::numeric_limits<int>::max()
? absl::optional<size_t>(target_pixels)
: absl::nullopt);
--adaptations_.resolution_adaptations;
RTC_DCHECK_GE(adaptations_.resolution_adaptations, 0);
}
bool CanDecreaseFrameRateTo(int max_frame_rate) {
const int fps_wanted = std::max(kMinFramerateFps, max_frame_rate);
return fps_wanted < rtc::dchecked_cast<int>(
source_restrictions_.max_frame_rate().value_or(
std::numeric_limits<int>::max()));
}
void DecreaseFrameRateTo(int max_frame_rate) {
RTC_DCHECK(CanDecreaseFrameRateTo(max_frame_rate));
max_frame_rate = std::max(kMinFramerateFps, max_frame_rate);
RTC_LOG(LS_INFO) << "Scaling down framerate: " << max_frame_rate;
source_restrictions_.set_max_frame_rate(
max_frame_rate != std::numeric_limits<int>::max()
? absl::optional<double>(max_frame_rate)
: absl::nullopt);
++adaptations_.fps_adaptations;
}
bool CanIncreaseFrameRateTo(int max_frame_rate) {
return max_frame_rate > rtc::dchecked_cast<int>(
source_restrictions_.max_frame_rate().value_or(
std::numeric_limits<int>::max()));
}
void IncreaseFrameRateTo(int max_frame_rate) {
RTC_DCHECK(CanIncreaseFrameRateTo(max_frame_rate));
RTC_LOG(LS_INFO) << "Scaling up framerate: " << max_frame_rate;
source_restrictions_.set_max_frame_rate(
max_frame_rate != std::numeric_limits<int>::max()
? absl::optional<double>(max_frame_rate)
: absl::nullopt);
--adaptations_.fps_adaptations;
RTC_DCHECK_GE(adaptations_.fps_adaptations, 0);
}
private:
static int GetIncreasedMaxPixelsWanted(int target_pixels) {
if (target_pixels == std::numeric_limits<int>::max())
return std::numeric_limits<int>::max();
// When we decrease resolution, we go down to at most 3/5 of current pixels.
// Thus to increase resolution, we need 3/5 to get back to where we started.
// When going up, the desired max_pixels_per_frame() has to be significantly
// higher than the target because the source's native resolutions might not
// match the target. We pick 12/5 of the target.
//
// (This value was historically 4 times the old target, which is (3/5)*4 of
// the new target - or 12/5 - assuming the target is adjusted according to
// the above steps.)
RTC_DCHECK(target_pixels != std::numeric_limits<int>::max());
return (target_pixels * 12) / 5;
}
VideoSourceRestrictions source_restrictions_;
AdaptationCounters adaptations_;
RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceRestrictor);
};
const int VideoStreamAdapter::kMinFramerateFps = 2;
// static
int VideoStreamAdapter::GetLowerFrameRateThan(int fps) {
return VideoSourceRestrictor::GetLowerFrameRateThan(fps);
}
// static
int VideoStreamAdapter::GetHigherFrameRateThan(int fps) {
return VideoSourceRestrictor::GetHigherFrameRateThan(fps);
}
// static
int VideoStreamAdapter::GetLowerResolutionThan(int pixel_count) {
return VideoSourceRestrictor::GetLowerResolutionThan(pixel_count);
}
// static
int VideoStreamAdapter::GetHigherResolutionThan(int pixel_count) {
return VideoSourceRestrictor::GetHigherResolutionThan(pixel_count);
}
VideoStreamAdapter::VideoStreamAdapter()
: source_restrictor_(std::make_unique<VideoSourceRestrictor>()) {}
VideoStreamAdapter::~VideoStreamAdapter() {}
VideoSourceRestrictions VideoStreamAdapter::source_restrictions() const {
return source_restrictor_->source_restrictions();
}
const AdaptationCounters& VideoStreamAdapter::adaptation_counters() const {
return source_restrictor_->adaptation_counters();
}
void VideoStreamAdapter::ClearRestrictions() {
source_restrictor_->ClearRestrictions();
}
bool VideoStreamAdapter::CanDecreaseResolutionTo(int target_pixels,
int min_pixels_per_frame) {
return source_restrictor_->CanDecreaseResolutionTo(target_pixels,
min_pixels_per_frame);
}
void VideoStreamAdapter::DecreaseResolutionTo(int target_pixels,
int min_pixels_per_frame) {
source_restrictor_->DecreaseResolutionTo(target_pixels, min_pixels_per_frame);
}
bool VideoStreamAdapter::CanIncreaseResolutionTo(int target_pixels) {
return source_restrictor_->CanIncreaseResolutionTo(target_pixels);
}
void VideoStreamAdapter::IncreaseResolutionTo(int target_pixels) {
source_restrictor_->IncreaseResolutionTo(target_pixels);
}
bool VideoStreamAdapter::CanDecreaseFrameRateTo(int max_frame_rate) {
return source_restrictor_->CanDecreaseFrameRateTo(max_frame_rate);
}
void VideoStreamAdapter::DecreaseFrameRateTo(int max_frame_rate) {
source_restrictor_->DecreaseFrameRateTo(max_frame_rate);
}
bool VideoStreamAdapter::CanIncreaseFrameRateTo(int max_frame_rate) {
return source_restrictor_->CanIncreaseFrameRateTo(max_frame_rate);
}
void VideoStreamAdapter::IncreaseFrameRateTo(int max_frame_rate) {
source_restrictor_->IncreaseFrameRateTo(max_frame_rate);
}
} // namespace webrtc

66
video/adaptation/video_stream_adapter.h Normal file
View File

@ -0,0 +1,66 @@
/*
* Copyright 2020 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef VIDEO_ADAPTATION_VIDEO_STREAM_ADAPTER_H_
#define VIDEO_ADAPTATION_VIDEO_STREAM_ADAPTER_H_
#include <memory>
#include "call/adaptation/video_source_restrictions.h"
#include "video/adaptation/adaptation_counters.h"
namespace webrtc {
// Owns the VideoSourceRestriction for a single stream and is responsible for
// adapting it up or down when told to do so. This class serves the following
// purposes:
// 1. Keep track of a stream's restrictions.
// 2. Provide valid ways to adapt up or down the stream's restrictions.
// 3. Modify the stream's restrictions in one of the valid ways.
class VideoStreamAdapter {
public:
static const int kMinFramerateFps;
static int GetLowerFrameRateThan(int fps);
static int GetHigherFrameRateThan(int fps);
static int GetLowerResolutionThan(int pixel_count);
static int GetHigherResolutionThan(int pixel_count);
VideoStreamAdapter();
~VideoStreamAdapter();
// TODO(hbos): Why isn't this const?
VideoSourceRestrictions source_restrictions() const;
const AdaptationCounters& adaptation_counters() const;
void ClearRestrictions();
// "Can adapt?" and "do adapt!" methods.
// TODO(https://crbug.com/webrtc/11393): Make the adapter responsible for
// deciding what the next step are, i.e. taking on degradation preference
// logic. Then, these can be expressed either as CanAdaptUp() and DoAdaptUp()
// or as GetNextRestrictionsUp() and ApplyRestrictions().
bool CanDecreaseResolutionTo(int target_pixels, int min_pixels_per_frame);
void DecreaseResolutionTo(int target_pixels, int min_pixels_per_frame);
bool CanIncreaseResolutionTo(int target_pixels);
void IncreaseResolutionTo(int target_pixels);
bool CanDecreaseFrameRateTo(int max_frame_rate);
void DecreaseFrameRateTo(int max_frame_rate);
bool CanIncreaseFrameRateTo(int max_frame_rate);
void IncreaseFrameRateTo(int max_frame_rate);
private:
class VideoSourceRestrictor;
const std::unique_ptr<VideoSourceRestrictor> source_restrictor_;
};
} // namespace webrtc
#endif // VIDEO_ADAPTATION_VIDEO_STREAM_ADAPTER_H_

2
video/encode_usage_resource.h
View File

@ -26,6 +26,8 @@ namespace webrtc {
// indirectly by usage in the OveruseFrameDetectorResourceAdaptationModule
// (which is only tested because of its usage in VideoStreamEncoder); all tests
// are currently in video_stream_encoder_unittest.cc.
// TODO(https://crbug.com/webrtc/11222): Move this class to the
// video/adaptation/ subdirectory.
class EncodeUsageResource : public Resource,
public AdaptationObserverInterface {
public:

248
video/overuse_frame_detector_resource_adaptation_module.cc
View File

@ -32,8 +32,6 @@ namespace webrtc {
namespace {
const int kMinFramerateFps = 2;
bool IsResolutionScalingEnabled(DegradationPreference degradation_preference) {
return degradation_preference == DegradationPreference::MAINTAIN_FRAMERATE ||
degradation_preference == DegradationPreference::BALANCED;
@ -68,9 +66,9 @@ VideoSourceRestrictions ApplyDegradationPreference(
}
// Returns AdaptationCounters where constraints that don't apply to the
// degradation preference are cleared. This behaviour must reflect that of
// ApplyDegradationPreference for SourceRestrictions. Any changed to that
// method must also change this one.
// degredation preference are cleared. This behaviour must reflect that of
// ApplyDegredationPreference for SourceRestrictions. Any to that method must
// also change this one.
AdaptationCounters ApplyDegradationPreference(
AdaptationCounters counters,
DegradationPreference degradation_preference) {
@ -95,180 +93,6 @@ AdaptationCounters ApplyDegradationPreference(
} // namespace
bool AdaptationCounters::operator==(const AdaptationCounters& rhs) const {
return fps_adaptations == rhs.fps_adaptations &&
resolution_adaptations == rhs.resolution_adaptations;
}
bool AdaptationCounters::operator!=(const AdaptationCounters& rhs) const {
return !(rhs == *this);
}
AdaptationCounters AdaptationCounters::operator+(
const AdaptationCounters& other) const {
return AdaptationCounters(
resolution_adaptations + other.resolution_adaptations,
fps_adaptations + other.fps_adaptations);
}
AdaptationCounters AdaptationCounters::operator-(
const AdaptationCounters& other) const {
return AdaptationCounters(
resolution_adaptations - other.resolution_adaptations,
fps_adaptations - other.fps_adaptations);
}
// VideoSourceRestrictor is responsible for keeping track of current
// VideoSourceRestrictions and how to modify them in response to adapting up or
// down. It is not reponsible for determining when we should adapt up or down -
// for that, see
// OveruseFrameDetectorResourceAdaptationModule::OnResourceUnderuse() and
// OnResourceOveruse() - only how to modify the source/sink restrictions when
// this happens. Note that it is also not responsible for reconfigruring the
// source/sink, it is only a keeper of desired restrictions.
class OveruseFrameDetectorResourceAdaptationModule::VideoSourceRestrictor {
public:
// For frame rate, the steps we take are 2/3 (down) and 3/2 (up).
static int GetLowerFrameRateThan(int fps) {
RTC_DCHECK(fps != std::numeric_limits<int>::max());
return (fps * 2) / 3;
}
// TODO(hbos): Use absl::optional<> instead?
static int GetHigherFrameRateThan(int fps) {
return fps != std::numeric_limits<int>::max()
? (fps * 3) / 2
: std::numeric_limits<int>::max();
}
// For resolution, the steps we take are 3/5 (down) and 5/3 (up).
// Notice the asymmetry of which restriction property is set depending on if
// we are adapting up or down:
// - DecreaseResolution() sets the max_pixels_per_frame() to the desired
// target and target_pixels_per_frame() to null.
// - IncreaseResolutionTo() sets the target_pixels_per_frame() to the desired
// target, and max_pixels_per_frame() is set according to
// GetIncreasedMaxPixelsWanted().
static int GetLowerResolutionThan(int pixel_count) {
RTC_DCHECK(pixel_count != std::numeric_limits<int>::max());
return (pixel_count * 3) / 5;
}
// TODO(hbos): Use absl::optional<> instead?
static int GetHigherResolutionThan(int pixel_count) {
return pixel_count != std::numeric_limits<int>::max()
? (pixel_count * 5) / 3
: std::numeric_limits<int>::max();
}
VideoSourceRestrictor() {}
VideoSourceRestrictions source_restrictions() {
return source_restrictions_;
}
const AdaptationCounters& adaptation_counters() const { return adaptations_; }
void ClearRestrictions() {
source_restrictions_ = VideoSourceRestrictions();
adaptations_ = AdaptationCounters();
}
bool CanDecreaseResolutionTo(int target_pixels, int min_pixels_per_frame) {
int max_pixels_per_frame = rtc::dchecked_cast<int>(
source_restrictions_.max_pixels_per_frame().value_or(
std::numeric_limits<int>::max()));
return target_pixels < max_pixels_per_frame &&
target_pixels >= min_pixels_per_frame;
}
void DecreaseResolutionTo(int target_pixels, int min_pixels_per_frame) {
RTC_DCHECK(CanDecreaseResolutionTo(target_pixels, min_pixels_per_frame));
RTC_LOG(LS_INFO) << "Scaling down resolution, max pixels: "
<< target_pixels;
source_restrictions_.set_max_pixels_per_frame(
target_pixels != std::numeric_limits<int>::max()
? absl::optional<size_t>(target_pixels)
: absl::nullopt);
source_restrictions_.set_target_pixels_per_frame(absl::nullopt);
++adaptations_.resolution_adaptations;
}
bool CanIncreaseResolutionTo(int target_pixels) {
int max_pixels_wanted = GetIncreasedMaxPixelsWanted(target_pixels);
int max_pixels_per_frame = rtc::dchecked_cast<int>(
source_restrictions_.max_pixels_per_frame().value_or(
std::numeric_limits<int>::max()));
return max_pixels_wanted > max_pixels_per_frame;
}
void IncreaseResolutionTo(int target_pixels) {
RTC_DCHECK(CanIncreaseResolutionTo(target_pixels));
int max_pixels_wanted = GetIncreasedMaxPixelsWanted(target_pixels);
RTC_LOG(LS_INFO) << "Scaling up resolution, max pixels: "
<< max_pixels_wanted;
source_restrictions_.set_max_pixels_per_frame(
max_pixels_wanted != std::numeric_limits<int>::max()
? absl::optional<size_t>(max_pixels_wanted)
: absl::nullopt);
source_restrictions_.set_target_pixels_per_frame(
max_pixels_wanted != std::numeric_limits<int>::max()
? absl::optional<size_t>(target_pixels)
: absl::nullopt);
--adaptations_.resolution_adaptations;
RTC_DCHECK_GE(adaptations_.resolution_adaptations, 0);
}
bool CanDecreaseFrameRateTo(int max_frame_rate) {
const int fps_wanted = std::max(kMinFramerateFps, max_frame_rate);
return fps_wanted < rtc::dchecked_cast<int>(
source_restrictions_.max_frame_rate().value_or(
std::numeric_limits<int>::max()));
}
void DecreaseFrameRateTo(int max_frame_rate) {
RTC_DCHECK(CanDecreaseFrameRateTo(max_frame_rate));
max_frame_rate = std::max(kMinFramerateFps, max_frame_rate);
RTC_LOG(LS_INFO) << "Scaling down framerate: " << max_frame_rate;
source_restrictions_.set_max_frame_rate(
max_frame_rate != std::numeric_limits<int>::max()
? absl::optional<double>(max_frame_rate)
: absl::nullopt);
++adaptations_.fps_adaptations;
}
bool CanIncreaseFrameRateTo(int max_frame_rate) {
return max_frame_rate > rtc::dchecked_cast<int>(
source_restrictions_.max_frame_rate().value_or(
std::numeric_limits<int>::max()));
}
void IncreaseFrameRateTo(int max_frame_rate) {
RTC_DCHECK(CanIncreaseFrameRateTo(max_frame_rate));
RTC_LOG(LS_INFO) << "Scaling up framerate: " << max_frame_rate;
source_restrictions_.set_max_frame_rate(
max_frame_rate != std::numeric_limits<int>::max()
? absl::optional<double>(max_frame_rate)
: absl::nullopt);
--adaptations_.fps_adaptations;
RTC_DCHECK_GE(adaptations_.fps_adaptations, 0);
}
private:
static int GetIncreasedMaxPixelsWanted(int target_pixels) {
if (target_pixels == std::numeric_limits<int>::max())
return std::numeric_limits<int>::max();
// When we decrease resolution, we go down to at most 3/5 of current pixels.
// Thus to increase resolution, we need 3/5 to get back to where we started.
// When going up, the desired max_pixels_per_frame() has to be significantly
// higher than the target because the source's native resolutions might not
// match the target. We pick 12/5 of the target.
//
// (This value was historically 4 times the old target, which is (3/5)*4 of
// the new target - or 12/5 - assuming the target is adjusted according to
// the above steps.)
RTC_DCHECK(target_pixels != std::numeric_limits<int>::max());
return (target_pixels * 12) / 5;
}
VideoSourceRestrictions source_restrictions_;
AdaptationCounters adaptations_;
RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceRestrictor);
};
class OveruseFrameDetectorResourceAdaptationModule::InitialFrameDropper {
public:
explicit InitialFrameDropper(QualityScalerResource* quality_scaler_resource)
@ -359,7 +183,7 @@ OveruseFrameDetectorResourceAdaptationModule::
degradation_preference_(DegradationPreference::DISABLED),
balanced_settings_(),
last_adaptation_request_(absl::nullopt),
source_restrictor_(std::make_unique<VideoSourceRestrictor>()),
stream_adapter_(std::make_unique<VideoStreamAdapter>()),
encode_usage_resource_(
std::make_unique<EncodeUsageResource>(std::move(overuse_detector))),
quality_scaler_resource_(std::make_unique<QualityScalerResource>()),
@ -442,7 +266,7 @@ void OveruseFrameDetectorResourceAdaptationModule::SetDegradationPreference(
last_adaptation_request_.reset();
if (degradation_preference == DegradationPreference::BALANCED ||
degradation_preference_ == DegradationPreference::BALANCED) {
source_restrictor_->ClearRestrictions();
stream_adapter_->ClearRestrictions();
active_counts_.fill(AdaptationCounters());
}
}
@ -484,7 +308,7 @@ void OveruseFrameDetectorResourceAdaptationModule::SetEncoderRates(
void OveruseFrameDetectorResourceAdaptationModule::
ResetVideoSourceRestrictions() {
last_adaptation_request_.reset();
source_restrictor_->ClearRestrictions();
stream_adapter_->ClearRestrictions();
active_counts_.fill(AdaptationCounters());
MaybeUpdateVideoSourceRestrictions();
}
@ -495,16 +319,15 @@ void OveruseFrameDetectorResourceAdaptationModule::OnFrame(
}
void OveruseFrameDetectorResourceAdaptationModule::OnFrameDroppedDueToSize() {
AdaptationCounters counters_before =
source_restrictor_->adaptation_counters();
AdaptationCounters counters_before = stream_adapter_->adaptation_counters();
OnResourceOveruse(AdaptationObserverInterface::AdaptReason::kQuality);
if (degradation_preference() == DegradationPreference::BALANCED &&
source_restrictor_->adaptation_counters().fps_adaptations >
stream_adapter_->adaptation_counters().fps_adaptations >
counters_before.fps_adaptations) {
// Adapt framerate in same step as resolution.
OnResourceOveruse(AdaptationObserverInterface::AdaptReason::kQuality);
}
if (source_restrictor_->adaptation_counters().resolution_adaptations >
if (stream_adapter_->adaptation_counters().resolution_adaptations >
counters_before.resolution_adaptations) {
encoder_stats_observer_->OnInitialQualityResolutionAdaptDown();
}
@ -677,7 +500,7 @@ OveruseFrameDetectorResourceAdaptationModule::GetAdaptUpTarget(
// Attempt to increase target frame rate.
int target_fps =
balanced_settings_.MaxFps(GetVideoCodecTypeOrGeneric(), input_pixels);
if (source_restrictor_->CanIncreaseFrameRateTo(target_fps)) {
if (stream_adapter_->CanIncreaseFrameRateTo(target_fps)) {
return AdaptationTarget(AdaptationAction::kIncreaseFrameRate,
target_fps);
}
@ -702,14 +525,13 @@ OveruseFrameDetectorResourceAdaptationModule::GetAdaptUpTarget(
}
// Attempt to increase pixel count.
int target_pixels = input_pixels;
if (source_restrictor_->adaptation_counters().resolution_adaptations ==
1) {
if (stream_adapter_->adaptation_counters().resolution_adaptations == 1) {
RTC_LOG(LS_INFO) << "Removing resolution down-scaling setting.";
target_pixels = std::numeric_limits<int>::max();
}
target_pixels =
VideoSourceRestrictor::GetHigherResolutionThan(target_pixels);
if (!source_restrictor_->CanIncreaseResolutionTo(target_pixels))
VideoStreamAdapter::GetHigherResolutionThan(target_pixels);
if (!stream_adapter_->CanIncreaseResolutionTo(target_pixels))
return absl::nullopt;
return AdaptationTarget(AdaptationAction::kIncreaseResolution,
target_pixels);
@ -717,12 +539,12 @@ OveruseFrameDetectorResourceAdaptationModule::GetAdaptUpTarget(
case DegradationPreference::MAINTAIN_RESOLUTION: {
// Scale up framerate.
int target_fps = input_fps;
if (source_restrictor_->adaptation_counters().fps_adaptations == 1) {
if (stream_adapter_->adaptation_counters().fps_adaptations == 1) {
RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting.";
target_fps = std::numeric_limits<int>::max();
}
target_fps = VideoSourceRestrictor::GetHigherFrameRateThan(target_fps);
if (!source_restrictor_->CanIncreaseFrameRateTo(target_fps))
target_fps = VideoStreamAdapter::GetHigherFrameRateThan(target_fps);
if (!stream_adapter_->CanIncreaseFrameRateTo(target_fps))
return absl::nullopt;
return AdaptationTarget(AdaptationAction::kIncreaseFrameRate, target_fps);
}
@ -755,8 +577,8 @@ OveruseFrameDetectorResourceAdaptationModule::GetAdaptDownTarget(
DegradationPreference::MAINTAIN_RESOLUTION) {
// TODO(hbos): This usage of |last_adaptation_was_down| looks like a mistake
// - delete it.
if (input_fps <= 0 ||
(last_adaptation_was_down && input_fps < kMinFramerateFps)) {
if (input_fps <= 0 || (last_adaptation_was_down &&
input_fps < VideoStreamAdapter::kMinFramerateFps)) {
return absl::nullopt;
}
}
@ -775,7 +597,7 @@ OveruseFrameDetectorResourceAdaptationModule::GetAdaptDownTarget(
// Try scale down framerate, if lower.
int target_fps =
balanced_settings_.MinFps(GetVideoCodecTypeOrGeneric(), input_pixels);
if (source_restrictor_->CanDecreaseFrameRateTo(target_fps)) {
if (stream_adapter_->CanDecreaseFrameRateTo(target_fps)) {
return AdaptationTarget(AdaptationAction::kDecreaseFrameRate,
target_fps);
}
@ -785,22 +607,22 @@ OveruseFrameDetectorResourceAdaptationModule::GetAdaptDownTarget(
case DegradationPreference::MAINTAIN_FRAMERATE: {
// Scale down resolution.
int target_pixels =
VideoSourceRestrictor::GetLowerResolutionThan(input_pixels);
VideoStreamAdapter::GetLowerResolutionThan(input_pixels);
// TODO(https://crbug.com/webrtc/11222): Move this logic to
// ApplyAdaptationTarget() or elsewhere - simply checking which adaptation
// target is available should not have side-effects.
if (target_pixels < min_pixels_per_frame)
encoder_stats_observer_->OnMinPixelLimitReached();
if (!source_restrictor_->CanDecreaseResolutionTo(target_pixels,
min_pixels_per_frame)) {
if (!stream_adapter_->CanDecreaseResolutionTo(target_pixels,
min_pixels_per_frame)) {
return absl::nullopt;
}
return AdaptationTarget(AdaptationAction::kDecreaseResolution,
target_pixels);
}
case DegradationPreference::MAINTAIN_RESOLUTION: {
int target_fps = VideoSourceRestrictor::GetLowerFrameRateThan(input_fps);
if (!source_restrictor_->CanDecreaseFrameRateTo(target_fps))
int target_fps = VideoStreamAdapter::GetLowerFrameRateThan(input_fps);
if (!stream_adapter_->CanDecreaseFrameRateTo(target_fps))
return absl::nullopt;
return AdaptationTarget(AdaptationAction::kDecreaseFrameRate, target_fps);
}
@ -816,28 +638,26 @@ void OveruseFrameDetectorResourceAdaptationModule::ApplyAdaptationTarget(
AdaptationObserverInterface::AdaptReason reason) {
switch (target.action) {
case AdaptationAction::kIncreaseResolution:
source_restrictor_->IncreaseResolutionTo(target.value);
stream_adapter_->IncreaseResolutionTo(target.value);
return;
case AdaptationAction::kDecreaseResolution:
source_restrictor_->DecreaseResolutionTo(target.value,
min_pixels_per_frame);
stream_adapter_->DecreaseResolutionTo(target.value, min_pixels_per_frame);
return;
case AdaptationAction::kIncreaseFrameRate:
source_restrictor_->IncreaseFrameRateTo(target.value);
stream_adapter_->IncreaseFrameRateTo(target.value);
// TODO(https://crbug.com/webrtc/11222): Don't adapt in two steps.
// GetAdaptUpTarget() should tell us the correct value, but BALANCED logic
// in DecrementFramerate() makes it hard to predict whether this will be
// the last step. Remove the dependency on GetConstAdaptCounter().
if (EffectiveDegradationPreference() == DegradationPreference::BALANCED &&
source_restrictor_->adaptation_counters().fps_adaptations == 0 &&
stream_adapter_->adaptation_counters().fps_adaptations == 0 &&
target.value != std::numeric_limits<int>::max()) {
RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting.";
source_restrictor_->IncreaseFrameRateTo(
std::numeric_limits<int>::max());
stream_adapter_->IncreaseFrameRateTo(std::numeric_limits<int>::max());
}
return;
case AdaptationAction::kDecreaseFrameRate:
source_restrictor_->DecreaseFrameRateTo(target.value);
stream_adapter_->DecreaseFrameRateTo(target.value);
return;
}
}
@ -956,7 +776,7 @@ int OveruseFrameDetectorResourceAdaptationModule::MinPixelsPerFrame() const {
void OveruseFrameDetectorResourceAdaptationModule::
MaybeUpdateVideoSourceRestrictions() {
VideoSourceRestrictions new_restrictions = ApplyDegradationPreference(
source_restrictor_->source_restrictions(), degradation_preference_);
stream_adapter_->source_restrictions(), degradation_preference_);
if (video_source_restrictions_ != new_restrictions) {
video_source_restrictions_ = std::move(new_restrictions);
adaptation_listener_->OnVideoSourceRestrictionsUpdated(
@ -977,7 +797,7 @@ void OveruseFrameDetectorResourceAdaptationModule::
// module. This is used to make sure overuse detection doesn't needlessly
// trigger in low and/or variable framerate scenarios.
absl::optional<double> target_frame_rate =
ApplyDegradationPreference(source_restrictor_->source_restrictions(),
ApplyDegradationPreference(stream_adapter_->source_restrictions(),
degradation_preference_)
.max_frame_rate();
if (!target_frame_rate.has_value() ||
@ -1052,7 +872,7 @@ void OveruseFrameDetectorResourceAdaptationModule::UpdateAdaptationStats(
AdaptationCounters& active_count = active_counts_[reason];
AdaptationCounters& other_active = active_counts_[(reason + 1) % 2];
const AdaptationCounters total_counts =
source_restrictor_->adaptation_counters();
stream_adapter_->adaptation_counters();
OnAdaptationCountChanged(total_counts, &active_count, &other_active);
@ -1126,7 +946,7 @@ bool OveruseFrameDetectorResourceAdaptationModule::CanAdaptUpResolution(
encoder_settings_.has_value()
? GetEncoderBitrateLimits(
encoder_settings_->encoder_info(),
VideoSourceRestrictor::GetHigherResolutionThan(pixels))
VideoStreamAdapter::GetHigherResolutionThan(pixels))
: absl::nullopt;
if (!bitrate_limits.has_value() || bitrate_bps == 0) {
return true; // No limit configured or bitrate provided.

30
video/overuse_frame_detector_resource_adaptation_module.h
View File

@ -32,32 +32,14 @@
#include "rtc_base/experiments/quality_scaler_settings.h"
#include "rtc_base/strings/string_builder.h"
#include "system_wrappers/include/clock.h"
#include "video/adaptation/adaptation_counters.h"
#include "video/adaptation/video_stream_adapter.h"
#include "video/encode_usage_resource.h"
#include "video/overuse_frame_detector.h"
#include "video/quality_scaler_resource.h"
namespace webrtc {
// Counts the number of adaptations have resulted due to resource overuse.
// Today we can adapt resolution and fps.
struct AdaptationCounters {
AdaptationCounters() : resolution_adaptations(0), fps_adaptations(0) {}
AdaptationCounters(int resolution_adaptations, int fps_adaptations)
: resolution_adaptations(resolution_adaptations),
fps_adaptations(fps_adaptations) {}
int Total() const { return fps_adaptations + resolution_adaptations; }
bool operator==(const AdaptationCounters& rhs) const;
bool operator!=(const AdaptationCounters& rhs) const;
AdaptationCounters operator+(const AdaptationCounters& other) const;
AdaptationCounters operator-(const AdaptationCounters& other) const;
int resolution_adaptations;
int fps_adaptations;
};
class VideoStreamEncoder;
// This class is used by the VideoStreamEncoder and is responsible for adapting
@ -70,10 +52,8 @@ class VideoStreamEncoder;
// TODO(hbos): Add unittests specific to this class, it is currently only tested
// indirectly in video_stream_encoder_unittest.cc and other tests exercising
// VideoStreamEncoder.
// TODO(hbos): Create and implement an abstract interface
// ResourceAdaptationModuleInterface and make this class inherit it. Use the
// generic interface in VideoStreamEncoder, unblocking other modules from being
// implemented and used.
// TODO(https://crbug.com/webrtc/11222): Rename this class to something more
// appropriate and move it to the video/adaptation/ subdirectory.
class OveruseFrameDetectorResourceAdaptationModule
: public ResourceAdaptationModuleInterface,
public ResourceListener {
@ -250,7 +230,7 @@ class OveruseFrameDetectorResourceAdaptationModule
// or AdaptDown signal.
absl::optional<AdaptationRequest> last_adaptation_request_;
// Keeps track of source restrictions that this adaptation module outputs.
const std::unique_ptr<VideoSourceRestrictor> source_restrictor_;
const std::unique_ptr<VideoStreamAdapter> stream_adapter_;
const std::unique_ptr<EncodeUsageResource> encode_usage_resource_;
const std::unique_ptr<QualityScalerResource> quality_scaler_resource_;
const std::unique_ptr<InitialFrameDropper> initial_frame_dropper_;

49
video/overuse_frame_detector_resource_adaptation_unittest.cc
View File

@ -12,42 +12,10 @@
#include "test/gmock.h"
#include "test/gtest.h"
#include "video/adaptation/adaptation_counters.h"
namespace webrtc {
TEST(AdaptationCountersTest, Addition) {
AdaptationCounters a;
AdaptationCounters b(1, 2);
AdaptationCounters total = a + b;
EXPECT_EQ(1, total.resolution_adaptations);
EXPECT_EQ(2, total.fps_adaptations);
}
TEST(AdaptationCountersTest, Subtraction) {
AdaptationCounters a(0, 1);
AdaptationCounters b(2, 1);
AdaptationCounters diff = a - b;
EXPECT_EQ(-2, diff.resolution_adaptations);
EXPECT_EQ(0, diff.fps_adaptations);
}
TEST(AdaptationCountersTest, Equality) {
AdaptationCounters a(1, 2);
AdaptationCounters b(2, 1);
EXPECT_EQ(a, a);
EXPECT_NE(a, b);
}
TEST(AdaptationCountersTest, SelfAdditionSubtraction) {
AdaptationCounters a(1, 0);
AdaptationCounters b(0, 1);
EXPECT_EQ(a, a + b - b);
EXPECT_EQ(a, b + a - b);
EXPECT_EQ(a, a - b + b);
EXPECT_EQ(a, b - b + a);
}
TEST(OveruseFrameDetectorResourceAdaptationModuleTest,
FirstAdaptationDown_Fps) {
AdaptationCounters cpu;
@ -108,11 +76,9 @@ TEST(OveruseFrameDetectorResourceAdaptationModuleTest,
AdaptationCounters qp(1, 0);
AdaptationCounters total(0, 1);
// CPU adaptation for resolution, but no
// resolution adaptation left from CPU.
// We then borrow the resolution
// adaptation from qp, and give qp the
// fps adaptation from CPU.
// CPU adaptation for resolution, but no resolution adaptation left from CPU.
// We then borrow the resolution adaptation from qp, and give qp the fps
// adaptation from CPU.
OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged(
total, &cpu, &qp);
@ -127,11 +93,8 @@ TEST(OveruseFrameDetectorResourceAdaptationModuleTest, AdaptUpWithBorrow_Fps) {
AdaptationCounters qp(0, 1);
AdaptationCounters total(1, 0);
// CPU adaptation for fps, but no
// fps adaptation left from CPU. We
// then borrow the fps adaptation
// from qp, and give qp the
// resolution adaptation from CPU.
// CPU adaptation for fps, but no fps adaptation left from CPU. We then borrow
// the fps adaptation from qp, and give qp the resolution adaptation from CPU.
OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged(
total, &cpu, &qp);

2
video/quality_scaler_resource.h
View File

@ -25,6 +25,8 @@ namespace webrtc {
// indirectly by usage in the OveruseFrameDetectorResourceAdaptationModule
// (which is only tested because of its usage in VideoStreamEncoder); all tests
// are currently in video_stream_encoder_unittest.cc.
// TODO(https://crbug.com/webrtc/11222): Move this class to the
// video/adaptation/ subdirectory.
class QualityScalerResource : public Resource,
public AdaptationObserverInterface {
public: