zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support injecting new Resources for overuse

Browse Source 
* This replaces the video stream methods for forcing adaptation
with a mock resource that triggers overuse.
* Resources can now be injected to the Module using the AddResource
function.
* Resources now have tests for adding and removing callbacks.
* Quality/EncoderUse% resources are tracked in the Resource list of
the adaptation module.
* The adaptation module ties all resources to a reason to keep stats
working as expected.

BUG=webrtc:11377

Change-Id: I1f5902f7416dc41b4915c0072e6f0da2bb3bb2b7
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/168948
Commit-Queue: Evan Shrubsole <eshr@google.com>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30610}
This commit is contained in:
Evan Shrubsole
2020-02-25 16:26:01 +01:00
committed by Commit Bot
parent 0165d5c32c
commit aa6fbc156e
15 changed files with 221 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
call/adaptation/BUILD.gn
View File

@ -29,6 +29,7 @@ rtc_library("resource_adaptation") {
"../../api/video_codecs:video_codecs_api",
"../../rtc_base:checks",
"../../rtc_base:rtc_base_approved",
"//third_party/abseil-cpp/absl/algorithm:container",
"//third_party/abseil-cpp/absl/types:optional",
]
}
@ -37,7 +38,10 @@ if (rtc_include_tests) {
rtc_library("resource_adaptation_tests") {
testonly = true
sources = [ "resource_adaptation_processor_unittest.cc" ]
sources = [
"resource_adaptation_processor_unittest.cc",
"resource_unittest.cc",
]
deps = [
":resource_adaptation",
":resource_adaptation_test_utilities",

14
call/adaptation/resource.cc
View File

@ -10,6 +10,7 @@
#include "call/adaptation/resource.h"
#include "absl/algorithm/container.h"
#include "rtc_base/checks.h"
namespace webrtc {
@ -18,13 +19,24 @@ ResourceListener::~ResourceListener() {}
Resource::Resource() : usage_state_(ResourceUsageState::kStable) {}
Resource::~Resource() {}
Resource::~Resource() {
RTC_DCHECK(listeners_.empty());
}
void Resource::RegisterListener(ResourceListener* listener) {
RTC_DCHECK(listener);
RTC_DCHECK(absl::c_find(listeners_, listener) == listeners_.end())
<< "ResourceListener was added twice.";
listeners_.push_back(listener);
}
void Resource::UnregisterListener(ResourceListener* listener) {
RTC_DCHECK(listener);
auto it = absl::c_find(listeners_, listener);
if (it != listeners_.end())
listeners_.erase(it);
}
ResourceUsageState Resource::usage_state() const {
return usage_state_;
}

6
call/adaptation/resource.h
View File

@ -11,6 +11,7 @@
#ifndef CALL_ADAPTATION_RESOURCE_H_
#define CALL_ADAPTATION_RESOURCE_H_
#include <string>
#include <vector>
#include "absl/types/optional.h"
@ -78,12 +79,13 @@ class Resource {
Resource();
virtual ~Resource();
// TODO(https://crbug.com/webrtc/11222): Make it possible to unregister
// listeners and DCHECK that they're all unregistered in the destructor.
void RegisterListener(ResourceListener* listener);
void UnregisterListener(ResourceListener* listener);
ResourceUsageState usage_state() const;
virtual std::string name() const = 0;
protected:
// Updates the usage state and informs all registered listeners.
// Returns the result of the last listener's OnResourceUsageStateMeasured()

4
call/adaptation/resource_adaptation_module_interface.h
View File

@ -16,6 +16,7 @@
#include "api/video/video_frame.h"
#include "api/video_codecs/video_encoder.h"
#include "api/video_codecs/video_encoder_config.h"
#include "call/adaptation/resource.h"
#include "call/adaptation/video_source_restrictions.h"
namespace webrtc {
@ -78,6 +79,9 @@ class ResourceAdaptationModuleInterface {
ResourceAdaptationModuleListener* adaptation_listener) = 0;
virtual void StopResourceAdaptation() = 0;
// The resource must out-live the module.
virtual void AddResource(Resource* resource) = 0;
// The following methods are callable whether or not adaption is started.
// Informs the module whether we have input video. By default, the module must

52
call/adaptation/resource_unittest.cc Normal file
View File

@ -0,0 +1,52 @@
/*
* Copyright 2020 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "call/adaptation/resource.h"
#include "call/adaptation/test/fake_resource.h"
#include "test/gmock.h"
#include "test/gtest.h"
namespace webrtc {
using ::testing::_;
using ::testing::StrictMock;
class MockResourceListener : public ResourceListener {
public:
MOCK_METHOD(ResourceListenerResponse,
OnResourceUsageStateMeasured,
(const Resource& resource));
};
TEST(ResourceTest, AddingListenerReceivesCallbacks) {
StrictMock<MockResourceListener> resource_listener;
FakeResource fake_resource(ResourceUsageState::kStable);
fake_resource.RegisterListener(&resource_listener);
EXPECT_CALL(resource_listener, OnResourceUsageStateMeasured(_))
.Times(1)
.WillOnce([](const Resource& resource) {
EXPECT_EQ(ResourceUsageState::kOveruse, resource.usage_state());
return ResourceListenerResponse::kNothing;
});
fake_resource.set_usage_state(ResourceUsageState::kOveruse);
fake_resource.UnregisterListener(&resource_listener);
}
TEST(ResourceTest, RemovingListenerStopsCallbacks) {
StrictMock<MockResourceListener> resource_listener;
FakeResource fake_resource(ResourceUsageState::kStable);
fake_resource.RegisterListener(&resource_listener);
fake_resource.UnregisterListener(&resource_listener);
EXPECT_CALL(resource_listener, OnResourceUsageStateMeasured(_)).Times(0);
fake_resource.set_usage_state(ResourceUsageState::kOveruse);
}
} // namespace webrtc

12
call/adaptation/test/fake_resource.cc
View File

@ -14,14 +14,18 @@
namespace webrtc {
FakeResource::FakeResource(ResourceUsageState usage_state) : Resource() {
set_usage_state(usage_state);
}
FakeResource::FakeResource(ResourceUsageState usage_state)
: FakeResource(usage_state, "FakeResource") {}
FakeResource::~FakeResource() {}
void FakeResource::set_usage_state(ResourceUsageState usage_state) {
OnResourceUsageStateMeasured(usage_state);
last_response_ = OnResourceUsageStateMeasured(usage_state);
}
FakeResource::FakeResource(ResourceUsageState usage_state,
const std::string& name)
: Resource(), name_(name) {
set_usage_state(usage_state);
}
} // namespace webrtc

11
call/adaptation/test/fake_resource.h
View File

@ -21,9 +21,20 @@ namespace webrtc {
class FakeResource : public Resource {
public:
explicit FakeResource(ResourceUsageState usage_state);
FakeResource(ResourceUsageState usage_state, const std::string& name);
~FakeResource() override;
void set_usage_state(ResourceUsageState usage_state);
absl::optional<ResourceListenerResponse> last_response() const {
return last_response_;
}
std::string name() const override { return name_; }
private:
absl::optional<ResourceListenerResponse> last_response_;
const std::string name_;
};
} // namespace webrtc

1
video/BUILD.gn
View File

@ -566,6 +566,7 @@ if (rtc_include_tests) {
"../call:simulated_packet_receiver",
"../call:video_stream_api",
"../call/adaptation:resource_adaptation",
"../call/adaptation:resource_adaptation_test_utilities",
"../common_video",
"../common_video/test:utilities",
"../media:rtc_audio_video",

3
video/encode_usage_resource.h
View File

@ -12,6 +12,7 @@
#define VIDEO_ENCODE_USAGE_RESOURCE_H_
#include <memory>
#include <string>
#include "absl/types/optional.h"
#include "call/adaptation/resource.h"
@ -48,6 +49,8 @@ class EncodeUsageResource : public Resource,
void AdaptUp(AdaptReason reason) override;
bool AdaptDown(AdaptReason reason) override;
std::string name() const override { return "EncoderUsageResource"; }
private:
int TargetFrameRateAsInt();

66
video/overuse_frame_detector_resource_adaptation_module.cc
View File

@ -394,6 +394,7 @@ OveruseFrameDetectorResourceAdaptationModule::
ResourceAdaptationModuleListener* adaptation_listener)
: adaptation_listener_(adaptation_listener),
clock_(clock),
state_(State::kStopped),
experiment_cpu_load_estimator_(experiment_cpu_load_estimator),
has_input_video_(false),
degradation_preference_(DegradationPreference::DISABLED),
@ -416,26 +417,56 @@ OveruseFrameDetectorResourceAdaptationModule::
encoder_stats_observer_(encoder_stats_observer) {
RTC_DCHECK(adaptation_listener_);
RTC_DCHECK(encoder_stats_observer_);
encode_usage_resource_->RegisterListener(this);
quality_scaler_resource_->RegisterListener(this);
AddResource(encode_usage_resource_.get(),
AdaptationObserverInterface::AdaptReason::kCpu);
AddResource(quality_scaler_resource_.get(),
AdaptationObserverInterface::AdaptReason::kQuality);
}
OveruseFrameDetectorResourceAdaptationModule::
~OveruseFrameDetectorResourceAdaptationModule() {}
~OveruseFrameDetectorResourceAdaptationModule() {
RTC_DCHECK_EQ(state_, State::kStopped);
}
void OveruseFrameDetectorResourceAdaptationModule::StartResourceAdaptation(
ResourceAdaptationModuleListener* adaptation_listener) {
RTC_DCHECK_EQ(state_, State::kStopped);
RTC_DCHECK(encoder_settings_.has_value());
// TODO(https://crbug.com/webrtc/11222): Rethink when the adaptation listener
// should be passed in and why. If resources are separated from modules then
// those resources may be started or stopped separately from the module.
RTC_DCHECK_EQ(adaptation_listener, adaptation_listener_);
encode_usage_resource_->StartCheckForOveruse(GetCpuOveruseOptions());
for (auto& resource_and_reason : resources_) {
resource_and_reason.resource->RegisterListener(this);
}
state_ = State::kStarted;
}
void OveruseFrameDetectorResourceAdaptationModule::StopResourceAdaptation() {
encode_usage_resource_->StopCheckForOveruse();
quality_scaler_resource_->StopCheckForOveruse();
for (auto& resource_and_reason : resources_) {
resource_and_reason.resource->UnregisterListener(this);
}
state_ = State::kStopped;
}
void OveruseFrameDetectorResourceAdaptationModule::AddResource(
Resource* resource) {
return AddResource(resource, AdaptationObserverInterface::AdaptReason::kCpu);
}
void OveruseFrameDetectorResourceAdaptationModule::AddResource(
Resource* resource,
AdaptationObserverInterface::AdaptReason reason) {
RTC_DCHECK(resource);
RTC_DCHECK(absl::c_find_if(resources_,
[resource](const ResourceAndReason& r) {
return r.resource == resource;
}) == resources_.end())
<< "Resource " << resource->name() << " already was inserted";
resources_.emplace_back(resource, reason);
}
void OveruseFrameDetectorResourceAdaptationModule::SetHasInputVideo(
@ -618,14 +649,15 @@ void OveruseFrameDetectorResourceAdaptationModule::ConfigureQualityScaler(
ResourceListenerResponse
OveruseFrameDetectorResourceAdaptationModule::OnResourceUsageStateMeasured(
const Resource& resource) {
// If we didn't have this dependency on AdaptReason the module could be
// listening to other types of Resources.
RTC_DCHECK(&resource == encode_usage_resource_.get() ||
&resource == quality_scaler_resource_.get());
AdaptationObserverInterface::AdaptReason reason =
&resource == encode_usage_resource_.get()
? AdaptationObserverInterface::AdaptReason::kCpu
: AdaptationObserverInterface::AdaptReason::kQuality;
const auto& registered_resource =
absl::c_find_if(resources_, [&resource](const ResourceAndReason& r) {
return r.resource == &resource;
});
RTC_DCHECK(registered_resource != resources_.end())
<< resource.name() << " not found.";
const AdaptationObserverInterface::AdaptReason reason =
registered_resource->reason;
switch (resource.usage_state()) {
case ResourceUsageState::kOveruse:
return OnResourceOveruse(reason);
@ -644,17 +676,6 @@ OveruseFrameDetectorResourceAdaptationModule::OnResourceUsageStateMeasured(
}
}
void OveruseFrameDetectorResourceAdaptationModule::OnResourceUnderuseForTesting(
AdaptationObserverInterface::AdaptReason reason) {
OnResourceUnderuse(reason);
}
bool OveruseFrameDetectorResourceAdaptationModule::OnResourceOveruseForTesting(
AdaptationObserverInterface::AdaptReason reason) {
return OnResourceOveruse(reason) !=
ResourceListenerResponse::kQualityScalerShouldIncreaseFrequency;
}
bool OveruseFrameDetectorResourceAdaptationModule::CanAdaptUp(
AdaptationObserverInterface::AdaptReason reason,
const AdaptationRequest& adaptation_request) {
@ -1067,7 +1088,6 @@ bool OveruseFrameDetectorResourceAdaptationModule::CanAdaptUpResolution(
return bitrate_bps >=
static_cast<uint32_t>(bitrate_limits->min_start_bitrate_bps);
}
void OveruseFrameDetectorResourceAdaptationModule::
MaybePerformQualityRampupExperiment() {
if (!quality_scaler_resource_->is_started())

34
video/overuse_frame_detector_resource_adaptation_module.h
View File

@ -75,6 +75,10 @@ class OveruseFrameDetectorResourceAdaptationModule
void StartResourceAdaptation(
ResourceAdaptationModuleListener* adaptation_listener) override;
void StopResourceAdaptation() override;
// Uses a default AdaptReason of kCpu.
void AddResource(Resource* resource) override;
void AddResource(Resource* resource,
AdaptationObserverInterface::AdaptReason reason);
void SetHasInputVideo(bool has_input_video) override;
void SetDegradationPreference(
DegradationPreference degradation_preference) override;
@ -110,25 +114,13 @@ class OveruseFrameDetectorResourceAdaptationModule
ResourceListenerResponse OnResourceUsageStateMeasured(
const Resource& resource) override;
// Public versions of OnResourceUnderuse/OnResourceOveruse only used for
// testing.
// TODO(https://crbug.com/webrtc/11222): Control overuse/underuse from testing
// by injecting fake resources and remove these methods.
void OnResourceUnderuseForTesting(
AdaptationObserverInterface::AdaptReason reason);
// Returns false if OnResourceOveruse() returns
// ResourceListenerResponse::kQualityScalerShouldIncreaseFrequency.
// TODO(https://crbug.com/webrtc/11222): Get rid of the
// ResourceListenerResponse enum and the boolean return value of
// AdaptationObserverInterface::AdaptDown() that this method mimics.
bool OnResourceOveruseForTesting(
AdaptationObserverInterface::AdaptReason reason);
private:
class VideoSourceRestrictor;
class AdaptCounter;
class InitialFrameDropper;
enum class State { kStopped, kStarted };
struct AdaptationRequest {
// The pixel count produced by the source at the time of the adaptation.
int input_pixel_count_;
@ -192,6 +184,7 @@ class OveruseFrameDetectorResourceAdaptationModule
ResourceAdaptationModuleListener* const adaptation_listener_;
Clock* clock_;
State state_;
const bool experiment_cpu_load_estimator_;
// The restrictions that |adaptation_listener_| is informed of.
VideoSourceRestrictions video_source_restrictions_;
@ -222,6 +215,19 @@ class OveruseFrameDetectorResourceAdaptationModule
QualityRampupExperiment quality_rampup_experiment_;
absl::optional<EncoderSettings> encoder_settings_;
VideoStreamEncoderObserver* const encoder_stats_observer_;
// Ties a resource to a reason for statistical reporting. This AdaptReason is
// also used by this module to make decisions about how to adapt up/down.
struct ResourceAndReason {
ResourceAndReason(Resource* resource,
AdaptationObserverInterface::AdaptReason reason)
: resource(resource), reason(reason) {}
virtual ~ResourceAndReason() = default;
Resource* const resource;
const AdaptationObserverInterface::AdaptReason reason;
};
std::vector<ResourceAndReason> resources_;
};
} // namespace webrtc

3
video/quality_scaler_resource.h
View File

@ -12,6 +12,7 @@
#define VIDEO_QUALITY_SCALER_RESOURCE_H_
#include <memory>
#include <string>
#include "api/video_codecs/video_encoder.h"
#include "call/adaptation/resource.h"
@ -46,6 +47,8 @@ class QualityScalerResource : public Resource,
void AdaptUp(AdaptReason reason) override;
bool AdaptDown(AdaptReason reason) override;
std::string name() const override { return "QualityScalerResource"; }
private:
std::unique_ptr<QualityScaler> quality_scaler_;
};

17
video/video_stream_encoder.cc
View File

@ -1645,18 +1645,6 @@ bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const {
return false;
}
void VideoStreamEncoder::OnResourceUnderuseForTesting(
AdaptationObserverInterface::AdaptReason reason) {
RTC_DCHECK_RUN_ON(&encoder_queue_);
resource_adaptation_module_->OnResourceUnderuseForTesting(reason);
}
bool VideoStreamEncoder::OnResourceOveruseForTesting(
AdaptationObserverInterface::AdaptReason reason) {
RTC_DCHECK_RUN_ON(&encoder_queue_);
return resource_adaptation_module_->OnResourceOveruseForTesting(reason);
}
void VideoStreamEncoder::OnVideoSourceRestrictionsUpdated(
VideoSourceRestrictions restrictions) {
RTC_DCHECK_RUN_ON(&encoder_queue_);
@ -1926,5 +1914,10 @@ void VideoStreamEncoder::CheckForAnimatedContent(
video_source_sink_controller_->PushSourceSinkSettings();
}
}
void VideoStreamEncoder::InjectAdaptationResource(
Resource* resource,
AdaptationObserverInterface::AdaptReason reason) {
resource_adaptation_module_->AddResource(resource, reason);
}
} // namespace webrtc

14
video/video_stream_encoder.h
View File

@ -118,17 +118,15 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
// be called on |encoder_queue_|.
rtc::TaskQueue* encoder_queue() { return &encoder_queue_; }
// TODO(https://crbug.com/webrtc/11222): When the concept of "resources" that
// can be overused or underused has materialized, trigger overuse/underuse by
// injecting a fake Resource instead and remove these methods.
void OnResourceUnderuseForTesting(
AdaptationObserverInterface::AdaptReason reason);
bool OnResourceOveruseForTesting(
AdaptationObserverInterface::AdaptReason reason);
void OnVideoSourceRestrictionsUpdated(
VideoSourceRestrictions restrictions) override;
// Used for injected test resources.
// TODO(eshr): Move all adaptation tests out of VideoStreamEncoder tests.
void InjectAdaptationResource(Resource* resource,
AdaptationObserverInterface::AdaptReason reason)
RTC_RUN_ON(&encoder_queue_);
private:
class VideoFrameInfo {
public:

48
video/video_stream_encoder_unittest.cc
View File

@ -25,6 +25,7 @@
#include "api/video_codecs/video_encoder.h"
#include "api/video_codecs/vp8_temporal_layers.h"
#include "api/video_codecs/vp8_temporal_layers_factory.h"
#include "call/adaptation/test/fake_resource.h"
#include "common_video/h264/h264_common.h"
#include "common_video/include/video_frame_buffer.h"
#include "media/base/video_adapter.h"
@ -156,7 +157,19 @@ class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
std::unique_ptr<OveruseFrameDetector>(
overuse_detector_proxy_ =
new CpuOveruseDetectorProxy(stats_proxy)),
task_queue_factory) {}
task_queue_factory),
fake_cpu_resource_(
std::make_unique<FakeResource>(ResourceUsageState::kStable,
"FakeResource[CPU]")),
fake_quality_resource_(
std::make_unique<FakeResource>(ResourceUsageState::kStable,
"FakeResource[QP]")) {
InjectAdaptationResource(
fake_quality_resource_.get(),
AdaptationObserverInterface::AdaptReason::kQuality);
InjectAdaptationResource(fake_cpu_resource_.get(),
AdaptationObserverInterface::AdaptReason::kCpu);
}
void PostTaskAndWait(bool down,
AdaptationObserverInterface::AdaptReason reason) {
@ -168,10 +181,33 @@ class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
bool expected_results) {
rtc::Event event;
encoder_queue()->PostTask([this, &event, reason, down, expected_results] {
if (down)
EXPECT_EQ(expected_results, OnResourceOveruseForTesting(reason));
else
OnResourceUnderuseForTesting(reason);
ResourceUsageState usage_state =
down ? ResourceUsageState::kOveruse : ResourceUsageState::kUnderuse;
FakeResource* resource = nullptr;
switch (reason) {
case AdaptationObserverInterface::kQuality:
resource = fake_quality_resource_.get();
break;
case AdaptationObserverInterface::kCpu:
resource = fake_cpu_resource_.get();
break;
default:
RTC_NOTREACHED();
}
resource->set_usage_state(usage_state);
if (!expected_results) {
ASSERT_EQ(AdaptationObserverInterface::kQuality, reason)
<< "We can only assert adaptation result for quality resources";
EXPECT_EQ(
ResourceListenerResponse::kQualityScalerShouldIncreaseFrequency,
resource->last_response());
} else {
EXPECT_EQ(ResourceListenerResponse::kNothing,
resource->last_response());
}
event.Set();
});
ASSERT_TRUE(event.Wait(5000));
@ -212,6 +248,8 @@ class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
}
CpuOveruseDetectorProxy* overuse_detector_proxy_;
std::unique_ptr<FakeResource> fake_cpu_resource_;
std::unique_ptr<FakeResource> fake_quality_resource_;
};
class VideoStreamFactory