zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

VideoStreamEncoder: Introduce frame cadence adapter.

Browse Source 
This change introduces a new FrameCadenceAdapter class which takes the
role of being a VideoFrameSinkInterface<> instead of VideoStreamEncoder.
The FrameCadenceAdapter will see its functionality grow in future CLs
and eventually enable screenshare capture sources to have zero hertz as
the minimum capture frequency.

This CL moves logic related to UMA collection and constraints into the
adapter.

The adapter has two major modes. Future functionality is planned to be
added under the WebRTC-ZeroHertzScreenshare field trial. Unit tests are
added that verify passthrough operation when WebRTC-ZeroHertzScreenshare
isn't specified or disabled.

Just specifying the WebRTC-ZeroHertzScreenshare field trial isn't
enough to activate the feature, but the caller has to additionally
configure screen content type, minimum FPS 0, and maximum FPS > 0 for
the new mode.

go/rtc-0hz-present

Bug: chromium:1255737
Change-Id: I1799110ed40843152786ad80df10acfb83a608b1
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/236682
Commit-Queue: Markus Handell <handellm@webrtc.org>
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Stefan Holmer <stefan@webrtc.org>
Reviewed-by: Niels Moller <nisse@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#35315}
This commit is contained in:
Markus Handell
2021-11-05 12:00:55 +01:00
committed by WebRTC LUCI CQ
parent 0b5656312b
commit b4e96d48a2
10 changed files with 734 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
api/video/video_stream_encoder_interface.h
View File

@ -39,7 +39,7 @@ namespace webrtc {
//
// 2. Moving responsibility for simulcast and for software fallback into this
// class.
class VideoStreamEncoderInterface : public rtc::VideoSinkInterface<VideoFrame> {
class VideoStreamEncoderInterface {
public:
// Interface for receiving encoded video frames and notifications about
// configuration changes.
@ -58,6 +58,8 @@ class VideoStreamEncoderInterface : public rtc::VideoSinkInterface<VideoFrame> {
VideoLayersAllocation allocation) = 0;
};
virtual ~VideoStreamEncoderInterface() = default;
// If the resource is overusing, the VideoStreamEncoder will try to reduce
// resolution or frame rate until no resource is overusing.
// TODO(https://crbug.com/webrtc/11565): When the ResourceAdaptationProcessor

23
video/BUILD.gn
View File

@ -53,6 +53,7 @@ rtc_library("video") {
]
deps = [
":frame_cadence_adapter",
":frame_dumping_decoder",
":video_stream_encoder_impl",
"../api:array_view",
@ -257,6 +258,25 @@ rtc_library("frame_dumping_decoder") {
]
}
rtc_library("frame_cadence_adapter") {
visibility = [ "*" ]
sources = [
"frame_cadence_adapter.cc",
"frame_cadence_adapter.h",
]
deps = [
"../api:sequence_checker",
"../api/video:video_frame",
"../rtc_base:logging",
"../rtc_base:macromagic",
"../rtc_base:rtc_base_approved",
"../rtc_base/synchronization:mutex",
"../system_wrappers:field_trial",
"../system_wrappers:metrics",
]
}
rtc_library("video_stream_encoder_impl") {
visibility = [ "*" ]
@ -277,6 +297,7 @@ rtc_library("video_stream_encoder_impl") {
]
deps = [
":frame_cadence_adapter",
"../api:rtp_parameters",
"../api:sequence_checker",
"../api/adaptation:resource_adaptation_api",
@ -611,6 +632,7 @@ if (rtc_include_tests) {
"end_to_end_tests/ssrc_tests.cc",
"end_to_end_tests/stats_tests.cc",
"end_to_end_tests/transport_feedback_tests.cc",
"frame_cadence_adapter_unittest.cc",
"frame_encode_metadata_writer_unittest.cc",
"picture_id_tests.cc",
"quality_limitation_reason_tracker_unittest.cc",
@ -635,6 +657,7 @@ if (rtc_include_tests) {
"video_stream_encoder_unittest.cc",
]
deps = [
":frame_cadence_adapter",
":video",
":video_legacy",
":video_mocks",

164
video/frame_cadence_adapter.cc Normal file
View File

@ -0,0 +1,164 @@
/*
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "video/frame_cadence_adapter.h"
#include <memory>
#include "api/sequence_checker.h"
#include "rtc_base/logging.h"
#include "rtc_base/race_checker.h"
#include "rtc_base/synchronization/mutex.h"
#include "system_wrappers/include/field_trial.h"
#include "system_wrappers/include/metrics.h"
namespace webrtc {
namespace {
class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface {
public:
FrameCadenceAdapterImpl();
// FrameCadenceAdapterInterface overrides.
void Initialize(Callback* callback) override;
void SetZeroHertzModeEnabled(bool enabled) override;
// VideoFrameSink overrides.
void OnFrame(const VideoFrame& frame) override;
void OnDiscardedFrame() override { callback_->OnDiscardedFrame(); }
void OnConstraintsChanged(
const VideoTrackSourceConstraints& constraints) override;
private:
// Called to report on constraint UMAs.
void MaybeReportFrameRateConstraintUmas()
RTC_RUN_ON(&incoming_frame_race_checker_) RTC_LOCKS_EXCLUDED(mutex_);
// True if we support frame entry for screenshare with a minimum frequency of
// 0 Hz.
const bool zero_hertz_screenshare_enabled_;
// Set up during Initialize.
Callback* callback_ = nullptr;
// Lock protecting zero-hertz activation state. This is needed because the
// threading contexts of OnFrame, OnConstraintsChanged, and ConfigureEncoder
// are mutating it.
Mutex mutex_;
// The source's constraints.
absl::optional<VideoTrackSourceConstraints> source_constraints_
RTC_GUARDED_BY(mutex_);
// Whether zero-hertz and UMA reporting is enabled.
bool zero_hertz_and_uma_reporting_enabled_ RTC_GUARDED_BY(mutex_) = false;
// Race checker for incoming frames. This is the network thread in chromium,
// but may vary from test contexts.
rtc::RaceChecker incoming_frame_race_checker_;
bool has_reported_screenshare_frame_rate_umas_ RTC_GUARDED_BY(mutex_) = false;
};
FrameCadenceAdapterImpl::FrameCadenceAdapterImpl()
: zero_hertz_screenshare_enabled_(
field_trial::IsEnabled("WebRTC-ZeroHertzScreenshare")) {}
void FrameCadenceAdapterImpl::Initialize(Callback* callback) {
callback_ = callback;
}
void FrameCadenceAdapterImpl::SetZeroHertzModeEnabled(bool enabled) {
// This method is called on the worker thread.
MutexLock lock(&mutex_);
if (enabled && !zero_hertz_and_uma_reporting_enabled_)
has_reported_screenshare_frame_rate_umas_ = false;
zero_hertz_and_uma_reporting_enabled_ = enabled;
}
void FrameCadenceAdapterImpl::OnFrame(const VideoFrame& frame) {
// This method is called on the network thread under Chromium, or other
// various contexts in test.
RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_);
callback_->OnFrame(frame);
MaybeReportFrameRateConstraintUmas();
}
void FrameCadenceAdapterImpl::OnConstraintsChanged(
const VideoTrackSourceConstraints& constraints) {
RTC_LOG(LS_INFO) << __func__ << " min_fps "
<< constraints.min_fps.value_or(-1) << " max_fps "
<< constraints.max_fps.value_or(-1);
MutexLock lock(&mutex_);
source_constraints_ = constraints;
}
// RTC_RUN_ON(&incoming_frame_race_checker_)
void FrameCadenceAdapterImpl::MaybeReportFrameRateConstraintUmas() {
MutexLock lock(&mutex_);
if (has_reported_screenshare_frame_rate_umas_)
return;
has_reported_screenshare_frame_rate_umas_ = true;
if (!zero_hertz_and_uma_reporting_enabled_)
return;
RTC_HISTOGRAM_BOOLEAN("WebRTC.Screenshare.FrameRateConstraints.Exists",
source_constraints_.has_value());
if (!source_constraints_.has_value())
return;
RTC_HISTOGRAM_BOOLEAN("WebRTC.Screenshare.FrameRateConstraints.Min.Exists",
source_constraints_->min_fps.has_value());
if (source_constraints_->min_fps.has_value()) {
RTC_HISTOGRAM_COUNTS_100(
"WebRTC.Screenshare.FrameRateConstraints.Min.Value",
source_constraints_->min_fps.value());
}
RTC_HISTOGRAM_BOOLEAN("WebRTC.Screenshare.FrameRateConstraints.Max.Exists",
source_constraints_->max_fps.has_value());
if (source_constraints_->max_fps.has_value()) {
RTC_HISTOGRAM_COUNTS_100(
"WebRTC.Screenshare.FrameRateConstraints.Max.Value",
source_constraints_->max_fps.value());
}
if (!source_constraints_->min_fps.has_value()) {
if (source_constraints_->max_fps.has_value()) {
RTC_HISTOGRAM_COUNTS_100(
"WebRTC.Screenshare.FrameRateConstraints.MinUnset.Max",
source_constraints_->max_fps.value());
}
} else if (source_constraints_->max_fps.has_value()) {
if (source_constraints_->min_fps.value() <
source_constraints_->max_fps.value()) {
RTC_HISTOGRAM_COUNTS_100(
"WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Min",
source_constraints_->min_fps.value());
RTC_HISTOGRAM_COUNTS_100(
"WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Max",
source_constraints_->max_fps.value());
}
// Multi-dimensional histogram for min and max FPS making it possible to
// uncover min and max combinations. See
// https://chromium.googlesource.com/chromium/src.git/+/HEAD/tools/metrics/histograms/README.md#multidimensional-histograms
constexpr int kMaxBucketCount =
60 * /*max min_fps=*/60 + /*max max_fps=*/60 - 1;
RTC_HISTOGRAM_ENUMERATION_SPARSE(
"WebRTC.Screenshare.FrameRateConstraints.60MinPlusMaxMinusOne",
source_constraints_->min_fps.value() * 60 +
source_constraints_->max_fps.value() - 1,
/*boundary=*/kMaxBucketCount);
}
}
} // namespace
std::unique_ptr<FrameCadenceAdapterInterface>
FrameCadenceAdapterInterface::Create() {
return std::make_unique<FrameCadenceAdapterImpl>();
}
} // namespace webrtc

55
video/frame_cadence_adapter.h Normal file
View File

@ -0,0 +1,55 @@
/*
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef VIDEO_FRAME_CADENCE_ADAPTER_H_
#define VIDEO_FRAME_CADENCE_ADAPTER_H_
#include <memory>
#include "api/video/video_frame.h"
#include "api/video/video_sink_interface.h"
#include "rtc_base/synchronization/mutex.h"
#include "rtc_base/thread_annotations.h"
namespace webrtc {
// A sink adapter implementing mutations to the received frame cadence.
// With the exception of construction & destruction which has to happen on the
// same sequence, this class is thread-safe because three different execution
// contexts call into it.
class FrameCadenceAdapterInterface
: public rtc::VideoSinkInterface<VideoFrame> {
public:
// Callback interface used to inform instance owners.
class Callback {
public:
virtual ~Callback() = default;
// Called when a frame arrives.
virtual void OnFrame(const VideoFrame& frame) = 0;
// Called when the source has discarded a frame.
virtual void OnDiscardedFrame() = 0;
};
// Factory function creating a production instance. Deletion of the returned
// instance needs to happen on the same sequence that Create() was called on.
static std::unique_ptr<FrameCadenceAdapterInterface> Create();
// Call before using the rest of the API.
virtual void Initialize(Callback* callback) = 0;
// Pass true in |enabled| as a prerequisite to enable zero-hertz operation.
virtual void SetZeroHertzModeEnabled(bool enabled) = 0;
};
} // namespace webrtc
#endif // VIDEO_FRAME_CADENCE_ADAPTER_H_

328
video/frame_cadence_adapter_unittest.cc Normal file
View File

@ -0,0 +1,328 @@
/*
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "video/frame_cadence_adapter.h"
#include <utility>
#include <vector>
#include "api/video/nv12_buffer.h"
#include "api/video/video_frame.h"
#include "rtc_base/ref_counted_object.h"
#include "system_wrappers/include/metrics.h"
#include "test/field_trial.h"
#include "test/gmock.h"
#include "test/gtest.h"
namespace webrtc {
namespace {
using ::testing::ElementsAre;
using ::testing::Mock;
using ::testing::Pair;
using ::testing::Ref;
using ::testing::UnorderedElementsAre;
VideoFrame CreateFrame() {
return VideoFrame::Builder()
.set_video_frame_buffer(
rtc::make_ref_counted<NV12Buffer>(/*width=*/16, /*height=*/16))
.build();
}
class MockCallback : public FrameCadenceAdapterInterface::Callback {
public:
MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
MOCK_METHOD(void, OnDiscardedFrame, (), (override));
};
class ZeroHertzFieldTrialDisabler : public test::ScopedFieldTrials {
public:
ZeroHertzFieldTrialDisabler()
: test::ScopedFieldTrials("WebRTC-ZeroHertzScreenshare/Disabled/") {}
};
TEST(FrameCadenceAdapterTest,
ForwardsFramesOnConstructionAndUnderDisabledFieldTrial) {
auto disabler = std::make_unique<ZeroHertzFieldTrialDisabler>();
for (int i = 0; i != 2; i++) {
MockCallback callback;
auto adapter = FrameCadenceAdapterInterface::Create();
adapter->Initialize(&callback);
VideoFrame frame = CreateFrame();
EXPECT_CALL(callback, OnFrame(Ref(frame))).Times(1);
adapter->OnFrame(frame);
Mock::VerifyAndClearExpectations(&callback);
EXPECT_CALL(callback, OnDiscardedFrame).Times(1);
adapter->OnDiscardedFrame();
Mock::VerifyAndClearExpectations(&callback);
disabler = nullptr;
}
}
class FrameCadenceAdapterMetricsTest : public ::testing::Test {
public:
FrameCadenceAdapterMetricsTest() { metrics::Reset(); }
};
TEST_F(FrameCadenceAdapterMetricsTest, RecordsNoUmasWithNoFrameTransfer) {
MockCallback callback;
auto adapter = FrameCadenceAdapterInterface::Create();
adapter->Initialize(&callback);
adapter->OnConstraintsChanged(
VideoTrackSourceConstraints{absl::nullopt, absl::nullopt});
adapter->OnConstraintsChanged(VideoTrackSourceConstraints{absl::nullopt, 1});
adapter->OnConstraintsChanged(VideoTrackSourceConstraints{2, 3});
adapter->OnConstraintsChanged(VideoTrackSourceConstraints{4, 4});
adapter->OnConstraintsChanged(VideoTrackSourceConstraints{5, absl::nullopt});
EXPECT_TRUE(metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Exists")
.empty());
EXPECT_TRUE(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Exists")
.empty());
EXPECT_TRUE(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Value")
.empty());
EXPECT_TRUE(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Exists")
.empty());
EXPECT_TRUE(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Value")
.empty());
EXPECT_TRUE(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.MinUnset.Max")
.empty());
EXPECT_TRUE(metrics::Samples(
"WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Min")
.empty());
EXPECT_TRUE(metrics::Samples(
"WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Max")
.empty());
EXPECT_TRUE(
metrics::Samples(
"WebRTC.Screenshare.FrameRateConstraints.60MinPlusMaxMinusOne")
.empty());
}
TEST_F(FrameCadenceAdapterMetricsTest, RecordsNoUmasWithoutEnabledContentType) {
MockCallback callback;
auto adapter = FrameCadenceAdapterInterface::Create();
adapter->Initialize(&callback);
adapter->OnFrame(CreateFrame());
adapter->OnConstraintsChanged(
VideoTrackSourceConstraints{absl::nullopt, absl::nullopt});
adapter->OnConstraintsChanged(VideoTrackSourceConstraints{absl::nullopt, 1});
adapter->OnConstraintsChanged(VideoTrackSourceConstraints{2, 3});
adapter->OnConstraintsChanged(VideoTrackSourceConstraints{4, 4});
adapter->OnConstraintsChanged(VideoTrackSourceConstraints{5, absl::nullopt});
EXPECT_TRUE(metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Exists")
.empty());
EXPECT_TRUE(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Exists")
.empty());
EXPECT_TRUE(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Value")
.empty());
EXPECT_TRUE(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Exists")
.empty());
EXPECT_TRUE(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Value")
.empty());
EXPECT_TRUE(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.MinUnset.Max")
.empty());
EXPECT_TRUE(metrics::Samples(
"WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Min")
.empty());
EXPECT_TRUE(metrics::Samples(
"WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Max")
.empty());
EXPECT_TRUE(
metrics::Samples(
"WebRTC.Screenshare.FrameRateConstraints.60MinPlusMaxMinusOne")
.empty());
}
TEST_F(FrameCadenceAdapterMetricsTest, RecordsNoConstraintsIfUnsetOnFrame) {
MockCallback callback;
auto adapter = FrameCadenceAdapterInterface::Create();
adapter->Initialize(&callback);
adapter->SetZeroHertzModeEnabled(true);
adapter->OnFrame(CreateFrame());
EXPECT_THAT(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Exists"),
ElementsAre(Pair(false, 1)));
}
TEST_F(FrameCadenceAdapterMetricsTest, RecordsEmptyConstraintsIfSetOnFrame) {
MockCallback callback;
auto adapter = FrameCadenceAdapterInterface::Create();
adapter->Initialize(&callback);
adapter->SetZeroHertzModeEnabled(true);
adapter->OnConstraintsChanged(
VideoTrackSourceConstraints{absl::nullopt, absl::nullopt});
adapter->OnFrame(CreateFrame());
EXPECT_THAT(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Exists"),
ElementsAre(Pair(true, 1)));
EXPECT_THAT(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Exists"),
ElementsAre(Pair(false, 1)));
EXPECT_TRUE(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Value")
.empty());
EXPECT_THAT(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Exists"),
ElementsAre(Pair(false, 1)));
EXPECT_TRUE(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Value")
.empty());
EXPECT_TRUE(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.MinUnset.Max")
.empty());
EXPECT_TRUE(metrics::Samples(
"WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Min")
.empty());
EXPECT_TRUE(metrics::Samples(
"WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Max")
.empty());
EXPECT_TRUE(
metrics::Samples(
"WebRTC.Screenshare.FrameRateConstraints.60MinPlusMaxMinusOne")
.empty());
}
TEST_F(FrameCadenceAdapterMetricsTest, RecordsMaxConstraintIfSetOnFrame) {
MockCallback callback;
auto adapter = FrameCadenceAdapterInterface::Create();
adapter->Initialize(&callback);
adapter->SetZeroHertzModeEnabled(true);
adapter->OnConstraintsChanged(
VideoTrackSourceConstraints{absl::nullopt, 2.0});
adapter->OnFrame(CreateFrame());
EXPECT_THAT(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Exists"),
ElementsAre(Pair(false, 1)));
EXPECT_TRUE(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Value")
.empty());
EXPECT_THAT(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Exists"),
ElementsAre(Pair(true, 1)));
EXPECT_THAT(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Value"),
ElementsAre(Pair(2.0, 1)));
EXPECT_THAT(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.MinUnset.Max"),
ElementsAre(Pair(2.0, 1)));
EXPECT_TRUE(metrics::Samples(
"WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Min")
.empty());
EXPECT_TRUE(metrics::Samples(
"WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Max")
.empty());
EXPECT_TRUE(
metrics::Samples(
"WebRTC.Screenshare.FrameRateConstraints.60MinPlusMaxMinusOne")
.empty());
}
TEST_F(FrameCadenceAdapterMetricsTest, RecordsMinConstraintIfSetOnFrame) {
MockCallback callback;
auto adapter = FrameCadenceAdapterInterface::Create();
adapter->Initialize(&callback);
adapter->SetZeroHertzModeEnabled(true);
adapter->OnConstraintsChanged(
VideoTrackSourceConstraints{3.0, absl::nullopt});
adapter->OnFrame(CreateFrame());
EXPECT_THAT(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Exists"),
ElementsAre(Pair(true, 1)));
EXPECT_THAT(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Value"),
ElementsAre(Pair(3.0, 1)));
EXPECT_THAT(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Exists"),
ElementsAre(Pair(false, 1)));
EXPECT_TRUE(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Value")
.empty());
EXPECT_TRUE(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.MinUnset.Max")
.empty());
EXPECT_TRUE(metrics::Samples(
"WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Min")
.empty());
EXPECT_TRUE(metrics::Samples(
"WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Max")
.empty());
EXPECT_TRUE(
metrics::Samples(
"WebRTC.Screenshare.FrameRateConstraints.60MinPlusMaxMinusOne")
.empty());
}
TEST_F(FrameCadenceAdapterMetricsTest, RecordsMinGtMaxConstraintIfSetOnFrame) {
MockCallback callback;
auto adapter = FrameCadenceAdapterInterface::Create();
adapter->Initialize(&callback);
adapter->SetZeroHertzModeEnabled(true);
adapter->OnConstraintsChanged(VideoTrackSourceConstraints{5.0, 4.0});
adapter->OnFrame(CreateFrame());
EXPECT_THAT(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Exists"),
ElementsAre(Pair(true, 1)));
EXPECT_THAT(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Value"),
ElementsAre(Pair(5.0, 1)));
EXPECT_THAT(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Exists"),
ElementsAre(Pair(true, 1)));
EXPECT_THAT(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Value"),
ElementsAre(Pair(4.0, 1)));
EXPECT_TRUE(
metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.MinUnset.Max")
.empty());
EXPECT_TRUE(metrics::Samples(
"WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Min")
.empty());
EXPECT_TRUE(metrics::Samples(
"WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Max")
.empty());
EXPECT_THAT(
metrics::Samples(
"WebRTC.Screenshare.FrameRateConstraints.60MinPlusMaxMinusOne"),
ElementsAre(Pair(60 * 5.0 + 4.0 - 1, 1)));
}
TEST_F(FrameCadenceAdapterMetricsTest, RecordsMinLtMaxConstraintIfSetOnFrame) {
MockCallback callback;
auto adapter = FrameCadenceAdapterInterface::Create();
adapter->Initialize(&callback);
adapter->SetZeroHertzModeEnabled(true);
adapter->OnConstraintsChanged(VideoTrackSourceConstraints{4.0, 5.0});
adapter->OnFrame(CreateFrame());
EXPECT_THAT(metrics::Samples(
"WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Min"),
ElementsAre(Pair(4.0, 1)));
EXPECT_THAT(metrics::Samples(
"WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Max"),
ElementsAre(Pair(5.0, 1)));
EXPECT_THAT(
metrics::Samples(
"WebRTC.Screenshare.FrameRateConstraints.60MinPlusMaxMinusOne"),
ElementsAre(Pair(60 * 4.0 + 5.0 - 1, 1)));
}
} // namespace
} // namespace webrtc

1
video/test/mock_video_stream_encoder.h
View File

@ -43,7 +43,6 @@ class MockVideoStreamEncoder : public VideoStreamEncoderInterface {
OnBitrateUpdated,
(DataRate, DataRate, DataRate, uint8_t, int64_t, double),
(override));
MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
MOCK_METHOD(void,
SetFecControllerOverride,
(FecControllerOverride*),

2
video/video_send_stream.cc
View File

@ -23,6 +23,7 @@
#include "system_wrappers/include/clock.h"
#include "system_wrappers/include/field_trial.h"
#include "video/adaptation/overuse_frame_detector.h"
#include "video/frame_cadence_adapter.h"
#include "video/video_stream_encoder.h"
namespace webrtc {
@ -135,6 +136,7 @@ VideoSendStream::VideoSendStream(
&stats_proxy_,
config_.encoder_settings,
std::make_unique<OveruseFrameDetector>(&stats_proxy_),
FrameCadenceAdapterInterface::Create(),
task_queue_factory,
network_queue,
GetBitrateAllocationCallbackType(config_))),

81
video/video_stream_encoder.cc
View File

@ -592,6 +592,7 @@ VideoStreamEncoder::VideoStreamEncoder(
VideoStreamEncoderObserver* encoder_stats_observer,
const VideoStreamEncoderSettings& settings,
std::unique_ptr<OveruseFrameDetector> overuse_detector,
std::unique_ptr<FrameCadenceAdapterInterface> frame_cadence_adapter,
TaskQueueFactory* task_queue_factory,
TaskQueueBase* network_queue,
BitrateAllocationCallbackType allocation_cb_type)
@ -604,6 +605,8 @@ VideoStreamEncoder::VideoStreamEncoder(
rate_control_settings_(RateControlSettings::ParseFromFieldTrials()),
encoder_selector_(settings.encoder_factory->GetEncoderSelector()),
encoder_stats_observer_(encoder_stats_observer),
cadence_callback_(*this),
frame_cadence_adapter_(std::move(frame_cadence_adapter)),
encoder_initialized_(false),
max_framerate_(-1),
pending_encoder_reconfiguration_(false),
@ -657,7 +660,7 @@ VideoStreamEncoder::VideoStreamEncoder(
settings_.experiment_cpu_load_estimator,
std::move(overuse_detector),
degradation_preference_manager_.get()),
video_source_sink_controller_(/*sink=*/this,
video_source_sink_controller_(/*sink=*/frame_cadence_adapter_.get(),
/*source=*/nullptr),
default_limits_allowed_(
!field_trial::IsEnabled("WebRTC-DefaultBitrateLimitsKillSwitch")),
@ -671,6 +674,7 @@ VideoStreamEncoder::VideoStreamEncoder(
RTC_DCHECK(encoder_stats_observer);
RTC_DCHECK_GE(number_of_cores, 1);
frame_cadence_adapter_->Initialize(&cadence_callback_);
stream_resource_manager_.Initialize(&encoder_queue_);
rtc::Event initialize_processor_event;
@ -821,6 +825,8 @@ void VideoStreamEncoder::SetStartBitrate(int start_bitrate_bps) {
void VideoStreamEncoder::ConfigureEncoder(VideoEncoderConfig config,
size_t max_data_payload_length) {
RTC_DCHECK_RUN_ON(worker_queue_);
frame_cadence_adapter_->SetZeroHertzModeEnabled(
config.content_type == VideoEncoderConfig::ContentType::kScreen);
encoder_queue_.PostTask(
[this, config = std::move(config), max_data_payload_length]() mutable {
RTC_DCHECK_RUN_ON(&encoder_queue_);
@ -830,8 +836,6 @@ void VideoStreamEncoder::ConfigureEncoder(VideoEncoderConfig config,
pending_encoder_creation_ =
(!encoder_ || encoder_config_.video_format != config.video_format ||
max_data_payload_length_ != max_data_payload_length);
if (encoder_config_.content_type != config.content_type)
has_reported_screenshare_frame_rate_umas_ = false;
encoder_config_ = std::move(config);
max_data_payload_length_ = max_data_payload_length;
pending_encoder_reconfiguration_ = true;
@ -1331,7 +1335,6 @@ void VideoStreamEncoder::OnFrame(const VideoFrame& video_frame) {
encoder_queue_.PostTask(
[this, incoming_frame, post_time_us, log_stats]() {
RTC_DCHECK_RUN_ON(&encoder_queue_);
MaybeReportFrameRateConstraintUmas();
encoder_stats_observer_->OnIncomingFrame(incoming_frame.width(),
incoming_frame.height());
++captured_frame_count_;
@ -1382,18 +1385,6 @@ void VideoStreamEncoder::OnDiscardedFrame() {
VideoStreamEncoderObserver::DropReason::kSource);
}
void VideoStreamEncoder::OnConstraintsChanged(
const webrtc::VideoTrackSourceConstraints& constraints) {
RTC_DCHECK_RUN_ON(network_queue_);
RTC_LOG(LS_INFO) << __func__ << " min_fps "
<< constraints.min_fps.value_or(-1) << " max_fps "
<< constraints.max_fps.value_or(-1);
worker_queue_->PostTask(ToQueuedTask(task_safety_, [this, constraints] {
RTC_DCHECK_RUN_ON(worker_queue_);
source_constraints_ = constraints;
}));
}
bool VideoStreamEncoder::EncoderPaused() const {
RTC_DCHECK_RUN_ON(&encoder_queue_);
// Pause video if paused by caller or as long as the network is down or the
@ -2364,64 +2355,6 @@ void VideoStreamEncoder::QueueRequestEncoderSwitch(
}));
}
// RTC_RUN_ON(&encoder_queue_)
void VideoStreamEncoder::MaybeReportFrameRateConstraintUmas() {
if (has_reported_screenshare_frame_rate_umas_)
return;
has_reported_screenshare_frame_rate_umas_ = true;
bool is_screenshare =
encoder_config_.content_type == VideoEncoderConfig::ContentType::kScreen;
if (!is_screenshare)
return;
worker_queue_->PostTask(ToQueuedTask(task_safety_, [this] {
RTC_DCHECK_RUN_ON(worker_queue_);
RTC_HISTOGRAM_BOOLEAN("WebRTC.Screenshare.FrameRateConstraints.Exists",
source_constraints_.has_value());
if (source_constraints_.has_value()) {
RTC_HISTOGRAM_BOOLEAN(
"WebRTC.Screenshare.FrameRateConstraints.Min.Exists",
source_constraints_->min_fps.has_value());
if (source_constraints_->min_fps.has_value()) {
RTC_HISTOGRAM_COUNTS_100(
"WebRTC.Screenshare.FrameRateConstraints.Min.Value",
source_constraints_->min_fps.value());
}
RTC_HISTOGRAM_BOOLEAN(
"WebRTC.Screenshare.FrameRateConstraints.Max.Exists",
source_constraints_->max_fps.has_value());
if (source_constraints_->max_fps.has_value()) {
RTC_HISTOGRAM_COUNTS_100(
"WebRTC.Screenshare.FrameRateConstraints.Max.Value",
source_constraints_->max_fps.value());
}
if (!source_constraints_->min_fps.has_value()) {
if (source_constraints_->max_fps.has_value()) {
RTC_HISTOGRAM_COUNTS_100(
"WebRTC.Screenshare.FrameRateConstraints.MinUnset.Max",
source_constraints_->max_fps.value());
}
} else if (source_constraints_->max_fps.has_value()) {
if (source_constraints_->min_fps.value() <
source_constraints_->max_fps.value()) {
RTC_HISTOGRAM_COUNTS_100(
"WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Min",
source_constraints_->min_fps.value());
RTC_HISTOGRAM_COUNTS_100(
"WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Max",
source_constraints_->max_fps.value());
}
constexpr int kMaxBucketCount =
60 * /*max min_fps=*/60 + /*max max_fps=*/60 - 1;
RTC_HISTOGRAM_ENUMERATION_SPARSE(
"WebRTC.Screenshare.FrameRateConstraints.60MinPlusMaxMinusOne",
source_constraints_->min_fps.value() * 60 +
source_constraints_->max_fps.value() - 1,
/*boundary=*/kMaxBucketCount);
}
}
}));
}
void VideoStreamEncoder::InjectAdaptationResource(
rtc::scoped_refptr<Resource> resource,
VideoAdaptationReason reason) {

58
video/video_stream_encoder.h
View File

@ -45,6 +45,7 @@
#include "system_wrappers/include/clock.h"
#include "video/adaptation/video_stream_encoder_resource_manager.h"
#include "video/encoder_bitrate_adjuster.h"
#include "video/frame_cadence_adapter.h"
#include "video/frame_encode_metadata_writer.h"
#include "video/video_source_sink_controller.h"
@ -69,14 +70,16 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
kVideoBitrateAllocationWhenScreenSharing,
kVideoLayersAllocation
};
VideoStreamEncoder(Clock* clock,
uint32_t number_of_cores,
VideoStreamEncoderObserver* encoder_stats_observer,
const VideoStreamEncoderSettings& settings,
std::unique_ptr<OveruseFrameDetector> overuse_detector,
TaskQueueFactory* task_queue_factory,
TaskQueueBase* network_queue,
BitrateAllocationCallbackType allocation_cb_type);
VideoStreamEncoder(
Clock* clock,
uint32_t number_of_cores,
VideoStreamEncoderObserver* encoder_stats_observer,
const VideoStreamEncoderSettings& settings,
std::unique_ptr<OveruseFrameDetector> overuse_detector,
std::unique_ptr<FrameCadenceAdapterInterface> frame_cadence_adapter,
TaskQueueFactory* task_queue_factory,
TaskQueueBase* network_queue,
BitrateAllocationCallbackType allocation_cb_type);
~VideoStreamEncoder() override;
void AddAdaptationResource(rtc::scoped_refptr<Resource> resource) override;
@ -138,6 +141,22 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
VideoSourceRestrictionsListener* restrictions_listener);
private:
class CadenceCallback : public FrameCadenceAdapterInterface::Callback {
public:
explicit CadenceCallback(VideoStreamEncoder& video_stream_encoder)
: video_stream_encoder_(video_stream_encoder) {}
// FrameCadenceAdapterInterface::Callback overrides.
void OnFrame(const VideoFrame& frame) override {
video_stream_encoder_.OnFrame(frame);
}
void OnDiscardedFrame() override {
video_stream_encoder_.OnDiscardedFrame();
}
private:
VideoStreamEncoder& video_stream_encoder_;
};
class VideoFrameInfo {
public:
VideoFrameInfo(int width, int height, bool is_texture)
@ -173,12 +192,8 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
void ReconfigureEncoder() RTC_RUN_ON(&encoder_queue_);
void OnEncoderSettingsChanged() RTC_RUN_ON(&encoder_queue_);
// Implements VideoSinkInterface.
void OnFrame(const VideoFrame& video_frame) override;
void OnDiscardedFrame() override;
void OnConstraintsChanged(
const webrtc::VideoTrackSourceConstraints& constraints) override;
void OnFrame(const VideoFrame& video_frame);
void OnDiscardedFrame();
void MaybeEncodeVideoFrame(const VideoFrame& frame,
int64_t time_when_posted_in_ms);
@ -229,9 +244,6 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
void QueueRequestEncoderSwitch(const webrtc::SdpVideoFormat& format)
RTC_RUN_ON(&encoder_queue_);
// Reports UMAs on frame rate constraints usage on the first call.
void MaybeReportFrameRateConstraintUmas() RTC_RUN_ON(&encoder_queue_);
TaskQueueBase* const worker_queue_;
TaskQueueBase* const network_queue_;
@ -245,12 +257,12 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
std::unique_ptr<VideoEncoderFactory::EncoderSelectorInterface> const
encoder_selector_;
VideoStreamEncoderObserver* const encoder_stats_observer_;
// The source's constraints.
absl::optional<VideoTrackSourceConstraints> source_constraints_
RTC_GUARDED_BY(worker_queue_);
bool has_reported_screenshare_frame_rate_umas_
RTC_GUARDED_BY(&encoder_queue_) = false;
// Adapter that avoids public inheritance of the cadence adapter's callback
// interface.
CadenceCallback cadence_callback_;
// Frame cadence encoder adapter. Frames enter this adapter first, and it then
// forwards them to our OnFrame method.
const std::unique_ptr<FrameCadenceAdapterInterface> frame_cadence_adapter_;
VideoEncoderConfig encoder_config_ RTC_GUARDED_BY(&encoder_queue_);
std::unique_ptr<VideoEncoder> encoder_ RTC_GUARDED_BY(&encoder_queue_)

117
video/video_stream_encoder_unittest.cc
View File

@ -18,6 +18,7 @@
#include "absl/memory/memory.h"
#include "api/task_queue/default_task_queue_factory.h"
#include "api/task_queue/task_queue_factory.h"
#include "api/test/mock_fec_controller_override.h"
#include "api/test/mock_video_encoder.h"
#include "api/test/mock_video_encoder_factory.h"
@ -62,6 +63,7 @@
#include "test/mappable_native_buffer.h"
#include "test/time_controller/simulated_time_controller.h"
#include "test/video_encoder_proxy_factory.h"
#include "video/frame_cadence_adapter.h"
#include "video/send_statistics_proxy.h"
namespace webrtc {
@ -76,7 +78,9 @@ using ::testing::Gt;
using ::testing::Le;
using ::testing::Lt;
using ::testing::Matcher;
using ::testing::Mock;
using ::testing::NiceMock;
using ::testing::Optional;
using ::testing::Return;
using ::testing::SizeIs;
using ::testing::StrictMock;
@ -355,6 +359,7 @@ class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
std::unique_ptr<OveruseFrameDetector>(
overuse_detector_proxy_ =
new CpuOveruseDetectorProxy(stats_proxy)),
FrameCadenceAdapterInterface::Create(),
task_queue_factory,
TaskQueueBase::Current(),
allocation_callback_type),
@ -626,6 +631,79 @@ class MockableSendStatisticsProxy : public SendStatisticsProxy {
std::function<void(DropReason)> on_frame_dropped_;
};
class SimpleVideoStreamEncoderFactory {
public:
class AdaptedVideoStreamEncoder : public VideoStreamEncoder {
public:
using VideoStreamEncoder::VideoStreamEncoder;
~AdaptedVideoStreamEncoder() { Stop(); }
};
SimpleVideoStreamEncoderFactory()
: time_controller_(Timestamp::Millis(0)),
task_queue_factory_(time_controller_.CreateTaskQueueFactory()),
stats_proxy_(std::make_unique<MockableSendStatisticsProxy>(
time_controller_.GetClock(),
VideoSendStream::Config(nullptr),
webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
encoder_settings_(
VideoEncoder::Capabilities(/*loss_notification=*/false)),
fake_encoder_(time_controller_.GetClock()),
encoder_factory_(&fake_encoder_) {
encoder_settings_.encoder_factory = &encoder_factory_;
}
std::unique_ptr<AdaptedVideoStreamEncoder> Create(
std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter) {
auto result = std::make_unique<AdaptedVideoStreamEncoder>(
time_controller_.GetClock(),
/*number_of_cores=*/1,
/*stats_proxy=*/stats_proxy_.get(), encoder_settings_,
std::make_unique<CpuOveruseDetectorProxy>(/*stats_proxy=*/nullptr),
std::move(zero_hertz_adapter), task_queue_factory_.get(),
TaskQueueBase::Current(),
VideoStreamEncoder::BitrateAllocationCallbackType::
kVideoBitrateAllocation);
result->SetSink(&sink_, /*rotation_applied=*/false);
return result;
}
private:
class NullEncoderSink : public VideoStreamEncoderInterface::EncoderSink {
public:
~NullEncoderSink() override = default;
void OnEncoderConfigurationChanged(
std::vector<VideoStream> streams,
bool is_svc,
VideoEncoderConfig::ContentType content_type,
int min_transmit_bitrate_bps) override {}
void OnBitrateAllocationUpdated(
const VideoBitrateAllocation& allocation) override {}
void OnVideoLayersAllocationUpdated(
VideoLayersAllocation allocation) override {}
Result OnEncodedImage(
const EncodedImage& encoded_image,
const CodecSpecificInfo* codec_specific_info) override {
return Result(EncodedImageCallback::Result::OK);
}
};
GlobalSimulatedTimeController time_controller_;
std::unique_ptr<TaskQueueFactory> task_queue_factory_;
std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
VideoStreamEncoderSettings encoder_settings_;
test::FakeEncoder fake_encoder_;
test::VideoEncoderProxyFactory encoder_factory_;
NullEncoderSink sink_;
};
class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface {
public:
MOCK_METHOD(void, Initialize, (Callback * callback), (override));
MOCK_METHOD(void, SetZeroHertzModeEnabled, (bool), (override));
MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
};
class MockEncoderSelector
: public VideoEncoderFactory::EncoderSelectorInterface {
public:
@ -8709,4 +8787,43 @@ TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) {
RunTest({config1, config2}, /*expected_num_init_encode=*/2);
}
TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) {
auto adapter = std::make_unique<MockFrameCadenceAdapter>();
auto* adapter_ptr = adapter.get();
SimpleVideoStreamEncoderFactory factory;
auto video_stream_encoder = factory.Create(std::move(adapter));
EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(true));
VideoEncoderConfig config;
config.content_type = VideoEncoderConfig::ContentType::kScreen;
video_stream_encoder->ConfigureEncoder(std::move(config), 0);
Mock::VerifyAndClearExpectations(adapter_ptr);
EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(false));
VideoEncoderConfig config2;
config2.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo;
video_stream_encoder->ConfigureEncoder(std::move(config2), 0);
}
TEST(VideoStreamEncoderFrameCadenceTest,
ForwardsFramesIntoFrameCadenceAdapter) {
auto adapter = std::make_unique<MockFrameCadenceAdapter>();
auto* adapter_ptr = adapter.get();
test::FrameForwarder video_source;
SimpleVideoStreamEncoderFactory factory;
auto video_stream_encoder = factory.Create(std::move(adapter));
video_stream_encoder->SetSource(
&video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
EXPECT_CALL(*adapter_ptr, OnFrame);
auto buffer = rtc::make_ref_counted<NV12Buffer>(/*width=*/16, /*height=*/16);
video_source.IncomingCapturedFrame(
VideoFrame::Builder()
.set_video_frame_buffer(std::move(buffer))
.set_ntp_time_ms(0)
.set_timestamp_ms(0)
.set_rotation(kVideoRotation_0)
.build());
}
} // namespace webrtc