From b4e96d48a20f88447fbd5fb8e0f382fb8f483eda Mon Sep 17 00:00:00 2001 From: Markus Handell Date: Fri, 5 Nov 2021 12:00:55 +0100 Subject: [PATCH] VideoStreamEncoder: Introduce frame cadence adapter. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change introduces a new FrameCadenceAdapter class which takes the role of being a VideoFrameSinkInterface<> instead of VideoStreamEncoder. The FrameCadenceAdapter will see its functionality grow in future CLs and eventually enable screenshare capture sources to have zero hertz as the minimum capture frequency. This CL moves logic related to UMA collection and constraints into the adapter. The adapter has two major modes. Future functionality is planned to be added under the WebRTC-ZeroHertzScreenshare field trial. Unit tests are added that verify passthrough operation when WebRTC-ZeroHertzScreenshare isn't specified or disabled. Just specifying the WebRTC-ZeroHertzScreenshare field trial isn't enough to activate the feature, but the caller has to additionally configure screen content type, minimum FPS 0, and maximum FPS > 0 for the new mode. go/rtc-0hz-present Bug: chromium:1255737 Change-Id: I1799110ed40843152786ad80df10acfb83a608b1 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/236682 Commit-Queue: Markus Handell Reviewed-by: Henrik Boström Reviewed-by: Ilya Nikolaevskiy Reviewed-by: Stefan Holmer Reviewed-by: Niels Moller Cr-Commit-Position: refs/heads/main@{#35315} --- api/video/video_stream_encoder_interface.h | 4 +- video/BUILD.gn | 23 ++ video/frame_cadence_adapter.cc | 164 +++++++++++ video/frame_cadence_adapter.h | 55 ++++ video/frame_cadence_adapter_unittest.cc | 328 +++++++++++++++++++++ video/test/mock_video_stream_encoder.h | 1 - video/video_send_stream.cc | 2 + video/video_stream_encoder.cc | 81 +---- video/video_stream_encoder.h | 58 ++-- video/video_stream_encoder_unittest.cc | 117 ++++++++ 10 files changed, 734 insertions(+), 99 deletions(-) create mode 100644 video/frame_cadence_adapter.cc create mode 100644 video/frame_cadence_adapter.h create mode 100644 video/frame_cadence_adapter_unittest.cc diff --git a/api/video/video_stream_encoder_interface.h b/api/video/video_stream_encoder_interface.h index 69d0ad2363..f2d7e131e6 100644 --- a/api/video/video_stream_encoder_interface.h +++ b/api/video/video_stream_encoder_interface.h @@ -39,7 +39,7 @@ namespace webrtc { // // 2. Moving responsibility for simulcast and for software fallback into this // class. -class VideoStreamEncoderInterface : public rtc::VideoSinkInterface { +class VideoStreamEncoderInterface { public: // Interface for receiving encoded video frames and notifications about // configuration changes. @@ -58,6 +58,8 @@ class VideoStreamEncoderInterface : public rtc::VideoSinkInterface { VideoLayersAllocation allocation) = 0; }; + virtual ~VideoStreamEncoderInterface() = default; + // If the resource is overusing, the VideoStreamEncoder will try to reduce // resolution or frame rate until no resource is overusing. // TODO(https://crbug.com/webrtc/11565): When the ResourceAdaptationProcessor diff --git a/video/BUILD.gn b/video/BUILD.gn index e8cd0cfede..c108e0a0ea 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -53,6 +53,7 @@ rtc_library("video") { ] deps = [ + ":frame_cadence_adapter", ":frame_dumping_decoder", ":video_stream_encoder_impl", "../api:array_view", @@ -257,6 +258,25 @@ rtc_library("frame_dumping_decoder") { ] } +rtc_library("frame_cadence_adapter") { + visibility = [ "*" ] + sources = [ + "frame_cadence_adapter.cc", + "frame_cadence_adapter.h", + ] + + deps = [ + "../api:sequence_checker", + "../api/video:video_frame", + "../rtc_base:logging", + "../rtc_base:macromagic", + "../rtc_base:rtc_base_approved", + "../rtc_base/synchronization:mutex", + "../system_wrappers:field_trial", + "../system_wrappers:metrics", + ] +} + rtc_library("video_stream_encoder_impl") { visibility = [ "*" ] @@ -277,6 +297,7 @@ rtc_library("video_stream_encoder_impl") { ] deps = [ + ":frame_cadence_adapter", "../api:rtp_parameters", "../api:sequence_checker", "../api/adaptation:resource_adaptation_api", @@ -611,6 +632,7 @@ if (rtc_include_tests) { "end_to_end_tests/ssrc_tests.cc", "end_to_end_tests/stats_tests.cc", "end_to_end_tests/transport_feedback_tests.cc", + "frame_cadence_adapter_unittest.cc", "frame_encode_metadata_writer_unittest.cc", "picture_id_tests.cc", "quality_limitation_reason_tracker_unittest.cc", @@ -635,6 +657,7 @@ if (rtc_include_tests) { "video_stream_encoder_unittest.cc", ] deps = [ + ":frame_cadence_adapter", ":video", ":video_legacy", ":video_mocks", diff --git a/video/frame_cadence_adapter.cc b/video/frame_cadence_adapter.cc new file mode 100644 index 0000000000..7c7ba94bd5 --- /dev/null +++ b/video/frame_cadence_adapter.cc @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/frame_cadence_adapter.h" + +#include + +#include "api/sequence_checker.h" +#include "rtc_base/logging.h" +#include "rtc_base/race_checker.h" +#include "rtc_base/synchronization/mutex.h" +#include "system_wrappers/include/field_trial.h" +#include "system_wrappers/include/metrics.h" + +namespace webrtc { +namespace { + +class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface { + public: + FrameCadenceAdapterImpl(); + + // FrameCadenceAdapterInterface overrides. + void Initialize(Callback* callback) override; + void SetZeroHertzModeEnabled(bool enabled) override; + + // VideoFrameSink overrides. + void OnFrame(const VideoFrame& frame) override; + void OnDiscardedFrame() override { callback_->OnDiscardedFrame(); } + void OnConstraintsChanged( + const VideoTrackSourceConstraints& constraints) override; + + private: + // Called to report on constraint UMAs. + void MaybeReportFrameRateConstraintUmas() + RTC_RUN_ON(&incoming_frame_race_checker_) RTC_LOCKS_EXCLUDED(mutex_); + + // True if we support frame entry for screenshare with a minimum frequency of + // 0 Hz. + const bool zero_hertz_screenshare_enabled_; + + // Set up during Initialize. + Callback* callback_ = nullptr; + + // Lock protecting zero-hertz activation state. This is needed because the + // threading contexts of OnFrame, OnConstraintsChanged, and ConfigureEncoder + // are mutating it. + Mutex mutex_; + + // The source's constraints. + absl::optional source_constraints_ + RTC_GUARDED_BY(mutex_); + + // Whether zero-hertz and UMA reporting is enabled. + bool zero_hertz_and_uma_reporting_enabled_ RTC_GUARDED_BY(mutex_) = false; + + // Race checker for incoming frames. This is the network thread in chromium, + // but may vary from test contexts. + rtc::RaceChecker incoming_frame_race_checker_; + bool has_reported_screenshare_frame_rate_umas_ RTC_GUARDED_BY(mutex_) = false; +}; + +FrameCadenceAdapterImpl::FrameCadenceAdapterImpl() + : zero_hertz_screenshare_enabled_( + field_trial::IsEnabled("WebRTC-ZeroHertzScreenshare")) {} + +void FrameCadenceAdapterImpl::Initialize(Callback* callback) { + callback_ = callback; +} + +void FrameCadenceAdapterImpl::SetZeroHertzModeEnabled(bool enabled) { + // This method is called on the worker thread. + MutexLock lock(&mutex_); + if (enabled && !zero_hertz_and_uma_reporting_enabled_) + has_reported_screenshare_frame_rate_umas_ = false; + zero_hertz_and_uma_reporting_enabled_ = enabled; +} + +void FrameCadenceAdapterImpl::OnFrame(const VideoFrame& frame) { + // This method is called on the network thread under Chromium, or other + // various contexts in test. + RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_); + callback_->OnFrame(frame); + MaybeReportFrameRateConstraintUmas(); +} + +void FrameCadenceAdapterImpl::OnConstraintsChanged( + const VideoTrackSourceConstraints& constraints) { + RTC_LOG(LS_INFO) << __func__ << " min_fps " + << constraints.min_fps.value_or(-1) << " max_fps " + << constraints.max_fps.value_or(-1); + MutexLock lock(&mutex_); + source_constraints_ = constraints; +} + +// RTC_RUN_ON(&incoming_frame_race_checker_) +void FrameCadenceAdapterImpl::MaybeReportFrameRateConstraintUmas() { + MutexLock lock(&mutex_); + if (has_reported_screenshare_frame_rate_umas_) + return; + has_reported_screenshare_frame_rate_umas_ = true; + if (!zero_hertz_and_uma_reporting_enabled_) + return; + RTC_HISTOGRAM_BOOLEAN("WebRTC.Screenshare.FrameRateConstraints.Exists", + source_constraints_.has_value()); + if (!source_constraints_.has_value()) + return; + RTC_HISTOGRAM_BOOLEAN("WebRTC.Screenshare.FrameRateConstraints.Min.Exists", + source_constraints_->min_fps.has_value()); + if (source_constraints_->min_fps.has_value()) { + RTC_HISTOGRAM_COUNTS_100( + "WebRTC.Screenshare.FrameRateConstraints.Min.Value", + source_constraints_->min_fps.value()); + } + RTC_HISTOGRAM_BOOLEAN("WebRTC.Screenshare.FrameRateConstraints.Max.Exists", + source_constraints_->max_fps.has_value()); + if (source_constraints_->max_fps.has_value()) { + RTC_HISTOGRAM_COUNTS_100( + "WebRTC.Screenshare.FrameRateConstraints.Max.Value", + source_constraints_->max_fps.value()); + } + if (!source_constraints_->min_fps.has_value()) { + if (source_constraints_->max_fps.has_value()) { + RTC_HISTOGRAM_COUNTS_100( + "WebRTC.Screenshare.FrameRateConstraints.MinUnset.Max", + source_constraints_->max_fps.value()); + } + } else if (source_constraints_->max_fps.has_value()) { + if (source_constraints_->min_fps.value() < + source_constraints_->max_fps.value()) { + RTC_HISTOGRAM_COUNTS_100( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Min", + source_constraints_->min_fps.value()); + RTC_HISTOGRAM_COUNTS_100( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Max", + source_constraints_->max_fps.value()); + } + // Multi-dimensional histogram for min and max FPS making it possible to + // uncover min and max combinations. See + // https://chromium.googlesource.com/chromium/src.git/+/HEAD/tools/metrics/histograms/README.md#multidimensional-histograms + constexpr int kMaxBucketCount = + 60 * /*max min_fps=*/60 + /*max max_fps=*/60 - 1; + RTC_HISTOGRAM_ENUMERATION_SPARSE( + "WebRTC.Screenshare.FrameRateConstraints.60MinPlusMaxMinusOne", + source_constraints_->min_fps.value() * 60 + + source_constraints_->max_fps.value() - 1, + /*boundary=*/kMaxBucketCount); + } +} + +} // namespace + +std::unique_ptr +FrameCadenceAdapterInterface::Create() { + return std::make_unique(); +} + +} // namespace webrtc diff --git a/video/frame_cadence_adapter.h b/video/frame_cadence_adapter.h new file mode 100644 index 0000000000..c0348c742e --- /dev/null +++ b/video/frame_cadence_adapter.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_FRAME_CADENCE_ADAPTER_H_ +#define VIDEO_FRAME_CADENCE_ADAPTER_H_ + +#include + +#include "api/video/video_frame.h" +#include "api/video/video_sink_interface.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/thread_annotations.h" + +namespace webrtc { + +// A sink adapter implementing mutations to the received frame cadence. +// With the exception of construction & destruction which has to happen on the +// same sequence, this class is thread-safe because three different execution +// contexts call into it. +class FrameCadenceAdapterInterface + : public rtc::VideoSinkInterface { + public: + // Callback interface used to inform instance owners. + class Callback { + public: + virtual ~Callback() = default; + + // Called when a frame arrives. + virtual void OnFrame(const VideoFrame& frame) = 0; + + // Called when the source has discarded a frame. + virtual void OnDiscardedFrame() = 0; + }; + + // Factory function creating a production instance. Deletion of the returned + // instance needs to happen on the same sequence that Create() was called on. + static std::unique_ptr Create(); + + // Call before using the rest of the API. + virtual void Initialize(Callback* callback) = 0; + + // Pass true in |enabled| as a prerequisite to enable zero-hertz operation. + virtual void SetZeroHertzModeEnabled(bool enabled) = 0; +}; + +} // namespace webrtc + +#endif // VIDEO_FRAME_CADENCE_ADAPTER_H_ diff --git a/video/frame_cadence_adapter_unittest.cc b/video/frame_cadence_adapter_unittest.cc new file mode 100644 index 0000000000..475425ac06 --- /dev/null +++ b/video/frame_cadence_adapter_unittest.cc @@ -0,0 +1,328 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/frame_cadence_adapter.h" + +#include +#include + +#include "api/video/nv12_buffer.h" +#include "api/video/video_frame.h" +#include "rtc_base/ref_counted_object.h" +#include "system_wrappers/include/metrics.h" +#include "test/field_trial.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +using ::testing::ElementsAre; +using ::testing::Mock; +using ::testing::Pair; +using ::testing::Ref; +using ::testing::UnorderedElementsAre; + +VideoFrame CreateFrame() { + return VideoFrame::Builder() + .set_video_frame_buffer( + rtc::make_ref_counted(/*width=*/16, /*height=*/16)) + .build(); +} + +class MockCallback : public FrameCadenceAdapterInterface::Callback { + public: + MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override)); + MOCK_METHOD(void, OnDiscardedFrame, (), (override)); +}; + +class ZeroHertzFieldTrialDisabler : public test::ScopedFieldTrials { + public: + ZeroHertzFieldTrialDisabler() + : test::ScopedFieldTrials("WebRTC-ZeroHertzScreenshare/Disabled/") {} +}; + +TEST(FrameCadenceAdapterTest, + ForwardsFramesOnConstructionAndUnderDisabledFieldTrial) { + auto disabler = std::make_unique(); + for (int i = 0; i != 2; i++) { + MockCallback callback; + auto adapter = FrameCadenceAdapterInterface::Create(); + adapter->Initialize(&callback); + VideoFrame frame = CreateFrame(); + EXPECT_CALL(callback, OnFrame(Ref(frame))).Times(1); + adapter->OnFrame(frame); + Mock::VerifyAndClearExpectations(&callback); + EXPECT_CALL(callback, OnDiscardedFrame).Times(1); + adapter->OnDiscardedFrame(); + Mock::VerifyAndClearExpectations(&callback); + + disabler = nullptr; + } +} + +class FrameCadenceAdapterMetricsTest : public ::testing::Test { + public: + FrameCadenceAdapterMetricsTest() { metrics::Reset(); } +}; + +TEST_F(FrameCadenceAdapterMetricsTest, RecordsNoUmasWithNoFrameTransfer) { + MockCallback callback; + auto adapter = FrameCadenceAdapterInterface::Create(); + adapter->Initialize(&callback); + adapter->OnConstraintsChanged( + VideoTrackSourceConstraints{absl::nullopt, absl::nullopt}); + adapter->OnConstraintsChanged(VideoTrackSourceConstraints{absl::nullopt, 1}); + adapter->OnConstraintsChanged(VideoTrackSourceConstraints{2, 3}); + adapter->OnConstraintsChanged(VideoTrackSourceConstraints{4, 4}); + adapter->OnConstraintsChanged(VideoTrackSourceConstraints{5, absl::nullopt}); + EXPECT_TRUE(metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Exists") + .empty()); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Exists") + .empty()); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Value") + .empty()); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Exists") + .empty()); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Value") + .empty()); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.MinUnset.Max") + .empty()); + EXPECT_TRUE(metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Min") + .empty()); + EXPECT_TRUE(metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Max") + .empty()); + EXPECT_TRUE( + metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.60MinPlusMaxMinusOne") + .empty()); +} + +TEST_F(FrameCadenceAdapterMetricsTest, RecordsNoUmasWithoutEnabledContentType) { + MockCallback callback; + auto adapter = FrameCadenceAdapterInterface::Create(); + adapter->Initialize(&callback); + adapter->OnFrame(CreateFrame()); + adapter->OnConstraintsChanged( + VideoTrackSourceConstraints{absl::nullopt, absl::nullopt}); + adapter->OnConstraintsChanged(VideoTrackSourceConstraints{absl::nullopt, 1}); + adapter->OnConstraintsChanged(VideoTrackSourceConstraints{2, 3}); + adapter->OnConstraintsChanged(VideoTrackSourceConstraints{4, 4}); + adapter->OnConstraintsChanged(VideoTrackSourceConstraints{5, absl::nullopt}); + EXPECT_TRUE(metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Exists") + .empty()); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Exists") + .empty()); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Value") + .empty()); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Exists") + .empty()); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Value") + .empty()); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.MinUnset.Max") + .empty()); + EXPECT_TRUE(metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Min") + .empty()); + EXPECT_TRUE(metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Max") + .empty()); + EXPECT_TRUE( + metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.60MinPlusMaxMinusOne") + .empty()); +} + +TEST_F(FrameCadenceAdapterMetricsTest, RecordsNoConstraintsIfUnsetOnFrame) { + MockCallback callback; + auto adapter = FrameCadenceAdapterInterface::Create(); + adapter->Initialize(&callback); + adapter->SetZeroHertzModeEnabled(true); + adapter->OnFrame(CreateFrame()); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Exists"), + ElementsAre(Pair(false, 1))); +} + +TEST_F(FrameCadenceAdapterMetricsTest, RecordsEmptyConstraintsIfSetOnFrame) { + MockCallback callback; + auto adapter = FrameCadenceAdapterInterface::Create(); + adapter->Initialize(&callback); + adapter->SetZeroHertzModeEnabled(true); + adapter->OnConstraintsChanged( + VideoTrackSourceConstraints{absl::nullopt, absl::nullopt}); + adapter->OnFrame(CreateFrame()); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Exists"), + ElementsAre(Pair(true, 1))); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Exists"), + ElementsAre(Pair(false, 1))); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Value") + .empty()); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Exists"), + ElementsAre(Pair(false, 1))); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Value") + .empty()); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.MinUnset.Max") + .empty()); + EXPECT_TRUE(metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Min") + .empty()); + EXPECT_TRUE(metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Max") + .empty()); + EXPECT_TRUE( + metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.60MinPlusMaxMinusOne") + .empty()); +} + +TEST_F(FrameCadenceAdapterMetricsTest, RecordsMaxConstraintIfSetOnFrame) { + MockCallback callback; + auto adapter = FrameCadenceAdapterInterface::Create(); + adapter->Initialize(&callback); + adapter->SetZeroHertzModeEnabled(true); + adapter->OnConstraintsChanged( + VideoTrackSourceConstraints{absl::nullopt, 2.0}); + adapter->OnFrame(CreateFrame()); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Exists"), + ElementsAre(Pair(false, 1))); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Value") + .empty()); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Exists"), + ElementsAre(Pair(true, 1))); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Value"), + ElementsAre(Pair(2.0, 1))); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.MinUnset.Max"), + ElementsAre(Pair(2.0, 1))); + EXPECT_TRUE(metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Min") + .empty()); + EXPECT_TRUE(metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Max") + .empty()); + EXPECT_TRUE( + metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.60MinPlusMaxMinusOne") + .empty()); +} + +TEST_F(FrameCadenceAdapterMetricsTest, RecordsMinConstraintIfSetOnFrame) { + MockCallback callback; + auto adapter = FrameCadenceAdapterInterface::Create(); + adapter->Initialize(&callback); + adapter->SetZeroHertzModeEnabled(true); + adapter->OnConstraintsChanged( + VideoTrackSourceConstraints{3.0, absl::nullopt}); + adapter->OnFrame(CreateFrame()); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Exists"), + ElementsAre(Pair(true, 1))); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Value"), + ElementsAre(Pair(3.0, 1))); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Exists"), + ElementsAre(Pair(false, 1))); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Value") + .empty()); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.MinUnset.Max") + .empty()); + EXPECT_TRUE(metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Min") + .empty()); + EXPECT_TRUE(metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Max") + .empty()); + EXPECT_TRUE( + metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.60MinPlusMaxMinusOne") + .empty()); +} + +TEST_F(FrameCadenceAdapterMetricsTest, RecordsMinGtMaxConstraintIfSetOnFrame) { + MockCallback callback; + auto adapter = FrameCadenceAdapterInterface::Create(); + adapter->Initialize(&callback); + adapter->SetZeroHertzModeEnabled(true); + adapter->OnConstraintsChanged(VideoTrackSourceConstraints{5.0, 4.0}); + adapter->OnFrame(CreateFrame()); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Exists"), + ElementsAre(Pair(true, 1))); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Min.Value"), + ElementsAre(Pair(5.0, 1))); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Exists"), + ElementsAre(Pair(true, 1))); + EXPECT_THAT( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.Max.Value"), + ElementsAre(Pair(4.0, 1))); + EXPECT_TRUE( + metrics::Samples("WebRTC.Screenshare.FrameRateConstraints.MinUnset.Max") + .empty()); + EXPECT_TRUE(metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Min") + .empty()); + EXPECT_TRUE(metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Max") + .empty()); + EXPECT_THAT( + metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.60MinPlusMaxMinusOne"), + ElementsAre(Pair(60 * 5.0 + 4.0 - 1, 1))); +} + +TEST_F(FrameCadenceAdapterMetricsTest, RecordsMinLtMaxConstraintIfSetOnFrame) { + MockCallback callback; + auto adapter = FrameCadenceAdapterInterface::Create(); + adapter->Initialize(&callback); + adapter->SetZeroHertzModeEnabled(true); + adapter->OnConstraintsChanged(VideoTrackSourceConstraints{4.0, 5.0}); + adapter->OnFrame(CreateFrame()); + EXPECT_THAT(metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Min"), + ElementsAre(Pair(4.0, 1))); + EXPECT_THAT(metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Max"), + ElementsAre(Pair(5.0, 1))); + EXPECT_THAT( + metrics::Samples( + "WebRTC.Screenshare.FrameRateConstraints.60MinPlusMaxMinusOne"), + ElementsAre(Pair(60 * 4.0 + 5.0 - 1, 1))); +} + +} // namespace +} // namespace webrtc diff --git a/video/test/mock_video_stream_encoder.h b/video/test/mock_video_stream_encoder.h index 2af613e3ad..8ea87acc0f 100644 --- a/video/test/mock_video_stream_encoder.h +++ b/video/test/mock_video_stream_encoder.h @@ -43,7 +43,6 @@ class MockVideoStreamEncoder : public VideoStreamEncoderInterface { OnBitrateUpdated, (DataRate, DataRate, DataRate, uint8_t, int64_t, double), (override)); - MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override)); MOCK_METHOD(void, SetFecControllerOverride, (FecControllerOverride*), diff --git a/video/video_send_stream.cc b/video/video_send_stream.cc index 10c209f143..e62a666802 100644 --- a/video/video_send_stream.cc +++ b/video/video_send_stream.cc @@ -23,6 +23,7 @@ #include "system_wrappers/include/clock.h" #include "system_wrappers/include/field_trial.h" #include "video/adaptation/overuse_frame_detector.h" +#include "video/frame_cadence_adapter.h" #include "video/video_stream_encoder.h" namespace webrtc { @@ -135,6 +136,7 @@ VideoSendStream::VideoSendStream( &stats_proxy_, config_.encoder_settings, std::make_unique(&stats_proxy_), + FrameCadenceAdapterInterface::Create(), task_queue_factory, network_queue, GetBitrateAllocationCallbackType(config_))), diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 7e0e7fc63e..010c1d1ffb 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -592,6 +592,7 @@ VideoStreamEncoder::VideoStreamEncoder( VideoStreamEncoderObserver* encoder_stats_observer, const VideoStreamEncoderSettings& settings, std::unique_ptr overuse_detector, + std::unique_ptr frame_cadence_adapter, TaskQueueFactory* task_queue_factory, TaskQueueBase* network_queue, BitrateAllocationCallbackType allocation_cb_type) @@ -604,6 +605,8 @@ VideoStreamEncoder::VideoStreamEncoder( rate_control_settings_(RateControlSettings::ParseFromFieldTrials()), encoder_selector_(settings.encoder_factory->GetEncoderSelector()), encoder_stats_observer_(encoder_stats_observer), + cadence_callback_(*this), + frame_cadence_adapter_(std::move(frame_cadence_adapter)), encoder_initialized_(false), max_framerate_(-1), pending_encoder_reconfiguration_(false), @@ -657,7 +660,7 @@ VideoStreamEncoder::VideoStreamEncoder( settings_.experiment_cpu_load_estimator, std::move(overuse_detector), degradation_preference_manager_.get()), - video_source_sink_controller_(/*sink=*/this, + video_source_sink_controller_(/*sink=*/frame_cadence_adapter_.get(), /*source=*/nullptr), default_limits_allowed_( !field_trial::IsEnabled("WebRTC-DefaultBitrateLimitsKillSwitch")), @@ -671,6 +674,7 @@ VideoStreamEncoder::VideoStreamEncoder( RTC_DCHECK(encoder_stats_observer); RTC_DCHECK_GE(number_of_cores, 1); + frame_cadence_adapter_->Initialize(&cadence_callback_); stream_resource_manager_.Initialize(&encoder_queue_); rtc::Event initialize_processor_event; @@ -821,6 +825,8 @@ void VideoStreamEncoder::SetStartBitrate(int start_bitrate_bps) { void VideoStreamEncoder::ConfigureEncoder(VideoEncoderConfig config, size_t max_data_payload_length) { RTC_DCHECK_RUN_ON(worker_queue_); + frame_cadence_adapter_->SetZeroHertzModeEnabled( + config.content_type == VideoEncoderConfig::ContentType::kScreen); encoder_queue_.PostTask( [this, config = std::move(config), max_data_payload_length]() mutable { RTC_DCHECK_RUN_ON(&encoder_queue_); @@ -830,8 +836,6 @@ void VideoStreamEncoder::ConfigureEncoder(VideoEncoderConfig config, pending_encoder_creation_ = (!encoder_ || encoder_config_.video_format != config.video_format || max_data_payload_length_ != max_data_payload_length); - if (encoder_config_.content_type != config.content_type) - has_reported_screenshare_frame_rate_umas_ = false; encoder_config_ = std::move(config); max_data_payload_length_ = max_data_payload_length; pending_encoder_reconfiguration_ = true; @@ -1331,7 +1335,6 @@ void VideoStreamEncoder::OnFrame(const VideoFrame& video_frame) { encoder_queue_.PostTask( [this, incoming_frame, post_time_us, log_stats]() { RTC_DCHECK_RUN_ON(&encoder_queue_); - MaybeReportFrameRateConstraintUmas(); encoder_stats_observer_->OnIncomingFrame(incoming_frame.width(), incoming_frame.height()); ++captured_frame_count_; @@ -1382,18 +1385,6 @@ void VideoStreamEncoder::OnDiscardedFrame() { VideoStreamEncoderObserver::DropReason::kSource); } -void VideoStreamEncoder::OnConstraintsChanged( - const webrtc::VideoTrackSourceConstraints& constraints) { - RTC_DCHECK_RUN_ON(network_queue_); - RTC_LOG(LS_INFO) << __func__ << " min_fps " - << constraints.min_fps.value_or(-1) << " max_fps " - << constraints.max_fps.value_or(-1); - worker_queue_->PostTask(ToQueuedTask(task_safety_, [this, constraints] { - RTC_DCHECK_RUN_ON(worker_queue_); - source_constraints_ = constraints; - })); -} - bool VideoStreamEncoder::EncoderPaused() const { RTC_DCHECK_RUN_ON(&encoder_queue_); // Pause video if paused by caller or as long as the network is down or the @@ -2364,64 +2355,6 @@ void VideoStreamEncoder::QueueRequestEncoderSwitch( })); } -// RTC_RUN_ON(&encoder_queue_) -void VideoStreamEncoder::MaybeReportFrameRateConstraintUmas() { - if (has_reported_screenshare_frame_rate_umas_) - return; - has_reported_screenshare_frame_rate_umas_ = true; - bool is_screenshare = - encoder_config_.content_type == VideoEncoderConfig::ContentType::kScreen; - if (!is_screenshare) - return; - worker_queue_->PostTask(ToQueuedTask(task_safety_, [this] { - RTC_DCHECK_RUN_ON(worker_queue_); - RTC_HISTOGRAM_BOOLEAN("WebRTC.Screenshare.FrameRateConstraints.Exists", - source_constraints_.has_value()); - if (source_constraints_.has_value()) { - RTC_HISTOGRAM_BOOLEAN( - "WebRTC.Screenshare.FrameRateConstraints.Min.Exists", - source_constraints_->min_fps.has_value()); - if (source_constraints_->min_fps.has_value()) { - RTC_HISTOGRAM_COUNTS_100( - "WebRTC.Screenshare.FrameRateConstraints.Min.Value", - source_constraints_->min_fps.value()); - } - RTC_HISTOGRAM_BOOLEAN( - "WebRTC.Screenshare.FrameRateConstraints.Max.Exists", - source_constraints_->max_fps.has_value()); - if (source_constraints_->max_fps.has_value()) { - RTC_HISTOGRAM_COUNTS_100( - "WebRTC.Screenshare.FrameRateConstraints.Max.Value", - source_constraints_->max_fps.value()); - } - if (!source_constraints_->min_fps.has_value()) { - if (source_constraints_->max_fps.has_value()) { - RTC_HISTOGRAM_COUNTS_100( - "WebRTC.Screenshare.FrameRateConstraints.MinUnset.Max", - source_constraints_->max_fps.value()); - } - } else if (source_constraints_->max_fps.has_value()) { - if (source_constraints_->min_fps.value() < - source_constraints_->max_fps.value()) { - RTC_HISTOGRAM_COUNTS_100( - "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Min", - source_constraints_->min_fps.value()); - RTC_HISTOGRAM_COUNTS_100( - "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Max", - source_constraints_->max_fps.value()); - } - constexpr int kMaxBucketCount = - 60 * /*max min_fps=*/60 + /*max max_fps=*/60 - 1; - RTC_HISTOGRAM_ENUMERATION_SPARSE( - "WebRTC.Screenshare.FrameRateConstraints.60MinPlusMaxMinusOne", - source_constraints_->min_fps.value() * 60 + - source_constraints_->max_fps.value() - 1, - /*boundary=*/kMaxBucketCount); - } - } - })); -} - void VideoStreamEncoder::InjectAdaptationResource( rtc::scoped_refptr resource, VideoAdaptationReason reason) { diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 0738e1d2f5..9d6bb4bc2f 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -45,6 +45,7 @@ #include "system_wrappers/include/clock.h" #include "video/adaptation/video_stream_encoder_resource_manager.h" #include "video/encoder_bitrate_adjuster.h" +#include "video/frame_cadence_adapter.h" #include "video/frame_encode_metadata_writer.h" #include "video/video_source_sink_controller.h" @@ -69,14 +70,16 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, kVideoBitrateAllocationWhenScreenSharing, kVideoLayersAllocation }; - VideoStreamEncoder(Clock* clock, - uint32_t number_of_cores, - VideoStreamEncoderObserver* encoder_stats_observer, - const VideoStreamEncoderSettings& settings, - std::unique_ptr overuse_detector, - TaskQueueFactory* task_queue_factory, - TaskQueueBase* network_queue, - BitrateAllocationCallbackType allocation_cb_type); + VideoStreamEncoder( + Clock* clock, + uint32_t number_of_cores, + VideoStreamEncoderObserver* encoder_stats_observer, + const VideoStreamEncoderSettings& settings, + std::unique_ptr overuse_detector, + std::unique_ptr frame_cadence_adapter, + TaskQueueFactory* task_queue_factory, + TaskQueueBase* network_queue, + BitrateAllocationCallbackType allocation_cb_type); ~VideoStreamEncoder() override; void AddAdaptationResource(rtc::scoped_refptr resource) override; @@ -138,6 +141,22 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, VideoSourceRestrictionsListener* restrictions_listener); private: + class CadenceCallback : public FrameCadenceAdapterInterface::Callback { + public: + explicit CadenceCallback(VideoStreamEncoder& video_stream_encoder) + : video_stream_encoder_(video_stream_encoder) {} + // FrameCadenceAdapterInterface::Callback overrides. + void OnFrame(const VideoFrame& frame) override { + video_stream_encoder_.OnFrame(frame); + } + void OnDiscardedFrame() override { + video_stream_encoder_.OnDiscardedFrame(); + } + + private: + VideoStreamEncoder& video_stream_encoder_; + }; + class VideoFrameInfo { public: VideoFrameInfo(int width, int height, bool is_texture) @@ -173,12 +192,8 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, void ReconfigureEncoder() RTC_RUN_ON(&encoder_queue_); void OnEncoderSettingsChanged() RTC_RUN_ON(&encoder_queue_); - - // Implements VideoSinkInterface. - void OnFrame(const VideoFrame& video_frame) override; - void OnDiscardedFrame() override; - void OnConstraintsChanged( - const webrtc::VideoTrackSourceConstraints& constraints) override; + void OnFrame(const VideoFrame& video_frame); + void OnDiscardedFrame(); void MaybeEncodeVideoFrame(const VideoFrame& frame, int64_t time_when_posted_in_ms); @@ -229,9 +244,6 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, void QueueRequestEncoderSwitch(const webrtc::SdpVideoFormat& format) RTC_RUN_ON(&encoder_queue_); - // Reports UMAs on frame rate constraints usage on the first call. - void MaybeReportFrameRateConstraintUmas() RTC_RUN_ON(&encoder_queue_); - TaskQueueBase* const worker_queue_; TaskQueueBase* const network_queue_; @@ -245,12 +257,12 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, std::unique_ptr const encoder_selector_; VideoStreamEncoderObserver* const encoder_stats_observer_; - - // The source's constraints. - absl::optional source_constraints_ - RTC_GUARDED_BY(worker_queue_); - bool has_reported_screenshare_frame_rate_umas_ - RTC_GUARDED_BY(&encoder_queue_) = false; + // Adapter that avoids public inheritance of the cadence adapter's callback + // interface. + CadenceCallback cadence_callback_; + // Frame cadence encoder adapter. Frames enter this adapter first, and it then + // forwards them to our OnFrame method. + const std::unique_ptr frame_cadence_adapter_; VideoEncoderConfig encoder_config_ RTC_GUARDED_BY(&encoder_queue_); std::unique_ptr encoder_ RTC_GUARDED_BY(&encoder_queue_) diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index b41b4b0d1d..d143a639c5 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -18,6 +18,7 @@ #include "absl/memory/memory.h" #include "api/task_queue/default_task_queue_factory.h" +#include "api/task_queue/task_queue_factory.h" #include "api/test/mock_fec_controller_override.h" #include "api/test/mock_video_encoder.h" #include "api/test/mock_video_encoder_factory.h" @@ -62,6 +63,7 @@ #include "test/mappable_native_buffer.h" #include "test/time_controller/simulated_time_controller.h" #include "test/video_encoder_proxy_factory.h" +#include "video/frame_cadence_adapter.h" #include "video/send_statistics_proxy.h" namespace webrtc { @@ -76,7 +78,9 @@ using ::testing::Gt; using ::testing::Le; using ::testing::Lt; using ::testing::Matcher; +using ::testing::Mock; using ::testing::NiceMock; +using ::testing::Optional; using ::testing::Return; using ::testing::SizeIs; using ::testing::StrictMock; @@ -355,6 +359,7 @@ class VideoStreamEncoderUnderTest : public VideoStreamEncoder { std::unique_ptr( overuse_detector_proxy_ = new CpuOveruseDetectorProxy(stats_proxy)), + FrameCadenceAdapterInterface::Create(), task_queue_factory, TaskQueueBase::Current(), allocation_callback_type), @@ -626,6 +631,79 @@ class MockableSendStatisticsProxy : public SendStatisticsProxy { std::function on_frame_dropped_; }; +class SimpleVideoStreamEncoderFactory { + public: + class AdaptedVideoStreamEncoder : public VideoStreamEncoder { + public: + using VideoStreamEncoder::VideoStreamEncoder; + ~AdaptedVideoStreamEncoder() { Stop(); } + }; + + SimpleVideoStreamEncoderFactory() + : time_controller_(Timestamp::Millis(0)), + task_queue_factory_(time_controller_.CreateTaskQueueFactory()), + stats_proxy_(std::make_unique( + time_controller_.GetClock(), + VideoSendStream::Config(nullptr), + webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)), + encoder_settings_( + VideoEncoder::Capabilities(/*loss_notification=*/false)), + fake_encoder_(time_controller_.GetClock()), + encoder_factory_(&fake_encoder_) { + encoder_settings_.encoder_factory = &encoder_factory_; + } + + std::unique_ptr Create( + std::unique_ptr zero_hertz_adapter) { + auto result = std::make_unique( + time_controller_.GetClock(), + /*number_of_cores=*/1, + /*stats_proxy=*/stats_proxy_.get(), encoder_settings_, + std::make_unique(/*stats_proxy=*/nullptr), + std::move(zero_hertz_adapter), task_queue_factory_.get(), + TaskQueueBase::Current(), + VideoStreamEncoder::BitrateAllocationCallbackType:: + kVideoBitrateAllocation); + result->SetSink(&sink_, /*rotation_applied=*/false); + return result; + } + + private: + class NullEncoderSink : public VideoStreamEncoderInterface::EncoderSink { + public: + ~NullEncoderSink() override = default; + void OnEncoderConfigurationChanged( + std::vector streams, + bool is_svc, + VideoEncoderConfig::ContentType content_type, + int min_transmit_bitrate_bps) override {} + void OnBitrateAllocationUpdated( + const VideoBitrateAllocation& allocation) override {} + void OnVideoLayersAllocationUpdated( + VideoLayersAllocation allocation) override {} + Result OnEncodedImage( + const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info) override { + return Result(EncodedImageCallback::Result::OK); + } + }; + + GlobalSimulatedTimeController time_controller_; + std::unique_ptr task_queue_factory_; + std::unique_ptr stats_proxy_; + VideoStreamEncoderSettings encoder_settings_; + test::FakeEncoder fake_encoder_; + test::VideoEncoderProxyFactory encoder_factory_; + NullEncoderSink sink_; +}; + +class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface { + public: + MOCK_METHOD(void, Initialize, (Callback * callback), (override)); + MOCK_METHOD(void, SetZeroHertzModeEnabled, (bool), (override)); + MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override)); +}; + class MockEncoderSelector : public VideoEncoderFactory::EncoderSelectorInterface { public: @@ -8709,4 +8787,43 @@ TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) { RunTest({config1, config2}, /*expected_num_init_encode=*/2); } +TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) { + auto adapter = std::make_unique(); + auto* adapter_ptr = adapter.get(); + SimpleVideoStreamEncoderFactory factory; + auto video_stream_encoder = factory.Create(std::move(adapter)); + + EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(true)); + VideoEncoderConfig config; + config.content_type = VideoEncoderConfig::ContentType::kScreen; + video_stream_encoder->ConfigureEncoder(std::move(config), 0); + Mock::VerifyAndClearExpectations(adapter_ptr); + + EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(false)); + VideoEncoderConfig config2; + config2.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo; + video_stream_encoder->ConfigureEncoder(std::move(config2), 0); +} + +TEST(VideoStreamEncoderFrameCadenceTest, + ForwardsFramesIntoFrameCadenceAdapter) { + auto adapter = std::make_unique(); + auto* adapter_ptr = adapter.get(); + test::FrameForwarder video_source; + SimpleVideoStreamEncoderFactory factory; + auto video_stream_encoder = factory.Create(std::move(adapter)); + video_stream_encoder->SetSource( + &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE); + + EXPECT_CALL(*adapter_ptr, OnFrame); + auto buffer = rtc::make_ref_counted(/*width=*/16, /*height=*/16); + video_source.IncomingCapturedFrame( + VideoFrame::Builder() + .set_video_frame_buffer(std::move(buffer)) + .set_ntp_time_ms(0) + .set_timestamp_ms(0) + .set_rotation(kVideoRotation_0) + .build()); +} + } // namespace webrtc