diff --git a/common_video/generic_frame_descriptor/generic_frame_info.h b/common_video/generic_frame_descriptor/generic_frame_info.h index 3a570a6ee0..19f413b5d4 100644 --- a/common_video/generic_frame_descriptor/generic_frame_info.h +++ b/common_video/generic_frame_descriptor/generic_frame_info.h @@ -11,6 +11,7 @@ #ifndef COMMON_VIDEO_GENERIC_FRAME_DESCRIPTOR_GENERIC_FRAME_INFO_H_ #define COMMON_VIDEO_GENERIC_FRAME_DESCRIPTOR_GENERIC_FRAME_INFO_H_ +#include #include #include @@ -40,6 +41,7 @@ struct GenericFrameInfo : public FrameDependencyTemplate { absl::InlinedVector encoder_buffers; std::vector part_of_chain; + std::bitset<32> active_decode_targets = ~uint32_t{0}; }; class GenericFrameInfo::Builder { diff --git a/modules/video_coding/codecs/av1/BUILD.gn b/modules/video_coding/codecs/av1/BUILD.gn index 128ed8d0bc..a927db293d 100644 --- a/modules/video_coding/codecs/av1/BUILD.gn +++ b/modules/video_coding/codecs/av1/BUILD.gn @@ -42,6 +42,7 @@ rtc_source_set("scalable_video_controller") { ] deps = [ "../../../../api/transport/rtp:dependency_descriptor", + "../../../../api/video:video_bitrate_allocation", "../../../../common_video/generic_frame_descriptor", "../../../../rtc_base:checks", ] diff --git a/modules/video_coding/codecs/av1/libaom_av1_encoder.cc b/modules/video_coding/codecs/av1/libaom_av1_encoder.cc index 0ba515e51b..0b2c2dacf7 100644 --- a/modules/video_coding/codecs/av1/libaom_av1_encoder.cc +++ b/modules/video_coding/codecs/av1/libaom_av1_encoder.cc @@ -544,6 +544,7 @@ void LibaomAv1Encoder::SetRates(const RateControlParameters& parameters) { RTC_DCHECK_LE(rc_target_bitrate_kbps, encoder_settings_.maxBitrate); RTC_DCHECK_GE(rc_target_bitrate_kbps, encoder_settings_.minBitrate); + svc_controller_->OnRatesUpdated(parameters.bitrate); // Set target bit rate. cfg_.rc_target_bitrate = rc_target_bitrate_kbps; diff --git a/modules/video_coding/codecs/av1/libaom_av1_encoder_unittest.cc b/modules/video_coding/codecs/av1/libaom_av1_encoder_unittest.cc index 492e8f006a..341a82774d 100644 --- a/modules/video_coding/codecs/av1/libaom_av1_encoder_unittest.cc +++ b/modules/video_coding/codecs/av1/libaom_av1_encoder_unittest.cc @@ -11,15 +11,38 @@ #include "modules/video_coding/codecs/av1/libaom_av1_encoder.h" #include +#include +#include "absl/types/optional.h" #include "api/video_codecs/video_codec.h" #include "api/video_codecs/video_encoder.h" +#include "modules/video_coding/codecs/av1/scalability_structure_l1t2.h" +#include "modules/video_coding/codecs/test/encoded_video_frame_producer.h" #include "modules/video_coding/include/video_error_codes.h" +#include "test/gmock.h" #include "test/gtest.h" namespace webrtc { namespace { +using ::testing::SizeIs; + +VideoCodec DefaultCodecSettings() { + VideoCodec codec_settings; + codec_settings.width = 320; + codec_settings.height = 180; + codec_settings.maxFramerate = 30; + codec_settings.maxBitrate = 1000; + codec_settings.qpMax = 63; + return codec_settings; +} + +VideoEncoder::Settings DefaultEncoderSettings() { + return VideoEncoder::Settings( + VideoEncoder::Capabilities(/*loss_notification=*/false), + /*number_of_cores=*/1, /*max_payload_size=*/1200); +} + TEST(LibaomAv1EncoderTest, CanCreate) { std::unique_ptr encoder = CreateLibaomAv1Encoder(); EXPECT_TRUE(encoder); @@ -28,18 +51,37 @@ TEST(LibaomAv1EncoderTest, CanCreate) { TEST(LibaomAv1EncoderTest, InitAndRelease) { std::unique_ptr encoder = CreateLibaomAv1Encoder(); ASSERT_TRUE(encoder); - VideoCodec codec_settings; - codec_settings.width = 1280; - codec_settings.height = 720; - codec_settings.maxFramerate = 30; - codec_settings.qpMax = 63; - VideoEncoder::Capabilities capabilities(/*loss_notification=*/false); - VideoEncoder::Settings encoder_settings(capabilities, /*number_of_cores=*/1, - /*max_payload_size=*/1200); - EXPECT_EQ(encoder->InitEncode(&codec_settings, encoder_settings), + VideoCodec codec_settings = DefaultCodecSettings(); + EXPECT_EQ(encoder->InitEncode(&codec_settings, DefaultEncoderSettings()), WEBRTC_VIDEO_CODEC_OK); EXPECT_EQ(encoder->Release(), WEBRTC_VIDEO_CODEC_OK); } +TEST(LibaomAv1EncoderTest, NoBitrateOnTopLayerRefecltedInActiveDecodeTargets) { + // Configure encoder with 2 temporal layers. + std::unique_ptr encoder = + CreateLibaomAv1Encoder(std::make_unique()); + VideoCodec codec_settings = DefaultCodecSettings(); + ASSERT_EQ(encoder->InitEncode(&codec_settings, DefaultEncoderSettings()), + WEBRTC_VIDEO_CODEC_OK); + + VideoEncoder::RateControlParameters rate_parameters; + rate_parameters.framerate_fps = 30; + rate_parameters.bitrate.SetBitrate(0, /*temporal_index=*/0, 300'000); + rate_parameters.bitrate.SetBitrate(0, /*temporal_index=*/1, 0); + encoder->SetRates(rate_parameters); + + std::vector encoded_frames = + EncodedVideoFrameProducer(*encoder).SetNumInputFrames(1).Encode(); + ASSERT_THAT(encoded_frames, SizeIs(1)); + ASSERT_NE(encoded_frames[0].codec_specific_info.generic_frame_info, + absl::nullopt); + // Assuming L1T2 structure uses 1st decode target for T0 and 2nd decode target + // for T0+T1 frames, expect only 1st decode target is active. + EXPECT_EQ(encoded_frames[0] + .codec_specific_info.generic_frame_info->active_decode_targets, + 0b01); +} + } // namespace } // namespace webrtc diff --git a/modules/video_coding/codecs/av1/scalability_structure_l1t2.cc b/modules/video_coding/codecs/av1/scalability_structure_l1t2.cc index b38a61637e..ae4c879224 100644 --- a/modules/video_coding/codecs/av1/scalability_structure_l1t2.cc +++ b/modules/video_coding/codecs/av1/scalability_structure_l1t2.cc @@ -56,8 +56,15 @@ FrameDependencyStructure ScalabilityStructureL1T2::DependencyStructure() const { std::vector ScalabilityStructureL1T2::NextFrameConfig(bool restart) { + if (!active_decode_targets_[0]) { + RTC_LOG(LS_WARNING) << "No bitrate allocated for temporal layer 0, yet " + "frame is requested. No frame will be encoded."; + return {}; + } if (restart) { next_pattern_ = kKeyFrame; + } else if (!active_decode_targets_[1]) { + next_pattern_ = kDeltaFrameT0; } std::vector result(1); @@ -97,7 +104,20 @@ absl::optional ScalabilityStructureL1T2::OnEncodeDone( frame_info->decode_target_indications.assign(std::begin(kDtis[config.Id()]), std::end(kDtis[config.Id()])); frame_info->part_of_chain = {config.TemporalId() == 0}; + frame_info->active_decode_targets = active_decode_targets_; return frame_info; } +void ScalabilityStructureL1T2::OnRatesUpdated( + const VideoBitrateAllocation& bitrates) { + if (bitrates.GetBitrate(0, 0) == 0) { + // It is unclear what frame can be produced when base layer is disabled, + // so mark all decode targets as inactive to produce no frames. + active_decode_targets_.reset(); + return; + } + active_decode_targets_.set(0, true); + active_decode_targets_.set(1, bitrates.GetBitrate(0, 1) > 0); +} + } // namespace webrtc diff --git a/modules/video_coding/codecs/av1/scalability_structure_l1t2.h b/modules/video_coding/codecs/av1/scalability_structure_l1t2.h index 72a9659c34..55a9e8bbb0 100644 --- a/modules/video_coding/codecs/av1/scalability_structure_l1t2.h +++ b/modules/video_coding/codecs/av1/scalability_structure_l1t2.h @@ -10,6 +10,7 @@ #ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L1T2_H_ #define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L1T2_H_ +#include #include #include "api/transport/rtp/dependency_descriptor.h" @@ -29,6 +30,8 @@ class ScalabilityStructureL1T2 : public ScalableVideoController { absl::optional OnEncodeDone( LayerFrameConfig config) override; + void OnRatesUpdated(const VideoBitrateAllocation& bitrates) override; + private: enum FramePattern { kKeyFrame, @@ -37,6 +40,7 @@ class ScalabilityStructureL1T2 : public ScalableVideoController { }; FramePattern next_pattern_ = kKeyFrame; + std::bitset<32> active_decode_targets_ = 0b11; }; } // namespace webrtc diff --git a/modules/video_coding/codecs/av1/scalable_video_controller.h b/modules/video_coding/codecs/av1/scalable_video_controller.h index 6b2c00b346..d10aca2ce5 100644 --- a/modules/video_coding/codecs/av1/scalable_video_controller.h +++ b/modules/video_coding/codecs/av1/scalable_video_controller.h @@ -15,6 +15,7 @@ #include "absl/container/inlined_vector.h" #include "absl/types/optional.h" #include "api/transport/rtp/dependency_descriptor.h" +#include "api/video/video_bitrate_allocation.h" #include "common_video/generic_frame_descriptor/generic_frame_info.h" namespace webrtc { @@ -77,6 +78,13 @@ class ScalableVideoController { // dependency descriptor rtp header extension. virtual FrameDependencyStructure DependencyStructure() const = 0; + // Notifies Controller with updated bitrates per layer. In particular notifies + // when certain layers should be disabled. + // Controller shouldn't produce LayerFrameConfig for disabled layers. + // TODO(bugs.webrtc.org/11404): Make pure virtual when implemented by all + // structures. + virtual void OnRatesUpdated(const VideoBitrateAllocation& bitrates) {} + // When `restart` is true, first `LayerFrameConfig` should have `is_keyframe` // set to true. // Returned vector shouldn't be empty.